Path:
strictdoc/core/traceability_index_builder.py
Lines:
988
Non-empty lines:
898
Non-empty lines covered with requirements:
898 / 898 (100.0%)
Functions:
8
Functions covered by requirements:
8 / 8 (100.0%)
- "4.1. Traceability index" (REQUIREMENT)
- "14.3. Incremental generation of documents" (REQUIREMENT)
1
"""2
@relation(SDOC-SRS-28, SDOC-SRS-2, scope=file)3
"""4
5
import datetime
6
import glob
7
import os
8
import posixpath
9
import sys
10
from typing import Any, Dict, Iterator, List, Optional, Set, Union
11
12
from textx import TextXSyntaxError
13
14
from strictdoc.backend.markdown.reader import SDMarkdownReader
15
from strictdoc.backend.sdoc.constants import SDocMarkup
16
from strictdoc.backend.sdoc.error_handling import StrictDocSemanticError
17
from strictdoc.backend.sdoc.models.anchor import Anchor
18
from strictdoc.backend.sdoc.models.document import SDocDocument
19
from strictdoc.backend.sdoc.models.document_from_file import DocumentFromFile
20
from strictdoc.backend.sdoc.models.document_grammar import DocumentGrammar
21
from strictdoc.backend.sdoc.models.grammar_element import (
22
GrammarElement,
23
ReferenceType,
24
)25
from strictdoc.backend.sdoc.models.inline_link import InlineLink
26
from strictdoc.backend.sdoc.models.model import (
27
SDocDocumentFromFileIF,
28
SDocElementIF,
29
SDocNodeIF,
30
)31
from strictdoc.backend.sdoc.models.node import SDocNode
32
from strictdoc.backend.sdoc.models.reference import (
33
ChildReqReference,
34
ParentReqReference,
35
)36
from strictdoc.backend.sdoc.node_filter import NodeFilter
37
from strictdoc.backend.sdoc.validations.sdoc_validator import SDocValidator
38
from strictdoc.backend.sdoc_source_code.caching_reader import (
39
SourceFileTraceabilityCachingReader,
40
)41
from strictdoc.core.constants import GraphLinkType
42
from strictdoc.core.document_iterator import SDocDocumentIterator
43
from strictdoc.core.document_tree import DocumentTree
44
from strictdoc.core.file_dependency_manager import FileDependencyManager
45
from strictdoc.core.file_system.document_finder import DocumentFinder
46
from strictdoc.core.file_system.source_files_finder import (
47
SourceFilesFinder,
48
)49
from strictdoc.core.file_system.source_tree import SourceFile, SourceTree
50
from strictdoc.core.file_traceability_index import FileTraceabilityIndex
51
from strictdoc.core.graph.many_to_many_set import ManyToManySet
52
from strictdoc.core.graph.one_to_one_dictionary import OneToOneDictionary
53
from strictdoc.core.graph_database import GraphDatabase
54
from strictdoc.core.project_config import (
55
ProjectConfig,
56
ProjectFeature,
57
SourceNodesEntry,
58
)59
from strictdoc.core.query_engine.query_object import (
60
QueryNullObject,
61
QueryObject,
62
)63
from strictdoc.core.query_engine.query_reader import QueryReader
64
from strictdoc.core.traceability_index import (
65
TraceabilityIndex,
66
)67
from strictdoc.core.tree_cycle_detector import TreeCycleDetector
68
from strictdoc.helpers.cast import assert_cast
69
from strictdoc.helpers.deprecation_engine import DEPRECATION_ENGINE
70
from strictdoc.helpers.exception import StrictDocException
71
from strictdoc.helpers.file_modification_time import (
72
get_file_modification_time,
73
)74
from strictdoc.helpers.mid import MID
75
from strictdoc.helpers.parallelizer import Parallelizer
76
from strictdoc.helpers.paths import shorten_path
77
from strictdoc.helpers.timing import (
78
measure_performance,
79
measure_performance_loop,
80
timing_decorator,
81
)82
83
84
class TraceabilityIndexBuilder:
85
@classmethod86
def create(
87
cls,
88
*,
89
project_config: ProjectConfig,
90
parallelizer: Parallelizer,
91
skip_source_files: bool = False,
92
) -> TraceabilityIndex:
93
# TODO: It would be great to hide this code behind --development flag.94
# There is no need for this to be activated in the Pip-released builds.95
strict_own_files_unfiltered: Iterator[str] = glob.iglob(
96
f"{project_config.get_strictdoc_root_path()}/strictdoc/**/*",
97
recursive=True,
98
)99
strict_own_files: List[str] = [
100
f101
for f in strict_own_files_unfiltered
102
if f.endswith(".html")
103
or f.endswith(".py")
104
or f.endswith(".jinja")
105
or f.endswith(".svg")
106
]107
latest_strictdoc_own_file = (
108
max(strict_own_files, key=os.path.getctime)
109
if len(strict_own_files) > 0
110
else None
111
)112
113
strictdoc_last_update: datetime.datetime = (
114
get_file_modification_time(latest_strictdoc_own_file)
115
if (latest_strictdoc_own_file is not None)
116
else datetime.datetime.fromtimestamp(0)
117
)118
if (
119
project_config.config_last_update is not None
120
and project_config.config_last_update > strictdoc_last_update
121
):122
strictdoc_last_update = project_config.config_last_update
123
124
document_tree, asset_manager = DocumentFinder.find_sdoc_content(
125
project_config=project_config, parallelizer=parallelizer
126
)127
128
# TODO: This is rather messy, but it is better than it used to be.129
# Currently, the traceability index holds everything that is later used130
# by HTML generators:131
# - traceability index itself132
# - document tree133
# - assets134
# - runtime configuration.135
traceability_index: TraceabilityIndex = (
136
TraceabilityIndexBuilder.create_from_document_tree(
137
document_tree,
138
project_config,
139
)140
)141
traceability_index.asset_manager = asset_manager
142
traceability_index.strictdoc_last_update = strictdoc_last_update
143
144
if node_filter_query := project_config.filter_nodes:
145
traceability_index.node_filter = cls._create_filter(
146
traceability_index=traceability_index,
147
filter_query=node_filter_query,
148
)149
150
#151
# File traceability-related calculations.152
#153
if not skip_source_files and project_config.is_feature_activated(
154
ProjectFeature.REQUIREMENT_TO_SOURCE_TRACEABILITY
155
):156
file_tracability_index = (
157
traceability_index.get_file_traceability_index()
158
)159
160
with measure_performance("Find source files"):
161
source_tree: SourceTree = SourceFilesFinder.find_source_files(
162
project_config=project_config
163
)164
165
source_files = source_tree.source_files
166
source_file: SourceFile
167
with measure_performance_loop(
168
"Reading source", len(source_files)
169
) as report_progress:
170
for source_file in source_files:
171
with report_progress(
172
source_file.in_doctree_source_file_rel_path,
173
short_title=shorten_path(
174
source_file.in_doctree_source_file_rel_path
175
),176
):177
source_nodes_cfg_entry = (
178
project_config.get_relevant_source_nodes_entry(
179
source_file.full_path
180
)181
)182
if source_nodes_cfg_entry is not None:
183
source_node_grammar_element = (
184
traceability_index.get_grammar_element(
185
source_nodes_cfg_entry.uid,
186
source_nodes_cfg_entry.node_type,
187
)188
)189
assert source_node_grammar_element is not None, (
190
"Missing grammar element for node: "191
f"{source_nodes_cfg_entry.uid} {source_nodes_cfg_entry.node_type}"
192
)193
source_node_tags = TraceabilityIndexBuilder.source_node_parser_tags(
194
source_nodes_cfg_entry,
195
source_node_grammar_element,
196
)197
else:
198
source_node_tags = None
199
200
traceability_info = (
201
SourceFileTraceabilityCachingReader.read_from_file(
202
source_file.full_path,
203
project_config,
204
source_node_tags,
205
)206
)207
208
if traceability_info:
209
traceability_index.create_traceability_info(
210
source_file,
211
traceability_info,
212
)213
# Is file referenced by backwards links?214
if len(traceability_info.markers) > 0:
215
source_file.is_referenced = True
216
217
file_tracability_index.validate_and_resolve(
218
traceability_index, project_config
219
)220
221
# Iterate again to resolve if the file is referenced.222
# FIXME: Not great to iterate two times.223
for source_file in file_tracability_index.indexed_source_files():
224
# Is file referenced by forward links?225
is_source_file_referenced = (
226
traceability_index.has_source_file_reqs(
227
source_file.in_doctree_source_file_rel_path_posix
228
)229
)230
if is_source_file_referenced:
231
source_file.is_referenced = True
232
233
source_file_reqs: Optional[List[SDocNode]] = (
234
traceability_index.get_source_file_reqs(
235
source_file.in_doctree_source_file_rel_path_posix
236
)237
)238
if source_file_reqs is None:
239
continue240
241
for node_ in source_file_reqs:
242
node_document = assert_cast(
243
node_.get_document(), SDocDocument
244
)245
assert node_document.meta is not None
246
247
traceability_index.file_dependency_manager.add_dependency(
248
source_file.full_path,
249
source_file.output_file_full_path,
250
)251
traceability_index.file_dependency_manager.add_dependency(
252
source_file.full_path,
253
node_document.meta.output_document_full_path,
254
)255
traceability_index.file_dependency_manager.add_dependency(
256
node_document.meta.input_doc_full_path,
257
source_file.output_file_full_path,
258
)259
260
traceability_index.document_tree.attach_source_tree(source_tree)
261
262
#263
# Resolve pending InlineLinks. This depends on UIDs and anchors from264
# static documents, generated documents and source nodes.265
#266
for inline_link in traceability_index.pending_inline_links:
267
if not traceability_index.graph_database.has_any_link(
268
link_type=GraphLinkType.UID_TO_NODE,
269
lhs_node=inline_link.link,
270
):271
raise StrictDocException(
272
"DocumentIndex: "273
"the inline link references an object with an UID "274
"that does not exist: "275
f"{inline_link.link}."
276
)277
traceability_index.create_inline_link(inline_link)
278
traceability_index.pending_inline_links.clear()
279
280
#281
# Resolve all modification dates to support the incremental generation of282
# all artifacts.283
#284
285
file_dependency_manager = traceability_index.file_dependency_manager
286
287
file_dependency_manager.resolve_modification_dates(
288
traceability_index.strictdoc_last_update
289
)290
291
if project_config.user_plugin is not None:
292
project_config.user_plugin.traceability_index_build_finished(
293
traceability_index294
)295
296
return traceability_index
297
298
@staticmethod299
@timing_decorator("Build traceability graph")
- "4.4. Link document nodes" (REQUIREMENT)
- "4.5. Automatic resolution of reverse relations" (REQUIREMENT)
300
def create_from_document_tree(
301
document_tree: DocumentTree,
302
project_config: ProjectConfig,
303
) -> TraceabilityIndex:
304
"""
305
@relation(SDOC-SRS-32, SDOC-SRS-102, scope=function)306
"""307
308
# FIXME: Too many things going on below. Would be great to simplify this309
# workflow.310
d_01_document_iterators: Dict[SDocDocument, SDocDocumentIterator] = {}
311
d_07_file_traceability_index = FileTraceabilityIndex()
312
313
graph_database = GraphDatabase(
314
[315
(316
GraphLinkType.MID_TO_NODE,
317
OneToOneDictionary(
318
MID,
319
(320
SDocNode,
321
SDocDocument,
322
InlineLink,
323
Anchor,
324
),325
),326
),327
(328
GraphLinkType.UID_TO_NODE,
329
OneToOneDictionary(str, (SDocDocument, SDocNode, Anchor)),
330
),331
(332
GraphLinkType.NODE_TO_PARENT_NODES,
333
ManyToManySet(SDocNode, SDocNode),
334
),335
(336
GraphLinkType.NODE_TO_CHILD_NODES,
337
ManyToManySet(SDocNode, SDocNode),
338
),339
(340
GraphLinkType.NODE_TO_INCOMING_LINKS,
341
ManyToManySet(MID, InlineLink),
342
),343
(344
GraphLinkType.DOCUMENT_TO_TAGS,
345
OneToOneDictionary(MID, dict),
346
),347
]348
)349
350
file_dependency_manager: FileDependencyManager = (
351
FileDependencyManager.create_from_cache(
352
project_config=project_config
353
)354
)355
356
traceability_index = TraceabilityIndex(
357
document_tree,
358
d_01_document_iterators,
359
file_traceability_index=d_07_file_traceability_index,
360
graph_database=graph_database,
361
file_dependency_manager=file_dependency_manager,
362
)363
364
# It seems to be impossible to accomplish everything in just one for365
# loop. One particular problem that requires two passes: it is not366
# possible to know after one iteration which of the requirements367
# parents do not exist for each given requirement.368
#369
# Step #1:370
# - Collect a dictionary of all requirements in the document tree:371
# {req_id: req} # noqa: ERA001372
# - Each requirement's 'parents_uids' is populated with the forward373
# declarations of its parents uids.374
# - A separate map is created: {req_id: [req_children]}375
# At this point some information is in place, but it was not known if376
# some UIDs could not be resolved which is the task of the second377
# step.378
#379
# Step #2:380
# - Check if each requirement's has valid parent relations.381
# - Resolve parent forward declarations382
# - Re-assign children declarations383
# - Detect cycles384
# - Calculate depth of both parent and child relations.385
for (
386
path_to_grammar_,
387
grammar_from_file_,
388
) in document_tree.map_grammars_by_filenames.items():
389
try:
390
SDocValidator.validate_grammar_from_file(
391
path_to_grammar_, grammar_from_file_
392
)393
except StrictDocSemanticError as exc:
394
print(exc.to_print_message()) # noqa: T201
395
sys.exit(1)
396
397
document: SDocDocument
398
for document in document_tree.document_list:
399
assert document.grammar is not None
400
assert document.meta is not None
401
402
traceability_index.file_dependency_manager.add_dependency(
403
document.meta.input_doc_full_path,
404
document.meta.output_document_full_path,
405
)406
407
if document.config.view_style_tag == "REQUIREMENT_STYLE":
408
DEPRECATION_ENGINE.add_message(
409
"DEPRECATED_REQUIREMENT_STYLE",
410
"WARNING: REQUIREMENT_STYLE is deprecated. Replace it to VIEW_STYLE.",
411
)412
if document.config.node_in_toc_tag == "REQUIREMENT_IN_TOC":
413
DEPRECATION_ENGINE.add_message(
414
"DEPRECATED_REQUIREMENT_IN_TOC",
415
"WARNING: REQUIREMENT_IN_TOC is deprecated. Replace it to NODE_IN_TOC.",
416
)417
418
#419
# First, resolve all grammars that are imported from grammar files.420
#421
if document.grammar.import_from_file is not None:
422
grammar_path = document.grammar.import_from_file
423
if grammar_path.startswith("@"):
424
grammar_path = project_config.grammars[grammar_path]
425
else:
426
grammar_path = posixpath.join(
427
document.meta.input_doc_dir_rel_path.relative_path_posix,
428
grammar_path,
429
)430
document_grammar: Optional[DocumentGrammar] = (
431
document_tree.get_grammar_by_filename(grammar_path)
432
)433
if document_grammar is None:
434
raise StrictDocException(
435
"TraceabilityIndex: "436
f'the document "{document.reserved_title}" '
437
"imports a grammar from a file that does not exist: "438
f'"{document.grammar.import_from_file}". One known '
439
f"source of this error is when only a single document "
440
f"file is provided as input to the export or server "
441
f"command, rather than the containing folder. To locate "
442
f"the grammar file, StrictDoc needs to be able to "
443
f"resolve it relative to the input path."
444
)445
446
document.grammar.update_with_elements(document_grammar.elements)
447
448
# This is for the backward compatibility with the existing users.449
# If the included project grammar has no TEXT element defined,450
# we add it here automatically.451
if not document.grammar.has_text_element():
452
document.grammar.add_element_first(
453
DocumentGrammar.create_default_text_element(
454
document.grammar,
455
enable_mid=document.config.enable_mid is True,
456
)457
)458
459
# This is important because due to the difference between the460
# normal grammar vs imported grammar, the parent may not be set at461
# this point.462
document.grammar.parent = document
463
464
if document.config.markup == SDocMarkup.MARKDOWN:
465
SDMarkdownReader.fixup_composite_nodes(document)
466
467
try:
468
SDocValidator.validate_document(document)
469
except StrictDocSemanticError as exc:
470
print(exc.to_print_message()) # noqa: T201
471
sys.exit(1)
472
473
if graph_database.has_any_link(
474
link_type=GraphLinkType.MID_TO_NODE,
475
lhs_node=document.reserved_mid,
476
):477
other_document: SDocDocument = graph_database.get_link_value(
478
link_type=GraphLinkType.MID_TO_NODE,
479
lhs_node=document.reserved_mid,
480
)481
raise StrictDocException(
482
"TraceabilityIndex: "483
"the document MID is not unique: "484
f"{document.reserved_mid}. "
485
"All machine identifiers (MID) must be unique values. "486
f"Affected documents:\n"
487
f"{other_document.get_debug_info()}\n"
488
f"and\n"
489
f"{document.get_debug_info()}."
490
)491
492
graph_database.create_link(
493
link_type=GraphLinkType.MID_TO_NODE,
494
lhs_node=document.reserved_mid,
495
rhs_node=document,
496
)497
if document.uid:
498
graph_database.create_link(
499
link_type=GraphLinkType.UID_TO_NODE,
500
lhs_node=document.uid,
501
rhs_node=document,
502
)503
504
document_tags: Dict[str, int] = {}
505
graph_database.create_link(
506
link_type=GraphLinkType.DOCUMENT_TO_TAGS,
507
lhs_node=document.reserved_mid,
508
rhs_node=document_tags,
509
)510
511
document_iterator = SDocDocumentIterator(document)
512
d_01_document_iterators[document] = document_iterator
513
514
for node, _ in document_iterator.all_content(
515
print_fragments=False,
516
):517
if isinstance(node, SDocNode):
518
try:
519
assert document.grammar is not None
520
SDocValidator.validate_node(
521
node,
522
document_grammar=document.grammar,
523
path_to_sdoc_file=document.meta.input_doc_full_path,
524
auto_uid_mode=project_config.auto_uid_mode,
525
)526
except StrictDocSemanticError as exc:
527
print(exc.to_print_message()) # noqa: T201
528
sys.exit(1)
529
530
if graph_database.has_any_link(
531
link_type=GraphLinkType.MID_TO_NODE,
532
lhs_node=node.reserved_mid,
533
):534
other_node: SDocDocument = graph_database.get_link_value(
535
link_type=GraphLinkType.MID_TO_NODE,
536
lhs_node=node.reserved_mid,
537
)538
raise StrictDocException(
539
"TraceabilityIndex: "540
"the node MID is not unique: "541
f"{node.reserved_mid}. "
542
"All machine identifiers (MID) must be unique values. "543
f"Affected nodes:\n"
544
f"{other_node.get_debug_info()}\n"
545
f"and\n"
546
f"{node.get_debug_info()}."
547
)548
graph_database.create_link(
549
link_type=GraphLinkType.MID_TO_NODE,
550
lhs_node=node.reserved_mid,
551
rhs_node=node,
552
)553
554
if node.reserved_uid is not None:
555
# @relation(SDOC-SRS-29, scope=range_start)556
if traceability_index.graph_database.has_any_link(
557
link_type=GraphLinkType.UID_TO_NODE,
558
lhs_node=node.reserved_uid,
559
):560
already_existing_node: SDocNode = (
561
traceability_index.graph_database.get_link_value(
562
link_type=GraphLinkType.UID_TO_NODE,
563
lhs_node=node.reserved_uid,
564
)565
)566
other_req_doc = assert_cast(
567
already_existing_node.get_document(), SDocDocument
568
)569
if other_req_doc == document:
570
print( # noqa: T201
571
"error: DocumentIndex: "572
"two nodes with the same UID "573
"exist in the same document: "574
f'{node.reserved_uid} in "{document.title}".'
575
)576
else:
577
print( # noqa: T201
578
"error: DocumentIndex: "579
"two nodes with the same UID "580
"exist in two different documents: "581
f'{node.reserved_uid} in "{other_req_doc.title}" '
582
f'and "{document.title}".'
583
)584
sys.exit(1)
585
# @relation(SDOC-SRS-29, scope=range_end)586
587
traceability_index.graph_database.create_link(
588
link_type=GraphLinkType.UID_TO_NODE,
589
lhs_node=node.reserved_uid,
590
rhs_node=node,
591
)592
593
if isinstance(node, SDocNode):
594
requirement_node: SDocNode = assert_cast(node, SDocNode)
595
if requirement_node.reserved_tags is not None:
596
for tag in requirement_node.reserved_tags:
597
document_tags.setdefault(tag, 0)
598
document_tags[tag] += 1
599
for node_field_ in node.enumerate_fields():
600
for part in node_field_.parts:
601
# The inline links are handled at the next big602
# For loop pass because the information about603
# all Nodes and Anchors have not been604
# collected yet at this point.605
# see create_inline_link below.606
if isinstance(part, Anchor):
607
graph_database.create_link(
608
link_type=GraphLinkType.MID_TO_NODE,
609
lhs_node=part.mid,
610
rhs_node=part,
611
)612
graph_database.create_link(
613
link_type=GraphLinkType.UID_TO_NODE,
614
lhs_node=part.value,
615
rhs_node=part,
616
)617
618
# Now iterate over the requirements again to build an in-depth map of619
# parents and children.620
requirement: SDocNode
621
622
for document in document_tree.document_list:
623
assert document.meta is not None
624
625
document_iterator = d_01_document_iterators[document]
626
627
if document.config.custom_metadata is not None:
628
for metadata_entry_ in document.config.custom_metadata.entries:
629
for part in metadata_entry_.parts:
630
if isinstance(part, InlineLink):
631
traceability_index.pending_inline_links.append(part)
632
633
for node, _ in document_iterator.all_content(
634
print_fragments=False,
635
):636
if not isinstance(node, SDocNode):
637
continue638
639
requirement = assert_cast(node, SDocNode)
640
for node_field_ in requirement.enumerate_fields():
641
for part in node_field_.parts:
642
if isinstance(part, InlineLink):
643
traceability_index.pending_inline_links.append(part)
644
if requirement.reserved_uid is None:
645
continue646
647
# Now it is possible to resolve parents first checking if they648
# indeed exist.649
for reference in requirement.relations:
650
if reference.ref_type == ReferenceType.FILE:
651
d_07_file_traceability_index.create_requirement_with_forward_source_links(
652
requirement653
)654
elif reference.ref_type == ReferenceType.PARENT:
655
parent_reference: ParentReqReference = assert_cast(
656
reference, ParentReqReference
657
)658
parent_requirement = traceability_index.graph_database.get_link_value_weak(
659
link_type=GraphLinkType.UID_TO_NODE,
660
lhs_node=parent_reference.ref_uid,
661
)662
if parent_requirement is None:
663
raise StrictDocException(
664
f"[DocumentIndex.create] "
665
f"Requirement {requirement.reserved_uid} "
666
f"references "
667
f"parent requirement which doesn't exist: "
668
f"{parent_reference.ref_uid}."
669
)670
traceability_index.graph_database.create_link(
671
link_type=GraphLinkType.NODE_TO_PARENT_NODES,
672
lhs_node=requirement,
673
rhs_node=parent_requirement,
674
edge=parent_reference.role,
675
)676
traceability_index.graph_database.create_link(
677
link_type=GraphLinkType.NODE_TO_CHILD_NODES,
678
lhs_node=parent_requirement,
679
rhs_node=requirement,
680
edge=parent_reference.role,
681
)682
683
# Set document dependencies.684
parent_document: SDocDocument = assert_cast(
685
parent_requirement.get_document(), SDocDocument
686
)687
if document != parent_document:
688
assert parent_document.meta is not None
689
690
# This is where we help the incremental generation to691
# understand that the related documents must be692
# re-generated together.693
file_dependency_manager.add_dependency(
694
document.meta.input_doc_full_path,
695
parent_document.meta.output_document_full_path,
696
)697
file_dependency_manager.add_dependency(
698
parent_document.meta.input_doc_full_path,
699
document.meta.output_document_full_path,
700
)701
elif reference.ref_type == ReferenceType.CHILD:
702
child_reference: ChildReqReference = assert_cast(
703
reference, ChildReqReference
704
)705
child_requirement = traceability_index.graph_database.get_link_value_weak(
706
link_type=GraphLinkType.UID_TO_NODE,
707
lhs_node=child_reference.ref_uid,
708
)709
if child_requirement is None:
710
raise StrictDocException(
711
f"[DocumentIndex.create] "
712
f"Requirement {requirement.reserved_uid} "
713
f"references a "
714
f"child requirement that doesn't exist: "
715
f"{child_reference.ref_uid}."
716
)717
traceability_index.graph_database.create_link(
718
link_type=GraphLinkType.NODE_TO_PARENT_NODES,
719
lhs_node=child_requirement,
720
rhs_node=requirement,
721
edge=child_reference.role,
722
)723
traceability_index.graph_database.create_link(
724
link_type=GraphLinkType.NODE_TO_CHILD_NODES,
725
lhs_node=requirement,
726
rhs_node=child_requirement,
727
edge=child_reference.role,
728
)729
# Set document dependencies.730
child_requirement_document = assert_cast(
731
child_requirement.get_document(), SDocDocument
732
)733
if document != child_requirement_document:
734
assert child_requirement_document.meta is not None
735
736
# This is where we help the incremental generation to737
# understand that the related documents must be738
# re-generated together.739
file_dependency_manager.add_dependency(
740
document.meta.input_doc_full_path,
741
child_requirement_document.meta.output_document_full_path,
742
)743
file_dependency_manager.add_dependency(
744
child_requirement_document.meta.input_doc_full_path,
745
document.meta.output_document_full_path,
746
)747
else:
748
raise AssertionError(reference.ref_type)
749
750
# Iterate for the third time to validate the graph against751
# requirement cycles.752
parents_cycle_detector = TreeCycleDetector()
753
children_cycle_detector = TreeCycleDetector()
754
for document in document_tree.document_list:
755
document_iterator = d_01_document_iterators[document]
756
757
for node, _ in document_iterator.all_content(
758
print_fragments=False,
759
):760
if not isinstance(node, SDocNode):
761
continue762
763
requirement = assert_cast(node, SDocNode)
764
765
if requirement.reserved_uid is None:
766
continue767
768
# @relation(SDOC-SRS-30, scope=range_start)769
# Detect cycles770
def parent_cycle_traverse_(node_id: str) -> Any:
771
current_node = (
772
traceability_index.graph_database.get_link_value(
773
link_type=GraphLinkType.UID_TO_NODE,
774
lhs_node=node_id,
775
)776
)777
return list(
778
map(
779
lambda node_: node_.reserved_uid,
780
traceability_index.graph_database.get_link_values(
781
link_type=GraphLinkType.NODE_TO_PARENT_NODES,
782
lhs_node=current_node,
783
),784
)785
)786
787
parents_cycle_detector.check_node(
788
requirement.reserved_uid,
789
parent_cycle_traverse_,
790
)791
792
def child_cycle_traverse_(node_id: str) -> Any:
793
current_node = (
794
traceability_index.graph_database.get_link_value(
795
link_type=GraphLinkType.UID_TO_NODE,
796
lhs_node=node_id,
797
)798
)799
return list(
800
map(
801
lambda node_: node_.reserved_uid,
802
traceability_index.graph_database.get_link_values(
803
link_type=GraphLinkType.NODE_TO_CHILD_NODES,
804
lhs_node=current_node,
805
),806
)807
)808
809
children_cycle_detector.check_node(
810
requirement.reserved_uid,
811
child_cycle_traverse_,
812
)813
# @relation(SDOC-SRS-30, scope=range_end)814
815
map_documents_by_input_rel_path: Dict[str, SDocDocument] = {}
816
for document_ in document_tree.document_list:
817
assert document_.meta is not None
818
819
map_documents_by_input_rel_path[
820
document_.meta.input_doc_full_path
821
] = document_
822
823
# @relation(SDOC-SRS-109, scope=range_start)824
unique_document_from_file_occurences: Set[str] = set()
825
for document_ in document_tree.document_list:
826
document_from_file_: SDocDocumentFromFileIF
827
for document_from_file_ in document_.fragments_from_files:
828
traceability_index.contains_included_documents = True
829
830
assert isinstance(document_from_file_, DocumentFromFile), (
831
document_from_file_832
)833
834
assert (
835
document_from_file_.resolved_full_path_to_document_file
836
is not None
837
)838
839
if (
840
document_from_file_.resolved_full_path_to_document_file
841
not in map_documents_by_input_rel_path
842
):843
raise StrictDocException(
844
"A document includes contains a link to another document "845
"which is not resolved in the current documentation tree: "846
f"'{document_from_file_.file}'. This can happen if a single "
847
f"document path is provided as input to a StrictDoc command. "
848
f"Try providing a path to a folder where all documents "
849
f"are stored."
850
)851
resolved_document: SDocDocument = (
852
map_documents_by_input_rel_path[
853
document_from_file_.resolved_full_path_to_document_file
854
]855
)856
857
if (
858
document_from_file_.resolved_full_path_to_document_file
859
in unique_document_from_file_occurences
860
) and resolved_document.has_any_requirements():
861
raise StrictDocException(
862
"[DOCUMENT_FROM_FILE]: "863
"A multiple inclusion of a document is detected. "864
"A document that contains requirements or other nodes "865
"can be only included once: "866
f"{document_from_file_.file}."
867
)868
unique_document_from_file_occurences.add(
869
document_from_file_.resolved_full_path_to_document_file
870
)871
872
document_from_file_.configure_with_resolved_document(
873
resolved_document874
)875
876
# @relation(SDOC-SRS-109, scope=range_end)877
878
return traceability_index
879
880
@classmethod881
def _create_filter(
882
cls, traceability_index: Any, filter_query: str
883
) -> "NodeFilter":
884
query_reader = QueryReader()
885
requirements_query_object: Union[QueryObject, QueryNullObject]
886
try:
887
requirements_query = query_reader.read(filter_query)
888
requirements_query_object = QueryObject(
889
requirements_query, traceability_index
890
)891
except TextXSyntaxError as textx_syntax_error_:
892
raise StrictDocException(
893
"Cannot parse filter query."894
) from textx_syntax_error_
895
896
blacklisted_nodes: set[SDocElementIF] = set()
897
898
try:
899
for document in traceability_index.document_tree.document_list:
900
document_iterator = traceability_index.get_document_iterator(
901
document902
)903
for node, _ in document_iterator.all_content():
904
if (
905
isinstance(node, SDocNode)
906
and node.node_type == "SECTION"
907
and not requirements_query_object.evaluate(node)
908
):909
blacklisted_nodes.add(node)
910
911
# If the node is the last one, we check if all other912
# nodes are filtered out and if so, mark the parent913
# section node as not whitelisted as well.914
if (
915
node.parent.section_contents[
916
len(node.parent.section_contents) - 1
917
]918
== node
919
):920
if (
921
isinstance(node.parent, SDocNode)
922
and node.parent.node_type == "SECTION"
923
):924
cls._blacklist_if_needed(
925
blacklisted_nodes, node.parent
926
)927
928
elif isinstance(
929
node, SDocNode
930
) and not requirements_query_object.evaluate(node):
931
blacklisted_nodes.add(node)
932
# If the node is the last one, we check if all other933
# nodes are filtered out and if so, mark the parent934
# section node as not whitelisted as well.935
if (
936
node.parent.section_contents[
937
len(node.parent.section_contents) - 1
938
]939
== node
940
):941
cls._blacklist_if_needed(
942
blacklisted_nodes, node.parent
943
)944
945
except (AttributeError, NameError, TypeError) as attribute_error_:
946
raise StrictDocException(
947
f"Cannot apply a filter query to a node: {attribute_error_}"
948
) from attribute_error_
949
950
return NodeFilter(blacklisted_nodes)
951
952
@classmethod953
def _blacklist_if_needed(
954
cls,
955
blacklisted_nodes: set[SDocElementIF],
956
node: SDocElementIF,
957
) -> None:
958
if isinstance(node, SDocDocumentFromFileIF):
959
return960
961
if node.section_contents is not None:
962
for node_ in node.section_contents:
963
if node_ not in blacklisted_nodes:
964
return965
966
blacklisted_nodes.add(node)
967
968
# If it turns out that all child nodes are blacklisted,969
# go up and blacklist the parent node if needed.970
if (
971
isinstance(node, SDocNodeIF)
972
and node.parent not in blacklisted_nodes
973
):974
cls._blacklist_if_needed(blacklisted_nodes, node.parent)
975
976
@staticmethod977
def source_node_parser_tags(
978
cfg_entry: SourceNodesEntry, grammar_element: GrammarElement
979
) -> set[str]:
980
tags = set(grammar_element.get_field_titles())
981
# For remapped fields, don't parse the names from grammar but those from the mapping.982
for (
983
sdoc_field_name,
984
source_field_name,
985
) in cfg_entry.sdoc_to_source_map.items():
986
tags.remove(sdoc_field_name)
987
tags.add(source_field_name)
988
return tags