StrictDoc Documentation
strictdoc/core/traceability_index_builder.py
Source file coverage
Path:
strictdoc/core/traceability_index_builder.py
Lines:
967
Non-empty lines:
879
Non-empty lines covered with requirements:
879 / 879 (100.0%)
Functions:
8
Functions covered by requirements:
8 / 8 (100.0%)
1
"""
2
@relation(SDOC-SRS-28, SDOC-SRS-2, scope=file)
3
"""
4
 
5
import datetime
6
import glob
7
import os
8
import posixpath
9
import sys
10
from typing import Any, Dict, Iterator, List, Optional, Set, Union
11
 
12
from textx import TextXSyntaxError
13
 
14
from strictdoc.backend.sdoc.error_handling import StrictDocSemanticError
15
from strictdoc.backend.sdoc.models.anchor import Anchor
16
from strictdoc.backend.sdoc.models.document import SDocDocument
17
from strictdoc.backend.sdoc.models.document_from_file import DocumentFromFile
18
from strictdoc.backend.sdoc.models.document_grammar import DocumentGrammar
19
from strictdoc.backend.sdoc.models.grammar_element import (
20
    GrammarElement,
21
    ReferenceType,
22
)
23
from strictdoc.backend.sdoc.models.inline_link import InlineLink
24
from strictdoc.backend.sdoc.models.model import (
25
    SDocDocumentFromFileIF,
26
    SDocElementIF,
27
    SDocNodeIF,
28
)
29
from strictdoc.backend.sdoc.models.node import SDocNode
30
from strictdoc.backend.sdoc.models.reference import (
31
    ChildReqReference,
32
    ParentReqReference,
33
)
34
from strictdoc.backend.sdoc.node_filter import NodeFilter
35
from strictdoc.backend.sdoc.validations.sdoc_validator import SDocValidator
36
from strictdoc.backend.sdoc_source_code.caching_reader import (
37
    SourceFileTraceabilityCachingReader,
38
)
39
from strictdoc.core.constants import GraphLinkType
40
from strictdoc.core.document_iterator import SDocDocumentIterator
41
from strictdoc.core.document_tree import DocumentTree
42
from strictdoc.core.file_dependency_manager import FileDependencyManager
43
from strictdoc.core.file_system.document_finder import DocumentFinder
44
from strictdoc.core.file_system.source_files_finder import (
45
    SourceFilesFinder,
46
)
47
from strictdoc.core.file_system.source_tree import SourceFile, SourceTree
48
from strictdoc.core.file_traceability_index import FileTraceabilityIndex
49
from strictdoc.core.graph.many_to_many_set import ManyToManySet
50
from strictdoc.core.graph.one_to_one_dictionary import OneToOneDictionary
51
from strictdoc.core.graph_database import GraphDatabase
52
from strictdoc.core.project_config import (
53
    ProjectConfig,
54
    ProjectFeature,
55
    SourceNodesEntry,
56
)
57
from strictdoc.core.query_engine.query_object import (
58
    QueryNullObject,
59
    QueryObject,
60
)
61
from strictdoc.core.query_engine.query_reader import QueryReader
62
from strictdoc.core.traceability_index import (
63
    TraceabilityIndex,
64
)
65
from strictdoc.core.tree_cycle_detector import TreeCycleDetector
66
from strictdoc.helpers.cast import assert_cast
67
from strictdoc.helpers.deprecation_engine import DEPRECATION_ENGINE
68
from strictdoc.helpers.exception import StrictDocException
69
from strictdoc.helpers.file_modification_time import (
70
    get_file_modification_time,
71
)
72
from strictdoc.helpers.mid import MID
73
from strictdoc.helpers.parallelizer import Parallelizer
74
from strictdoc.helpers.timing import measure_performance, timing_decorator
75
 
76
 
77
class TraceabilityIndexBuilder:
78
    @classmethod
79
    def create(
80
        cls,
81
        *,
82
        project_config: ProjectConfig,
83
        parallelizer: Parallelizer,
84
        skip_source_files: bool = False,
85
    ) -> TraceabilityIndex:
86
        # TODO: It would be great to hide this code behind --development flag.
87
        # There is no need for this to be activated in the Pip-released builds.
88
        strict_own_files_unfiltered: Iterator[str] = glob.iglob(
89
            f"{project_config.get_strictdoc_root_path()}/strictdoc/**/*",
90
            recursive=True,
91
        )
92
        strict_own_files: List[str] = [
93
            f
94
            for f in strict_own_files_unfiltered
95
            if f.endswith(".html")
96
            or f.endswith(".py")
97
            or f.endswith(".jinja")
98
            or f.endswith(".svg")
99
        ]
100
        latest_strictdoc_own_file = (
101
            max(strict_own_files, key=os.path.getctime)
102
            if len(strict_own_files) > 0
103
            else None
104
        )
105
 
106
        strictdoc_last_update: datetime.datetime = (
107
            get_file_modification_time(latest_strictdoc_own_file)
108
            if (latest_strictdoc_own_file is not None)
109
            else datetime.datetime.fromtimestamp(0)
110
        )
111
        if (
112
            project_config.config_last_update is not None
113
            and project_config.config_last_update > strictdoc_last_update
114
        ):
115
            strictdoc_last_update = project_config.config_last_update
116
 
117
        document_tree, asset_manager = DocumentFinder.find_sdoc_content(
118
            project_config=project_config, parallelizer=parallelizer
119
        )
120
 
121
        # TODO: This is rather messy, but it is better than it used to be.
122
        # Currently, the traceability index holds everything that is later used
123
        # by HTML generators:
124
        # - traceability index itself
125
        # - document tree
126
        # - assets
127
        # - runtime configuration.
128
        traceability_index: TraceabilityIndex = (
129
            TraceabilityIndexBuilder.create_from_document_tree(
130
                document_tree,
131
                project_config,
132
            )
133
        )
134
        traceability_index.asset_manager = asset_manager
135
        traceability_index.strictdoc_last_update = strictdoc_last_update
136
 
137
        if node_filter_query := project_config.filter_nodes:
138
            traceability_index.node_filter = cls._create_filter(
139
                traceability_index=traceability_index,
140
                filter_query=node_filter_query,
141
            )
142
 
143
        #
144
        # File traceability-related calculations.
145
        #
146
        if not skip_source_files and project_config.is_feature_activated(
147
            ProjectFeature.REQUIREMENT_TO_SOURCE_TRACEABILITY
148
        ):
149
            file_tracability_index = (
150
                traceability_index.get_file_traceability_index()
151
            )
152
 
153
            with measure_performance("Find source files"):
154
                source_tree: SourceTree = SourceFilesFinder.find_source_files(
155
                    project_config=project_config
156
                )
157
 
158
            source_files = source_tree.source_files
159
            source_file: SourceFile
160
            for source_file in source_files:
161
                with measure_performance(
162
                    f"Reading source: {source_file.in_doctree_source_file_rel_path}"
163
                ):
164
                    source_nodes_cfg_entry = (
165
                        project_config.get_relevant_source_nodes_entry(
166
                            source_file.full_path
167
                        )
168
                    )
169
                    if source_nodes_cfg_entry is not None:
170
                        source_node_grammar_element = (
171
                            traceability_index.get_grammar_element(
172
                                source_nodes_cfg_entry.uid,
173
                                source_nodes_cfg_entry.node_type,
174
                            )
175
                        )
176
                        assert source_node_grammar_element is not None, (
177
                            "Missing grammar element for node: "
178
                            f"{source_nodes_cfg_entry.uid} {source_nodes_cfg_entry.node_type}"
179
                        )
180
                        source_node_tags = (
181
                            TraceabilityIndexBuilder.source_node_parser_tags(
182
                                source_nodes_cfg_entry,
183
                                source_node_grammar_element,
184
                            )
185
                        )
186
                    else:
187
                        source_node_tags = None
188
 
189
                    traceability_info = (
190
                        SourceFileTraceabilityCachingReader.read_from_file(
191
                            source_file.full_path,
192
                            project_config,
193
                            source_node_tags,
194
                        )
195
                    )
196
 
197
                if traceability_info:
198
                    traceability_index.create_traceability_info(
199
                        source_file,
200
                        traceability_info,
201
                    )
202
                    # Is file referenced by backwards links?
203
                    if len(traceability_info.markers) > 0:
204
                        source_file.is_referenced = True
205
 
206
            file_tracability_index.validate_and_resolve(
207
                traceability_index, project_config
208
            )
209
 
210
            # Iterate again to resolve if the file is referenced.
211
            # FIXME: Not great to iterate two times.
212
            for source_file in file_tracability_index.indexed_source_files():
213
                # Is file referenced by forward links?
214
                is_source_file_referenced = (
215
                    traceability_index.has_source_file_reqs(
216
                        source_file.in_doctree_source_file_rel_path_posix
217
                    )
218
                )
219
                if is_source_file_referenced:
220
                    source_file.is_referenced = True
221
 
222
                    source_file_reqs: Optional[List[SDocNode]] = (
223
                        traceability_index.get_source_file_reqs(
224
                            source_file.in_doctree_source_file_rel_path_posix
225
                        )
226
                    )
227
                    if source_file_reqs is None:
228
                        continue
229
 
230
                    for node_ in source_file_reqs:
231
                        node_document = assert_cast(
232
                            node_.get_document(), SDocDocument
233
                        )
234
                        assert node_document.meta is not None
235
 
236
                        traceability_index.file_dependency_manager.add_dependency(
237
                            source_file.full_path,
238
                            source_file.output_file_full_path,
239
                        )
240
                        traceability_index.file_dependency_manager.add_dependency(
241
                            source_file.full_path,
242
                            node_document.meta.output_document_full_path,
243
                        )
244
                        traceability_index.file_dependency_manager.add_dependency(
245
                            node_document.meta.input_doc_full_path,
246
                            source_file.output_file_full_path,
247
                        )
248
 
249
            traceability_index.document_tree.attach_source_tree(source_tree)
250
 
251
        #
252
        # Resolve all modification dates to support the incremental generation of
253
        # all artifacts.
254
        #
255
 
256
        file_dependency_manager = traceability_index.file_dependency_manager
257
 
258
        file_dependency_manager.resolve_modification_dates(
259
            traceability_index.strictdoc_last_update
260
        )
261
 
262
        if project_config.user_plugin is not None:
263
            project_config.user_plugin.traceability_index_build_finished(
264
                traceability_index
265
            )
266
 
267
        return traceability_index
268
 
269
    @staticmethod
270
    @timing_decorator("Build traceability graph")
271
    def create_from_document_tree(
272
        document_tree: DocumentTree,
273
        project_config: ProjectConfig,
274
    ) -> TraceabilityIndex:
275
        """
276
        @relation(SDOC-SRS-32, SDOC-SRS-102, scope=function)
277
        """
278
 
279
        # FIXME: Too many things going on below. Would be great to simplify this
280
        # workflow.
281
        d_01_document_iterators: Dict[SDocDocument, SDocDocumentIterator] = {}
282
        d_07_file_traceability_index = FileTraceabilityIndex()
283
 
284
        graph_database = GraphDatabase(
285
            [
286
                (
287
                    GraphLinkType.MID_TO_NODE,
288
                    OneToOneDictionary(
289
                        MID,
290
                        (
291
                            SDocNode,
292
                            SDocDocument,
293
                            InlineLink,
294
                            Anchor,
295
                        ),
296
                    ),
297
                ),
298
                (
299
                    GraphLinkType.UID_TO_NODE,
300
                    OneToOneDictionary(str, (SDocDocument, SDocNode, Anchor)),
301
                ),
302
                (
303
                    GraphLinkType.NODE_TO_PARENT_NODES,
304
                    ManyToManySet(SDocNode, SDocNode),
305
                ),
306
                (
307
                    GraphLinkType.NODE_TO_CHILD_NODES,
308
                    ManyToManySet(SDocNode, SDocNode),
309
                ),
310
                (
311
                    GraphLinkType.NODE_TO_INCOMING_LINKS,
312
                    ManyToManySet(MID, InlineLink),
313
                ),
314
                (
315
                    GraphLinkType.DOCUMENT_TO_TAGS,
316
                    OneToOneDictionary(MID, dict),
317
                ),
318
            ]
319
        )
320
 
321
        file_dependency_manager: FileDependencyManager = (
322
            FileDependencyManager.create_from_cache(
323
                project_config=project_config
324
            )
325
        )
326
 
327
        traceability_index = TraceabilityIndex(
328
            document_tree,
329
            d_01_document_iterators,
330
            file_traceability_index=d_07_file_traceability_index,
331
            graph_database=graph_database,
332
            file_dependency_manager=file_dependency_manager,
333
        )
334
 
335
        # It seems to be impossible to accomplish everything in just one for
336
        # loop. One particular problem that requires two passes: it is not
337
        # possible to know after one iteration which of the requirements
338
        # parents do not exist for each given requirement.
339
        #
340
        # Step #1:
341
        # - Collect a dictionary of all requirements in the document tree:
342
        # {req_id: req}  # noqa: ERA001
343
        # - Each requirement's 'parents_uids' is populated with the forward
344
        # declarations of its parents uids.
345
        # - A separate map is created: {req_id: [req_children]}
346
        # At this point some information is in place, but it was not known if
347
        # some UIDs could not be resolved which is the task of the second
348
        # step.
349
        #
350
        # Step #2:
351
        # - Check if each requirement's has valid parent relations.
352
        # - Resolve parent forward declarations
353
        # - Re-assign children declarations
354
        # - Detect cycles
355
        # - Calculate depth of both parent and child relations.
356
        for (
357
            path_to_grammar_,
358
            grammar_from_file_,
359
        ) in document_tree.map_grammars_by_filenames.items():
360
            try:
361
                SDocValidator.validate_grammar_from_file(
362
                    path_to_grammar_, grammar_from_file_
363
                )
364
            except StrictDocSemanticError as exc:
365
                print(exc.to_print_message())  # noqa: T201
366
                sys.exit(1)
367
 
368
        document: SDocDocument
369
        for document in document_tree.document_list:
370
            assert document.grammar is not None
371
            assert document.meta is not None
372
 
373
            traceability_index.file_dependency_manager.add_dependency(
374
                document.meta.input_doc_full_path,
375
                document.meta.output_document_full_path,
376
            )
377
 
378
            if document.config.view_style_tag == "REQUIREMENT_STYLE":
379
                DEPRECATION_ENGINE.add_message(
380
                    "DEPRECATED_REQUIREMENT_STYLE",
381
                    "WARNING: REQUIREMENT_STYLE is deprecated. Replace it to VIEW_STYLE.",
382
                )
383
            if document.config.node_in_toc_tag == "REQUIREMENT_IN_TOC":
384
                DEPRECATION_ENGINE.add_message(
385
                    "DEPRECATED_REQUIREMENT_IN_TOC",
386
                    "WARNING: REQUIREMENT_IN_TOC is deprecated. Replace it to NODE_IN_TOC.",
387
                )
388
 
389
            #
390
            # First, resolve all grammars that are imported from grammar files.
391
            #
392
            if document.grammar.import_from_file is not None:
393
                grammar_path = document.grammar.import_from_file
394
                if grammar_path.startswith("@"):
395
                    grammar_path = project_config.grammars[grammar_path]
396
                else:
397
                    grammar_path = posixpath.join(
398
                        document.meta.input_doc_dir_rel_path.relative_path_posix,
399
                        grammar_path,
400
                    )
401
                document_grammar: Optional[DocumentGrammar] = (
402
                    document_tree.get_grammar_by_filename(grammar_path)
403
                )
404
                if document_grammar is None:
405
                    raise StrictDocException(
406
                        "TraceabilityIndex: "
407
                        f'the document "{document.reserved_title}" '
408
                        "imports a grammar from a file that does not exist: "
409
                        f'"{document.grammar.import_from_file}". One known '
410
                        f"source of this error is when only a single document "
411
                        f"file is provided as input to the export or server "
412
                        f"command, rather than the containing folder. To locate "
413
                        f"the grammar file, StrictDoc needs to be able to "
414
                        f"resolve it relative to the input path."
415
                    )
416
 
417
                document.grammar.update_with_elements(document_grammar.elements)
418
 
419
                # This is for the backward compatibility with the existing users.
420
                # If the included project grammar has no TEXT element defined,
421
                # we add it here automatically.
422
                if not document.grammar.has_text_element():
423
                    document.grammar.add_element_first(
424
                        DocumentGrammar.create_default_text_element(
425
                            document.grammar,
426
                            enable_mid=document.config.enable_mid is True,
427
                        )
428
                    )
429
 
430
            # This is important because due to the difference between the
431
            # normal grammar vs imported grammar, the parent may not be set at
432
            # this point.
433
            document.grammar.parent = document
434
 
435
            try:
436
                SDocValidator.validate_document(document)
437
            except StrictDocSemanticError as exc:
438
                print(exc.to_print_message())  # noqa: T201
439
                sys.exit(1)
440
 
441
            if graph_database.has_any_link(
442
                link_type=GraphLinkType.MID_TO_NODE,
443
                lhs_node=document.reserved_mid,
444
            ):
445
                other_document: SDocDocument = graph_database.get_link_value(
446
                    link_type=GraphLinkType.MID_TO_NODE,
447
                    lhs_node=document.reserved_mid,
448
                )
449
                raise StrictDocException(
450
                    "TraceabilityIndex: "
451
                    "the document MID is not unique: "
452
                    f"{document.reserved_mid}. "
453
                    "All machine identifiers (MID) must be unique values. "
454
                    f"Affected documents:\n"
455
                    f"{other_document.get_debug_info()}\n"
456
                    f"and\n"
457
                    f"{document.get_debug_info()}."
458
                )
459
 
460
            graph_database.create_link(
461
                link_type=GraphLinkType.MID_TO_NODE,
462
                lhs_node=document.reserved_mid,
463
                rhs_node=document,
464
            )
465
            if document.uid:
466
                graph_database.create_link(
467
                    link_type=GraphLinkType.UID_TO_NODE,
468
                    lhs_node=document.uid,
469
                    rhs_node=document,
470
                )
471
 
472
            document_tags: Dict[str, int] = {}
473
            graph_database.create_link(
474
                link_type=GraphLinkType.DOCUMENT_TO_TAGS,
475
                lhs_node=document.reserved_mid,
476
                rhs_node=document_tags,
477
            )
478
 
479
            document_iterator = SDocDocumentIterator(document)
480
            d_01_document_iterators[document] = document_iterator
481
 
482
            for node, _ in document_iterator.all_content(
483
                print_fragments=False,
484
            ):
485
                if isinstance(node, SDocNode):
486
                    try:
487
                        assert document.grammar is not None
488
                        SDocValidator.validate_node(
489
                            node,
490
                            document_grammar=document.grammar,
491
                            path_to_sdoc_file=document.meta.input_doc_full_path,
492
                            auto_uid_mode=project_config.auto_uid_mode,
493
                        )
494
                    except StrictDocSemanticError as exc:
495
                        print(exc.to_print_message())  # noqa: T201
496
                        sys.exit(1)
497
 
498
                if graph_database.has_any_link(
499
                    link_type=GraphLinkType.MID_TO_NODE,
500
                    lhs_node=node.reserved_mid,
501
                ):
502
                    other_node: SDocDocument = graph_database.get_link_value(
503
                        link_type=GraphLinkType.MID_TO_NODE,
504
                        lhs_node=node.reserved_mid,
505
                    )
506
                    raise StrictDocException(
507
                        "TraceabilityIndex: "
508
                        "the node MID is not unique: "
509
                        f"{node.reserved_mid}. "
510
                        "All machine identifiers (MID) must be unique values. "
511
                        f"Affected nodes:\n"
512
                        f"{other_node.get_debug_info()}\n"
513
                        f"and\n"
514
                        f"{node.get_debug_info()}."
515
                    )
516
                graph_database.create_link(
517
                    link_type=GraphLinkType.MID_TO_NODE,
518
                    lhs_node=node.reserved_mid,
519
                    rhs_node=node,
520
                )
521
 
522
                if node.reserved_uid is not None:
523
                    # @relation(SDOC-SRS-29, scope=range_start)
524
                    if traceability_index.graph_database.has_any_link(
525
                        link_type=GraphLinkType.UID_TO_NODE,
526
                        lhs_node=node.reserved_uid,
527
                    ):
528
                        already_existing_node: SDocNode = (
529
                            traceability_index.graph_database.get_link_value(
530
                                link_type=GraphLinkType.UID_TO_NODE,
531
                                lhs_node=node.reserved_uid,
532
                            )
533
                        )
534
                        other_req_doc = assert_cast(
535
                            already_existing_node.get_document(), SDocDocument
536
                        )
537
                        if other_req_doc == document:
538
                            print(  # noqa: T201
539
                                "error: DocumentIndex: "
540
                                "two nodes with the same UID "
541
                                "exist in the same document: "
542
                                f'{node.reserved_uid} in "{document.title}".'
543
                            )
544
                        else:
545
                            print(  # noqa: T201
546
                                "error: DocumentIndex: "
547
                                "two nodes with the same UID "
548
                                "exist in two different documents: "
549
                                f'{node.reserved_uid} in "{other_req_doc.title}" '
550
                                f'and "{document.title}".'
551
                            )
552
                        sys.exit(1)
553
                    # @relation(SDOC-SRS-29, scope=range_end)
554
 
555
                    traceability_index.graph_database.create_link(
556
                        link_type=GraphLinkType.UID_TO_NODE,
557
                        lhs_node=node.reserved_uid,
558
                        rhs_node=node,
559
                    )
560
 
561
                if isinstance(node, SDocNode):
562
                    requirement_node: SDocNode = assert_cast(node, SDocNode)
563
                    if requirement_node.reserved_tags is not None:
564
                        for tag in requirement_node.reserved_tags:
565
                            document_tags.setdefault(tag, 0)
566
                            document_tags[tag] += 1
567
                    for node_field_ in node.enumerate_fields():
568
                        for part in node_field_.parts:
569
                            # The inline links are handled at the next big
570
                            # For loop pass because the information about
571
                            # all Nodes and Anchors have not been
572
                            # collected yet at this point.
573
                            # see create_inline_link below.
574
                            if isinstance(part, Anchor):
575
                                graph_database.create_link(
576
                                    link_type=GraphLinkType.MID_TO_NODE,
577
                                    lhs_node=part.mid,
578
                                    rhs_node=part,
579
                                )
580
                                graph_database.create_link(
581
                                    link_type=GraphLinkType.UID_TO_NODE,
582
                                    lhs_node=part.value,
583
                                    rhs_node=part,
584
                                )
585
 
586
        # Now iterate over the requirements again to build an in-depth map of
587
        # parents and children.
588
        requirement: SDocNode
589
 
590
        for document in document_tree.document_list:
591
            assert document.meta is not None
592
 
593
            document_iterator = d_01_document_iterators[document]
594
 
595
            for node, _ in document_iterator.all_content(
596
                print_fragments=False,
597
            ):
598
                if not isinstance(node, SDocNode):
599
                    continue
600
 
601
                requirement = assert_cast(node, SDocNode)
602
 
603
                #
604
                # At this point, we resolve LINKs, and the expectation is that
605
                # all UIDs or ANCHORS (they also have UIDs) are registered at the
606
                # previous pass.
607
                #
608
                for node_field_ in requirement.enumerate_fields():
609
                    for part in node_field_.parts:
610
                        if isinstance(part, InlineLink):
611
                            if not graph_database.has_any_link(
612
                                link_type=GraphLinkType.UID_TO_NODE,
613
                                lhs_node=part.link,
614
                            ):
615
                                raise StrictDocException(
616
                                    "DocumentIndex: "
617
                                    "the inline link references an "
618
                                    "object with an UID "
619
                                    "that does not exist: "
620
                                    f"{part.link}."
621
                                )
622
                            traceability_index.create_inline_link(part)
623
                if requirement.reserved_uid is None:
624
                    continue
625
 
626
                # Now it is possible to resolve parents first checking if they
627
                # indeed exist.
628
                for reference in requirement.relations:
629
                    if reference.ref_type == ReferenceType.FILE:
630
                        d_07_file_traceability_index.create_requirement_with_forward_source_links(
631
                            requirement
632
                        )
633
                    elif reference.ref_type == ReferenceType.PARENT:
634
                        parent_reference: ParentReqReference = assert_cast(
635
                            reference, ParentReqReference
636
                        )
637
                        parent_requirement = traceability_index.graph_database.get_link_value_weak(
638
                            link_type=GraphLinkType.UID_TO_NODE,
639
                            lhs_node=parent_reference.ref_uid,
640
                        )
641
                        if parent_requirement is None:
642
                            raise StrictDocException(
643
                                f"[DocumentIndex.create] "
644
                                f"Requirement {requirement.reserved_uid} "
645
                                f"references "
646
                                f"parent requirement which doesn't exist: "
647
                                f"{parent_reference.ref_uid}."
648
                            )
649
                        traceability_index.graph_database.create_link(
650
                            link_type=GraphLinkType.NODE_TO_PARENT_NODES,
651
                            lhs_node=requirement,
652
                            rhs_node=parent_requirement,
653
                            edge=parent_reference.role,
654
                        )
655
                        traceability_index.graph_database.create_link(
656
                            link_type=GraphLinkType.NODE_TO_CHILD_NODES,
657
                            lhs_node=parent_requirement,
658
                            rhs_node=requirement,
659
                            edge=parent_reference.role,
660
                        )
661
 
662
                        # Set document dependencies.
663
                        parent_document: SDocDocument = assert_cast(
664
                            parent_requirement.get_document(), SDocDocument
665
                        )
666
                        if document != parent_document:
667
                            assert parent_document.meta is not None
668
 
669
                            # This is where we help the incremental generation to
670
                            # understand that the related documents must be
671
                            # re-generated together.
672
                            file_dependency_manager.add_dependency(
673
                                document.meta.input_doc_full_path,
674
                                parent_document.meta.output_document_full_path,
675
                            )
676
                            file_dependency_manager.add_dependency(
677
                                parent_document.meta.input_doc_full_path,
678
                                document.meta.output_document_full_path,
679
                            )
680
                    elif reference.ref_type == ReferenceType.CHILD:
681
                        child_reference: ChildReqReference = assert_cast(
682
                            reference, ChildReqReference
683
                        )
684
                        child_requirement = traceability_index.graph_database.get_link_value_weak(
685
                            link_type=GraphLinkType.UID_TO_NODE,
686
                            lhs_node=child_reference.ref_uid,
687
                        )
688
                        if child_requirement is None:
689
                            raise StrictDocException(
690
                                f"[DocumentIndex.create] "
691
                                f"Requirement {requirement.reserved_uid} "
692
                                f"references a "
693
                                f"child requirement that doesn't exist: "
694
                                f"{child_reference.ref_uid}."
695
                            )
696
                        traceability_index.graph_database.create_link(
697
                            link_type=GraphLinkType.NODE_TO_PARENT_NODES,
698
                            lhs_node=child_requirement,
699
                            rhs_node=requirement,
700
                            edge=child_reference.role,
701
                        )
702
                        traceability_index.graph_database.create_link(
703
                            link_type=GraphLinkType.NODE_TO_CHILD_NODES,
704
                            lhs_node=requirement,
705
                            rhs_node=child_requirement,
706
                            edge=child_reference.role,
707
                        )
708
                        # Set document dependencies.
709
                        child_requirement_document = assert_cast(
710
                            child_requirement.get_document(), SDocDocument
711
                        )
712
                        if document != child_requirement_document:
713
                            assert child_requirement_document.meta is not None
714
 
715
                            # This is where we help the incremental generation to
716
                            # understand that the related documents must be
717
                            # re-generated together.
718
                            file_dependency_manager.add_dependency(
719
                                document.meta.input_doc_full_path,
720
                                child_requirement_document.meta.output_document_full_path,
721
                            )
722
                            file_dependency_manager.add_dependency(
723
                                child_requirement_document.meta.input_doc_full_path,
724
                                document.meta.output_document_full_path,
725
                            )
726
                    else:
727
                        raise AssertionError(reference.ref_type)
728
 
729
        # Iterate for the third time to validate the graph against
730
        # requirement cycles.
731
        parents_cycle_detector = TreeCycleDetector()
732
        children_cycle_detector = TreeCycleDetector()
733
        for document in document_tree.document_list:
734
            document_iterator = d_01_document_iterators[document]
735
 
736
            for node, _ in document_iterator.all_content(
737
                print_fragments=False,
738
            ):
739
                if not isinstance(node, SDocNode):
740
                    continue
741
 
742
                requirement = assert_cast(node, SDocNode)
743
 
744
                if requirement.reserved_uid is None:
745
                    continue
746
 
747
                # @relation(SDOC-SRS-30, scope=range_start)
748
                # Detect cycles
749
                def parent_cycle_traverse_(node_id: str) -> Any:
750
                    current_node = (
751
                        traceability_index.graph_database.get_link_value(
752
                            link_type=GraphLinkType.UID_TO_NODE,
753
                            lhs_node=node_id,
754
                        )
755
                    )
756
                    return list(
757
                        map(
758
                            lambda node_: node_.reserved_uid,
759
                            traceability_index.graph_database.get_link_values(
760
                                link_type=GraphLinkType.NODE_TO_PARENT_NODES,
761
                                lhs_node=current_node,
762
                            ),
763
                        )
764
                    )
765
 
766
                parents_cycle_detector.check_node(
767
                    requirement.reserved_uid,
768
                    parent_cycle_traverse_,
769
                )
770
 
771
                def child_cycle_traverse_(node_id: str) -> Any:
772
                    current_node = (
773
                        traceability_index.graph_database.get_link_value(
774
                            link_type=GraphLinkType.UID_TO_NODE,
775
                            lhs_node=node_id,
776
                        )
777
                    )
778
                    return list(
779
                        map(
780
                            lambda node_: node_.reserved_uid,
781
                            traceability_index.graph_database.get_link_values(
782
                                link_type=GraphLinkType.NODE_TO_CHILD_NODES,
783
                                lhs_node=current_node,
784
                            ),
785
                        )
786
                    )
787
 
788
                children_cycle_detector.check_node(
789
                    requirement.reserved_uid,
790
                    child_cycle_traverse_,
791
                )
792
                # @relation(SDOC-SRS-30, scope=range_end)
793
 
794
        map_documents_by_input_rel_path: Dict[str, SDocDocument] = {}
795
        for document_ in document_tree.document_list:
796
            assert document_.meta is not None
797
 
798
            map_documents_by_input_rel_path[
799
                document_.meta.input_doc_full_path
800
            ] = document_
801
 
802
        # @relation(SDOC-SRS-109, scope=range_start)
803
        unique_document_from_file_occurences: Set[str] = set()
804
        for document_ in document_tree.document_list:
805
            document_from_file_: SDocDocumentFromFileIF
806
            for document_from_file_ in document_.fragments_from_files:
807
                traceability_index.contains_included_documents = True
808
 
809
                assert isinstance(document_from_file_, DocumentFromFile), (
810
                    document_from_file_
811
                )
812
 
813
                assert (
814
                    document_from_file_.resolved_full_path_to_document_file
815
                    is not None
816
                )
817
 
818
                if (
819
                    document_from_file_.resolved_full_path_to_document_file
820
                    not in map_documents_by_input_rel_path
821
                ):
822
                    raise StrictDocException(
823
                        "A document includes contains a link to another document "
824
                        "which is not resolved in the current documentation tree: "
825
                        f"'{document_from_file_.file}'. This can happen if a single "
826
                        f"document path is provided as input to a StrictDoc command. "
827
                        f"Try providing a path to a folder where all documents "
828
                        f"are stored."
829
                    )
830
                resolved_document: SDocDocument = (
831
                    map_documents_by_input_rel_path[
832
                        document_from_file_.resolved_full_path_to_document_file
833
                    ]
834
                )
835
 
836
                if (
837
                    document_from_file_.resolved_full_path_to_document_file
838
                    in unique_document_from_file_occurences
839
                ) and resolved_document.has_any_requirements():
840
                    raise StrictDocException(
841
                        "[DOCUMENT_FROM_FILE]: "
842
                        "A multiple inclusion of a document is detected. "
843
                        "A document that contains requirements or other nodes "
844
                        "can be only included once: "
845
                        f"{document_from_file_.file}."
846
                    )
847
                unique_document_from_file_occurences.add(
848
                    document_from_file_.resolved_full_path_to_document_file
849
                )
850
 
851
                document_from_file_.configure_with_resolved_document(
852
                    resolved_document
853
                )
854
 
855
        # @relation(SDOC-SRS-109, scope=range_end)
856
 
857
        return traceability_index
858
 
859
    @classmethod
860
    def _create_filter(
861
        cls, traceability_index: Any, filter_query: str
862
    ) -> "NodeFilter":
863
        query_reader = QueryReader()
864
        requirements_query_object: Union[QueryObject, QueryNullObject]
865
        try:
866
            requirements_query = query_reader.read(filter_query)
867
            requirements_query_object = QueryObject(
868
                requirements_query, traceability_index
869
            )
870
        except TextXSyntaxError as textx_syntax_error_:
871
            raise StrictDocException(
872
                "Cannot parse filter query."
873
            ) from textx_syntax_error_
874
 
875
        blacklisted_nodes: set[SDocElementIF] = set()
876
 
877
        try:
878
            for document in traceability_index.document_tree.document_list:
879
                document_iterator = traceability_index.get_document_iterator(
880
                    document
881
                )
882
                for node, _ in document_iterator.all_content():
883
                    if (
884
                        isinstance(node, SDocNode)
885
                        and node.node_type == "SECTION"
886
                        and not requirements_query_object.evaluate(node)
887
                    ):
888
                        blacklisted_nodes.add(node)
889
 
890
                        # If the node is the last one, we check if all other
891
                        # nodes are filtered out and if so, mark the parent
892
                        # section node as not whitelisted as well.
893
                        if (
894
                            node.parent.section_contents[
895
                                len(node.parent.section_contents) - 1
896
                            ]
897
                            == node
898
                        ):
899
                            if (
900
                                isinstance(node.parent, SDocNode)
901
                                and node.parent.node_type == "SECTION"
902
                            ):
903
                                cls._blacklist_if_needed(
904
                                    blacklisted_nodes, node.parent
905
                                )
906
 
907
                    elif isinstance(
908
                        node, SDocNode
909
                    ) and not requirements_query_object.evaluate(node):
910
                        blacklisted_nodes.add(node)
911
                        # If the node is the last one, we check if all other
912
                        # nodes are filtered out and if so, mark the parent
913
                        # section node as not whitelisted as well.
914
                        if (
915
                            node.parent.section_contents[
916
                                len(node.parent.section_contents) - 1
917
                            ]
918
                            == node
919
                        ):
920
                            cls._blacklist_if_needed(
921
                                blacklisted_nodes, node.parent
922
                            )
923
 
924
        except (AttributeError, NameError, TypeError) as attribute_error_:
925
            raise StrictDocException(
926
                f"Cannot apply a filter query to a node: {attribute_error_}"
927
            ) from attribute_error_
928
 
929
        return NodeFilter(blacklisted_nodes)
930
 
931
    @classmethod
932
    def _blacklist_if_needed(
933
        cls,
934
        blacklisted_nodes: set[SDocElementIF],
935
        node: SDocElementIF,
936
    ) -> None:
937
        if isinstance(node, SDocDocumentFromFileIF):
938
            return
939
 
940
        if node.section_contents is not None:
941
            for node_ in node.section_contents:
942
                if node_ not in blacklisted_nodes:
943
                    return
944
 
945
        blacklisted_nodes.add(node)
946
 
947
        # If it turns out that all child nodes are blacklisted,
948
        # go up and blacklist the parent node if needed.
949
        if (
950
            isinstance(node, SDocNodeIF)
951
            and node.parent not in blacklisted_nodes
952
        ):
953
            cls._blacklist_if_needed(blacklisted_nodes, node.parent)
954
 
955
    @staticmethod
956
    def source_node_parser_tags(
957
        cfg_entry: SourceNodesEntry, grammar_element: GrammarElement
958
    ) -> set[str]:
959
        tags = set(grammar_element.get_field_titles())
960
        # For remapped fields, don't parse the names from grammar but those from the mapping.
961
        for (
962
            sdoc_field_name,
963
            source_field_name,
964
        ) in cfg_entry.sdoc_to_source_map.items():
965
            tags.remove(sdoc_field_name)
966
            tags.add(source_field_name)
967
        return tags