StrictDoc Documentation
strictdoc/core/traceability_index_builder.py
Source file coverage
Path:
strictdoc/core/traceability_index_builder.py
Lines:
988
Non-empty lines:
898
Non-empty lines covered with requirements:
898 / 898 (100.0%)
Functions:
8
Functions covered by requirements:
8 / 8 (100.0%)
1
"""
2
@relation(SDOC-SRS-28, SDOC-SRS-2, scope=file)
3
"""
4
 
5
import datetime
6
import glob
7
import os
8
import posixpath
9
import sys
10
from typing import Any, Dict, Iterator, List, Optional, Set, Union
11
 
12
from textx import TextXSyntaxError
13
 
14
from strictdoc.backend.markdown.reader import SDMarkdownReader
15
from strictdoc.backend.sdoc.constants import SDocMarkup
16
from strictdoc.backend.sdoc.error_handling import StrictDocSemanticError
17
from strictdoc.backend.sdoc.models.anchor import Anchor
18
from strictdoc.backend.sdoc.models.document import SDocDocument
19
from strictdoc.backend.sdoc.models.document_from_file import DocumentFromFile
20
from strictdoc.backend.sdoc.models.document_grammar import DocumentGrammar
21
from strictdoc.backend.sdoc.models.grammar_element import (
22
    GrammarElement,
23
    ReferenceType,
24
)
25
from strictdoc.backend.sdoc.models.inline_link import InlineLink
26
from strictdoc.backend.sdoc.models.model import (
27
    SDocDocumentFromFileIF,
28
    SDocElementIF,
29
    SDocNodeIF,
30
)
31
from strictdoc.backend.sdoc.models.node import SDocNode
32
from strictdoc.backend.sdoc.models.reference import (
33
    ChildReqReference,
34
    ParentReqReference,
35
)
36
from strictdoc.backend.sdoc.node_filter import NodeFilter
37
from strictdoc.backend.sdoc.validations.sdoc_validator import SDocValidator
38
from strictdoc.backend.sdoc_source_code.caching_reader import (
39
    SourceFileTraceabilityCachingReader,
40
)
41
from strictdoc.core.constants import GraphLinkType
42
from strictdoc.core.document_iterator import SDocDocumentIterator
43
from strictdoc.core.document_tree import DocumentTree
44
from strictdoc.core.file_dependency_manager import FileDependencyManager
45
from strictdoc.core.file_system.document_finder import DocumentFinder
46
from strictdoc.core.file_system.source_files_finder import (
47
    SourceFilesFinder,
48
)
49
from strictdoc.core.file_system.source_tree import SourceFile, SourceTree
50
from strictdoc.core.file_traceability_index import FileTraceabilityIndex
51
from strictdoc.core.graph.many_to_many_set import ManyToManySet
52
from strictdoc.core.graph.one_to_one_dictionary import OneToOneDictionary
53
from strictdoc.core.graph_database import GraphDatabase
54
from strictdoc.core.project_config import (
55
    ProjectConfig,
56
    ProjectFeature,
57
    SourceNodesEntry,
58
)
59
from strictdoc.core.query_engine.query_object import (
60
    QueryNullObject,
61
    QueryObject,
62
)
63
from strictdoc.core.query_engine.query_reader import QueryReader
64
from strictdoc.core.traceability_index import (
65
    TraceabilityIndex,
66
)
67
from strictdoc.core.tree_cycle_detector import TreeCycleDetector
68
from strictdoc.helpers.cast import assert_cast
69
from strictdoc.helpers.deprecation_engine import DEPRECATION_ENGINE
70
from strictdoc.helpers.exception import StrictDocException
71
from strictdoc.helpers.file_modification_time import (
72
    get_file_modification_time,
73
)
74
from strictdoc.helpers.mid import MID
75
from strictdoc.helpers.parallelizer import Parallelizer
76
from strictdoc.helpers.paths import shorten_path
77
from strictdoc.helpers.timing import (
78
    measure_performance,
79
    measure_performance_loop,
80
    timing_decorator,
81
)
82
 
83
 
84
class TraceabilityIndexBuilder:
85
    @classmethod
86
    def create(
87
        cls,
88
        *,
89
        project_config: ProjectConfig,
90
        parallelizer: Parallelizer,
91
        skip_source_files: bool = False,
92
    ) -> TraceabilityIndex:
93
        # TODO: It would be great to hide this code behind --development flag.
94
        # There is no need for this to be activated in the Pip-released builds.
95
        strict_own_files_unfiltered: Iterator[str] = glob.iglob(
96
            f"{project_config.get_strictdoc_root_path()}/strictdoc/**/*",
97
            recursive=True,
98
        )
99
        strict_own_files: List[str] = [
100
            f
101
            for f in strict_own_files_unfiltered
102
            if f.endswith(".html")
103
            or f.endswith(".py")
104
            or f.endswith(".jinja")
105
            or f.endswith(".svg")
106
        ]
107
        latest_strictdoc_own_file = (
108
            max(strict_own_files, key=os.path.getctime)
109
            if len(strict_own_files) > 0
110
            else None
111
        )
112
 
113
        strictdoc_last_update: datetime.datetime = (
114
            get_file_modification_time(latest_strictdoc_own_file)
115
            if (latest_strictdoc_own_file is not None)
116
            else datetime.datetime.fromtimestamp(0)
117
        )
118
        if (
119
            project_config.config_last_update is not None
120
            and project_config.config_last_update > strictdoc_last_update
121
        ):
122
            strictdoc_last_update = project_config.config_last_update
123
 
124
        document_tree, asset_manager = DocumentFinder.find_sdoc_content(
125
            project_config=project_config, parallelizer=parallelizer
126
        )
127
 
128
        # TODO: This is rather messy, but it is better than it used to be.
129
        # Currently, the traceability index holds everything that is later used
130
        # by HTML generators:
131
        # - traceability index itself
132
        # - document tree
133
        # - assets
134
        # - runtime configuration.
135
        traceability_index: TraceabilityIndex = (
136
            TraceabilityIndexBuilder.create_from_document_tree(
137
                document_tree,
138
                project_config,
139
            )
140
        )
141
        traceability_index.asset_manager = asset_manager
142
        traceability_index.strictdoc_last_update = strictdoc_last_update
143
 
144
        if node_filter_query := project_config.filter_nodes:
145
            traceability_index.node_filter = cls._create_filter(
146
                traceability_index=traceability_index,
147
                filter_query=node_filter_query,
148
            )
149
 
150
        #
151
        # File traceability-related calculations.
152
        #
153
        if not skip_source_files and project_config.is_feature_activated(
154
            ProjectFeature.REQUIREMENT_TO_SOURCE_TRACEABILITY
155
        ):
156
            file_tracability_index = (
157
                traceability_index.get_file_traceability_index()
158
            )
159
 
160
            with measure_performance("Find source files"):
161
                source_tree: SourceTree = SourceFilesFinder.find_source_files(
162
                    project_config=project_config
163
                )
164
 
165
            source_files = source_tree.source_files
166
            source_file: SourceFile
167
            with measure_performance_loop(
168
                "Reading source", len(source_files)
169
            ) as report_progress:
170
                for source_file in source_files:
171
                    with report_progress(
172
                        source_file.in_doctree_source_file_rel_path,
173
                        short_title=shorten_path(
174
                            source_file.in_doctree_source_file_rel_path
175
                        ),
176
                    ):
177
                        source_nodes_cfg_entry = (
178
                            project_config.get_relevant_source_nodes_entry(
179
                                source_file.full_path
180
                            )
181
                        )
182
                        if source_nodes_cfg_entry is not None:
183
                            source_node_grammar_element = (
184
                                traceability_index.get_grammar_element(
185
                                    source_nodes_cfg_entry.uid,
186
                                    source_nodes_cfg_entry.node_type,
187
                                )
188
                            )
189
                            assert source_node_grammar_element is not None, (
190
                                "Missing grammar element for node: "
191
                                f"{source_nodes_cfg_entry.uid} {source_nodes_cfg_entry.node_type}"
192
                            )
193
                            source_node_tags = TraceabilityIndexBuilder.source_node_parser_tags(
194
                                source_nodes_cfg_entry,
195
                                source_node_grammar_element,
196
                            )
197
                        else:
198
                            source_node_tags = None
199
 
200
                        traceability_info = (
201
                            SourceFileTraceabilityCachingReader.read_from_file(
202
                                source_file.full_path,
203
                                project_config,
204
                                source_node_tags,
205
                            )
206
                        )
207
 
208
                    if traceability_info:
209
                        traceability_index.create_traceability_info(
210
                            source_file,
211
                            traceability_info,
212
                        )
213
                        # Is file referenced by backwards links?
214
                        if len(traceability_info.markers) > 0:
215
                            source_file.is_referenced = True
216
 
217
            file_tracability_index.validate_and_resolve(
218
                traceability_index, project_config
219
            )
220
 
221
            # Iterate again to resolve if the file is referenced.
222
            # FIXME: Not great to iterate two times.
223
            for source_file in file_tracability_index.indexed_source_files():
224
                # Is file referenced by forward links?
225
                is_source_file_referenced = (
226
                    traceability_index.has_source_file_reqs(
227
                        source_file.in_doctree_source_file_rel_path_posix
228
                    )
229
                )
230
                if is_source_file_referenced:
231
                    source_file.is_referenced = True
232
 
233
                    source_file_reqs: Optional[List[SDocNode]] = (
234
                        traceability_index.get_source_file_reqs(
235
                            source_file.in_doctree_source_file_rel_path_posix
236
                        )
237
                    )
238
                    if source_file_reqs is None:
239
                        continue
240
 
241
                    for node_ in source_file_reqs:
242
                        node_document = assert_cast(
243
                            node_.get_document(), SDocDocument
244
                        )
245
                        assert node_document.meta is not None
246
 
247
                        traceability_index.file_dependency_manager.add_dependency(
248
                            source_file.full_path,
249
                            source_file.output_file_full_path,
250
                        )
251
                        traceability_index.file_dependency_manager.add_dependency(
252
                            source_file.full_path,
253
                            node_document.meta.output_document_full_path,
254
                        )
255
                        traceability_index.file_dependency_manager.add_dependency(
256
                            node_document.meta.input_doc_full_path,
257
                            source_file.output_file_full_path,
258
                        )
259
 
260
            traceability_index.document_tree.attach_source_tree(source_tree)
261
 
262
        #
263
        # Resolve pending InlineLinks. This depends on UIDs and anchors from
264
        # static documents, generated documents and source nodes.
265
        #
266
        for inline_link in traceability_index.pending_inline_links:
267
            if not traceability_index.graph_database.has_any_link(
268
                link_type=GraphLinkType.UID_TO_NODE,
269
                lhs_node=inline_link.link,
270
            ):
271
                raise StrictDocException(
272
                    "DocumentIndex: "
273
                    "the inline link references an object with an UID "
274
                    "that does not exist: "
275
                    f"{inline_link.link}."
276
                )
277
            traceability_index.create_inline_link(inline_link)
278
        traceability_index.pending_inline_links.clear()
279
 
280
        #
281
        # Resolve all modification dates to support the incremental generation of
282
        # all artifacts.
283
        #
284
 
285
        file_dependency_manager = traceability_index.file_dependency_manager
286
 
287
        file_dependency_manager.resolve_modification_dates(
288
            traceability_index.strictdoc_last_update
289
        )
290
 
291
        if project_config.user_plugin is not None:
292
            project_config.user_plugin.traceability_index_build_finished(
293
                traceability_index
294
            )
295
 
296
        return traceability_index
297
 
298
    @staticmethod
299
    @timing_decorator("Build traceability graph")
300
    def create_from_document_tree(
301
        document_tree: DocumentTree,
302
        project_config: ProjectConfig,
303
    ) -> TraceabilityIndex:
304
        """
305
        @relation(SDOC-SRS-32, SDOC-SRS-102, scope=function)
306
        """
307
 
308
        # FIXME: Too many things going on below. Would be great to simplify this
309
        # workflow.
310
        d_01_document_iterators: Dict[SDocDocument, SDocDocumentIterator] = {}
311
        d_07_file_traceability_index = FileTraceabilityIndex()
312
 
313
        graph_database = GraphDatabase(
314
            [
315
                (
316
                    GraphLinkType.MID_TO_NODE,
317
                    OneToOneDictionary(
318
                        MID,
319
                        (
320
                            SDocNode,
321
                            SDocDocument,
322
                            InlineLink,
323
                            Anchor,
324
                        ),
325
                    ),
326
                ),
327
                (
328
                    GraphLinkType.UID_TO_NODE,
329
                    OneToOneDictionary(str, (SDocDocument, SDocNode, Anchor)),
330
                ),
331
                (
332
                    GraphLinkType.NODE_TO_PARENT_NODES,
333
                    ManyToManySet(SDocNode, SDocNode),
334
                ),
335
                (
336
                    GraphLinkType.NODE_TO_CHILD_NODES,
337
                    ManyToManySet(SDocNode, SDocNode),
338
                ),
339
                (
340
                    GraphLinkType.NODE_TO_INCOMING_LINKS,
341
                    ManyToManySet(MID, InlineLink),
342
                ),
343
                (
344
                    GraphLinkType.DOCUMENT_TO_TAGS,
345
                    OneToOneDictionary(MID, dict),
346
                ),
347
            ]
348
        )
349
 
350
        file_dependency_manager: FileDependencyManager = (
351
            FileDependencyManager.create_from_cache(
352
                project_config=project_config
353
            )
354
        )
355
 
356
        traceability_index = TraceabilityIndex(
357
            document_tree,
358
            d_01_document_iterators,
359
            file_traceability_index=d_07_file_traceability_index,
360
            graph_database=graph_database,
361
            file_dependency_manager=file_dependency_manager,
362
        )
363
 
364
        # It seems to be impossible to accomplish everything in just one for
365
        # loop. One particular problem that requires two passes: it is not
366
        # possible to know after one iteration which of the requirements
367
        # parents do not exist for each given requirement.
368
        #
369
        # Step #1:
370
        # - Collect a dictionary of all requirements in the document tree:
371
        # {req_id: req}  # noqa: ERA001
372
        # - Each requirement's 'parents_uids' is populated with the forward
373
        # declarations of its parents uids.
374
        # - A separate map is created: {req_id: [req_children]}
375
        # At this point some information is in place, but it was not known if
376
        # some UIDs could not be resolved which is the task of the second
377
        # step.
378
        #
379
        # Step #2:
380
        # - Check if each requirement's has valid parent relations.
381
        # - Resolve parent forward declarations
382
        # - Re-assign children declarations
383
        # - Detect cycles
384
        # - Calculate depth of both parent and child relations.
385
        for (
386
            path_to_grammar_,
387
            grammar_from_file_,
388
        ) in document_tree.map_grammars_by_filenames.items():
389
            try:
390
                SDocValidator.validate_grammar_from_file(
391
                    path_to_grammar_, grammar_from_file_
392
                )
393
            except StrictDocSemanticError as exc:
394
                print(exc.to_print_message())  # noqa: T201
395
                sys.exit(1)
396
 
397
        document: SDocDocument
398
        for document in document_tree.document_list:
399
            assert document.grammar is not None
400
            assert document.meta is not None
401
 
402
            traceability_index.file_dependency_manager.add_dependency(
403
                document.meta.input_doc_full_path,
404
                document.meta.output_document_full_path,
405
            )
406
 
407
            if document.config.view_style_tag == "REQUIREMENT_STYLE":
408
                DEPRECATION_ENGINE.add_message(
409
                    "DEPRECATED_REQUIREMENT_STYLE",
410
                    "WARNING: REQUIREMENT_STYLE is deprecated. Replace it to VIEW_STYLE.",
411
                )
412
            if document.config.node_in_toc_tag == "REQUIREMENT_IN_TOC":
413
                DEPRECATION_ENGINE.add_message(
414
                    "DEPRECATED_REQUIREMENT_IN_TOC",
415
                    "WARNING: REQUIREMENT_IN_TOC is deprecated. Replace it to NODE_IN_TOC.",
416
                )
417
 
418
            #
419
            # First, resolve all grammars that are imported from grammar files.
420
            #
421
            if document.grammar.import_from_file is not None:
422
                grammar_path = document.grammar.import_from_file
423
                if grammar_path.startswith("@"):
424
                    grammar_path = project_config.grammars[grammar_path]
425
                else:
426
                    grammar_path = posixpath.join(
427
                        document.meta.input_doc_dir_rel_path.relative_path_posix,
428
                        grammar_path,
429
                    )
430
                document_grammar: Optional[DocumentGrammar] = (
431
                    document_tree.get_grammar_by_filename(grammar_path)
432
                )
433
                if document_grammar is None:
434
                    raise StrictDocException(
435
                        "TraceabilityIndex: "
436
                        f'the document "{document.reserved_title}" '
437
                        "imports a grammar from a file that does not exist: "
438
                        f'"{document.grammar.import_from_file}". One known '
439
                        f"source of this error is when only a single document "
440
                        f"file is provided as input to the export or server "
441
                        f"command, rather than the containing folder. To locate "
442
                        f"the grammar file, StrictDoc needs to be able to "
443
                        f"resolve it relative to the input path."
444
                    )
445
 
446
                document.grammar.update_with_elements(document_grammar.elements)
447
 
448
                # This is for the backward compatibility with the existing users.
449
                # If the included project grammar has no TEXT element defined,
450
                # we add it here automatically.
451
                if not document.grammar.has_text_element():
452
                    document.grammar.add_element_first(
453
                        DocumentGrammar.create_default_text_element(
454
                            document.grammar,
455
                            enable_mid=document.config.enable_mid is True,
456
                        )
457
                    )
458
 
459
            # This is important because due to the difference between the
460
            # normal grammar vs imported grammar, the parent may not be set at
461
            # this point.
462
            document.grammar.parent = document
463
 
464
            if document.config.markup == SDocMarkup.MARKDOWN:
465
                SDMarkdownReader.fixup_composite_nodes(document)
466
 
467
            try:
468
                SDocValidator.validate_document(document)
469
            except StrictDocSemanticError as exc:
470
                print(exc.to_print_message())  # noqa: T201
471
                sys.exit(1)
472
 
473
            if graph_database.has_any_link(
474
                link_type=GraphLinkType.MID_TO_NODE,
475
                lhs_node=document.reserved_mid,
476
            ):
477
                other_document: SDocDocument = graph_database.get_link_value(
478
                    link_type=GraphLinkType.MID_TO_NODE,
479
                    lhs_node=document.reserved_mid,
480
                )
481
                raise StrictDocException(
482
                    "TraceabilityIndex: "
483
                    "the document MID is not unique: "
484
                    f"{document.reserved_mid}. "
485
                    "All machine identifiers (MID) must be unique values. "
486
                    f"Affected documents:\n"
487
                    f"{other_document.get_debug_info()}\n"
488
                    f"and\n"
489
                    f"{document.get_debug_info()}."
490
                )
491
 
492
            graph_database.create_link(
493
                link_type=GraphLinkType.MID_TO_NODE,
494
                lhs_node=document.reserved_mid,
495
                rhs_node=document,
496
            )
497
            if document.uid:
498
                graph_database.create_link(
499
                    link_type=GraphLinkType.UID_TO_NODE,
500
                    lhs_node=document.uid,
501
                    rhs_node=document,
502
                )
503
 
504
            document_tags: Dict[str, int] = {}
505
            graph_database.create_link(
506
                link_type=GraphLinkType.DOCUMENT_TO_TAGS,
507
                lhs_node=document.reserved_mid,
508
                rhs_node=document_tags,
509
            )
510
 
511
            document_iterator = SDocDocumentIterator(document)
512
            d_01_document_iterators[document] = document_iterator
513
 
514
            for node, _ in document_iterator.all_content(
515
                print_fragments=False,
516
            ):
517
                if isinstance(node, SDocNode):
518
                    try:
519
                        assert document.grammar is not None
520
                        SDocValidator.validate_node(
521
                            node,
522
                            document_grammar=document.grammar,
523
                            path_to_sdoc_file=document.meta.input_doc_full_path,
524
                            auto_uid_mode=project_config.auto_uid_mode,
525
                        )
526
                    except StrictDocSemanticError as exc:
527
                        print(exc.to_print_message())  # noqa: T201
528
                        sys.exit(1)
529
 
530
                if graph_database.has_any_link(
531
                    link_type=GraphLinkType.MID_TO_NODE,
532
                    lhs_node=node.reserved_mid,
533
                ):
534
                    other_node: SDocDocument = graph_database.get_link_value(
535
                        link_type=GraphLinkType.MID_TO_NODE,
536
                        lhs_node=node.reserved_mid,
537
                    )
538
                    raise StrictDocException(
539
                        "TraceabilityIndex: "
540
                        "the node MID is not unique: "
541
                        f"{node.reserved_mid}. "
542
                        "All machine identifiers (MID) must be unique values. "
543
                        f"Affected nodes:\n"
544
                        f"{other_node.get_debug_info()}\n"
545
                        f"and\n"
546
                        f"{node.get_debug_info()}."
547
                    )
548
                graph_database.create_link(
549
                    link_type=GraphLinkType.MID_TO_NODE,
550
                    lhs_node=node.reserved_mid,
551
                    rhs_node=node,
552
                )
553
 
554
                if node.reserved_uid is not None:
555
                    # @relation(SDOC-SRS-29, scope=range_start)
556
                    if traceability_index.graph_database.has_any_link(
557
                        link_type=GraphLinkType.UID_TO_NODE,
558
                        lhs_node=node.reserved_uid,
559
                    ):
560
                        already_existing_node: SDocNode = (
561
                            traceability_index.graph_database.get_link_value(
562
                                link_type=GraphLinkType.UID_TO_NODE,
563
                                lhs_node=node.reserved_uid,
564
                            )
565
                        )
566
                        other_req_doc = assert_cast(
567
                            already_existing_node.get_document(), SDocDocument
568
                        )
569
                        if other_req_doc == document:
570
                            print(  # noqa: T201
571
                                "error: DocumentIndex: "
572
                                "two nodes with the same UID "
573
                                "exist in the same document: "
574
                                f'{node.reserved_uid} in "{document.title}".'
575
                            )
576
                        else:
577
                            print(  # noqa: T201
578
                                "error: DocumentIndex: "
579
                                "two nodes with the same UID "
580
                                "exist in two different documents: "
581
                                f'{node.reserved_uid} in "{other_req_doc.title}" '
582
                                f'and "{document.title}".'
583
                            )
584
                        sys.exit(1)
585
                    # @relation(SDOC-SRS-29, scope=range_end)
586
 
587
                    traceability_index.graph_database.create_link(
588
                        link_type=GraphLinkType.UID_TO_NODE,
589
                        lhs_node=node.reserved_uid,
590
                        rhs_node=node,
591
                    )
592
 
593
                if isinstance(node, SDocNode):
594
                    requirement_node: SDocNode = assert_cast(node, SDocNode)
595
                    if requirement_node.reserved_tags is not None:
596
                        for tag in requirement_node.reserved_tags:
597
                            document_tags.setdefault(tag, 0)
598
                            document_tags[tag] += 1
599
                    for node_field_ in node.enumerate_fields():
600
                        for part in node_field_.parts:
601
                            # The inline links are handled at the next big
602
                            # For loop pass because the information about
603
                            # all Nodes and Anchors have not been
604
                            # collected yet at this point.
605
                            # see create_inline_link below.
606
                            if isinstance(part, Anchor):
607
                                graph_database.create_link(
608
                                    link_type=GraphLinkType.MID_TO_NODE,
609
                                    lhs_node=part.mid,
610
                                    rhs_node=part,
611
                                )
612
                                graph_database.create_link(
613
                                    link_type=GraphLinkType.UID_TO_NODE,
614
                                    lhs_node=part.value,
615
                                    rhs_node=part,
616
                                )
617
 
618
        # Now iterate over the requirements again to build an in-depth map of
619
        # parents and children.
620
        requirement: SDocNode
621
 
622
        for document in document_tree.document_list:
623
            assert document.meta is not None
624
 
625
            document_iterator = d_01_document_iterators[document]
626
 
627
            if document.config.custom_metadata is not None:
628
                for metadata_entry_ in document.config.custom_metadata.entries:
629
                    for part in metadata_entry_.parts:
630
                        if isinstance(part, InlineLink):
631
                            traceability_index.pending_inline_links.append(part)
632
 
633
            for node, _ in document_iterator.all_content(
634
                print_fragments=False,
635
            ):
636
                if not isinstance(node, SDocNode):
637
                    continue
638
 
639
                requirement = assert_cast(node, SDocNode)
640
                for node_field_ in requirement.enumerate_fields():
641
                    for part in node_field_.parts:
642
                        if isinstance(part, InlineLink):
643
                            traceability_index.pending_inline_links.append(part)
644
                if requirement.reserved_uid is None:
645
                    continue
646
 
647
                # Now it is possible to resolve parents first checking if they
648
                # indeed exist.
649
                for reference in requirement.relations:
650
                    if reference.ref_type == ReferenceType.FILE:
651
                        d_07_file_traceability_index.create_requirement_with_forward_source_links(
652
                            requirement
653
                        )
654
                    elif reference.ref_type == ReferenceType.PARENT:
655
                        parent_reference: ParentReqReference = assert_cast(
656
                            reference, ParentReqReference
657
                        )
658
                        parent_requirement = traceability_index.graph_database.get_link_value_weak(
659
                            link_type=GraphLinkType.UID_TO_NODE,
660
                            lhs_node=parent_reference.ref_uid,
661
                        )
662
                        if parent_requirement is None:
663
                            raise StrictDocException(
664
                                f"[DocumentIndex.create] "
665
                                f"Requirement {requirement.reserved_uid} "
666
                                f"references "
667
                                f"parent requirement which doesn't exist: "
668
                                f"{parent_reference.ref_uid}."
669
                            )
670
                        traceability_index.graph_database.create_link(
671
                            link_type=GraphLinkType.NODE_TO_PARENT_NODES,
672
                            lhs_node=requirement,
673
                            rhs_node=parent_requirement,
674
                            edge=parent_reference.role,
675
                        )
676
                        traceability_index.graph_database.create_link(
677
                            link_type=GraphLinkType.NODE_TO_CHILD_NODES,
678
                            lhs_node=parent_requirement,
679
                            rhs_node=requirement,
680
                            edge=parent_reference.role,
681
                        )
682
 
683
                        # Set document dependencies.
684
                        parent_document: SDocDocument = assert_cast(
685
                            parent_requirement.get_document(), SDocDocument
686
                        )
687
                        if document != parent_document:
688
                            assert parent_document.meta is not None
689
 
690
                            # This is where we help the incremental generation to
691
                            # understand that the related documents must be
692
                            # re-generated together.
693
                            file_dependency_manager.add_dependency(
694
                                document.meta.input_doc_full_path,
695
                                parent_document.meta.output_document_full_path,
696
                            )
697
                            file_dependency_manager.add_dependency(
698
                                parent_document.meta.input_doc_full_path,
699
                                document.meta.output_document_full_path,
700
                            )
701
                    elif reference.ref_type == ReferenceType.CHILD:
702
                        child_reference: ChildReqReference = assert_cast(
703
                            reference, ChildReqReference
704
                        )
705
                        child_requirement = traceability_index.graph_database.get_link_value_weak(
706
                            link_type=GraphLinkType.UID_TO_NODE,
707
                            lhs_node=child_reference.ref_uid,
708
                        )
709
                        if child_requirement is None:
710
                            raise StrictDocException(
711
                                f"[DocumentIndex.create] "
712
                                f"Requirement {requirement.reserved_uid} "
713
                                f"references a "
714
                                f"child requirement that doesn't exist: "
715
                                f"{child_reference.ref_uid}."
716
                            )
717
                        traceability_index.graph_database.create_link(
718
                            link_type=GraphLinkType.NODE_TO_PARENT_NODES,
719
                            lhs_node=child_requirement,
720
                            rhs_node=requirement,
721
                            edge=child_reference.role,
722
                        )
723
                        traceability_index.graph_database.create_link(
724
                            link_type=GraphLinkType.NODE_TO_CHILD_NODES,
725
                            lhs_node=requirement,
726
                            rhs_node=child_requirement,
727
                            edge=child_reference.role,
728
                        )
729
                        # Set document dependencies.
730
                        child_requirement_document = assert_cast(
731
                            child_requirement.get_document(), SDocDocument
732
                        )
733
                        if document != child_requirement_document:
734
                            assert child_requirement_document.meta is not None
735
 
736
                            # This is where we help the incremental generation to
737
                            # understand that the related documents must be
738
                            # re-generated together.
739
                            file_dependency_manager.add_dependency(
740
                                document.meta.input_doc_full_path,
741
                                child_requirement_document.meta.output_document_full_path,
742
                            )
743
                            file_dependency_manager.add_dependency(
744
                                child_requirement_document.meta.input_doc_full_path,
745
                                document.meta.output_document_full_path,
746
                            )
747
                    else:
748
                        raise AssertionError(reference.ref_type)
749
 
750
        # Iterate for the third time to validate the graph against
751
        # requirement cycles.
752
        parents_cycle_detector = TreeCycleDetector()
753
        children_cycle_detector = TreeCycleDetector()
754
        for document in document_tree.document_list:
755
            document_iterator = d_01_document_iterators[document]
756
 
757
            for node, _ in document_iterator.all_content(
758
                print_fragments=False,
759
            ):
760
                if not isinstance(node, SDocNode):
761
                    continue
762
 
763
                requirement = assert_cast(node, SDocNode)
764
 
765
                if requirement.reserved_uid is None:
766
                    continue
767
 
768
                # @relation(SDOC-SRS-30, scope=range_start)
769
                # Detect cycles
770
                def parent_cycle_traverse_(node_id: str) -> Any:
771
                    current_node = (
772
                        traceability_index.graph_database.get_link_value(
773
                            link_type=GraphLinkType.UID_TO_NODE,
774
                            lhs_node=node_id,
775
                        )
776
                    )
777
                    return list(
778
                        map(
779
                            lambda node_: node_.reserved_uid,
780
                            traceability_index.graph_database.get_link_values(
781
                                link_type=GraphLinkType.NODE_TO_PARENT_NODES,
782
                                lhs_node=current_node,
783
                            ),
784
                        )
785
                    )
786
 
787
                parents_cycle_detector.check_node(
788
                    requirement.reserved_uid,
789
                    parent_cycle_traverse_,
790
                )
791
 
792
                def child_cycle_traverse_(node_id: str) -> Any:
793
                    current_node = (
794
                        traceability_index.graph_database.get_link_value(
795
                            link_type=GraphLinkType.UID_TO_NODE,
796
                            lhs_node=node_id,
797
                        )
798
                    )
799
                    return list(
800
                        map(
801
                            lambda node_: node_.reserved_uid,
802
                            traceability_index.graph_database.get_link_values(
803
                                link_type=GraphLinkType.NODE_TO_CHILD_NODES,
804
                                lhs_node=current_node,
805
                            ),
806
                        )
807
                    )
808
 
809
                children_cycle_detector.check_node(
810
                    requirement.reserved_uid,
811
                    child_cycle_traverse_,
812
                )
813
                # @relation(SDOC-SRS-30, scope=range_end)
814
 
815
        map_documents_by_input_rel_path: Dict[str, SDocDocument] = {}
816
        for document_ in document_tree.document_list:
817
            assert document_.meta is not None
818
 
819
            map_documents_by_input_rel_path[
820
                document_.meta.input_doc_full_path
821
            ] = document_
822
 
823
        # @relation(SDOC-SRS-109, scope=range_start)
824
        unique_document_from_file_occurences: Set[str] = set()
825
        for document_ in document_tree.document_list:
826
            document_from_file_: SDocDocumentFromFileIF
827
            for document_from_file_ in document_.fragments_from_files:
828
                traceability_index.contains_included_documents = True
829
 
830
                assert isinstance(document_from_file_, DocumentFromFile), (
831
                    document_from_file_
832
                )
833
 
834
                assert (
835
                    document_from_file_.resolved_full_path_to_document_file
836
                    is not None
837
                )
838
 
839
                if (
840
                    document_from_file_.resolved_full_path_to_document_file
841
                    not in map_documents_by_input_rel_path
842
                ):
843
                    raise StrictDocException(
844
                        "A document includes contains a link to another document "
845
                        "which is not resolved in the current documentation tree: "
846
                        f"'{document_from_file_.file}'. This can happen if a single "
847
                        f"document path is provided as input to a StrictDoc command. "
848
                        f"Try providing a path to a folder where all documents "
849
                        f"are stored."
850
                    )
851
                resolved_document: SDocDocument = (
852
                    map_documents_by_input_rel_path[
853
                        document_from_file_.resolved_full_path_to_document_file
854
                    ]
855
                )
856
 
857
                if (
858
                    document_from_file_.resolved_full_path_to_document_file
859
                    in unique_document_from_file_occurences
860
                ) and resolved_document.has_any_requirements():
861
                    raise StrictDocException(
862
                        "[DOCUMENT_FROM_FILE]: "
863
                        "A multiple inclusion of a document is detected. "
864
                        "A document that contains requirements or other nodes "
865
                        "can be only included once: "
866
                        f"{document_from_file_.file}."
867
                    )
868
                unique_document_from_file_occurences.add(
869
                    document_from_file_.resolved_full_path_to_document_file
870
                )
871
 
872
                document_from_file_.configure_with_resolved_document(
873
                    resolved_document
874
                )
875
 
876
        # @relation(SDOC-SRS-109, scope=range_end)
877
 
878
        return traceability_index
879
 
880
    @classmethod
881
    def _create_filter(
882
        cls, traceability_index: Any, filter_query: str
883
    ) -> "NodeFilter":
884
        query_reader = QueryReader()
885
        requirements_query_object: Union[QueryObject, QueryNullObject]
886
        try:
887
            requirements_query = query_reader.read(filter_query)
888
            requirements_query_object = QueryObject(
889
                requirements_query, traceability_index
890
            )
891
        except TextXSyntaxError as textx_syntax_error_:
892
            raise StrictDocException(
893
                "Cannot parse filter query."
894
            ) from textx_syntax_error_
895
 
896
        blacklisted_nodes: set[SDocElementIF] = set()
897
 
898
        try:
899
            for document in traceability_index.document_tree.document_list:
900
                document_iterator = traceability_index.get_document_iterator(
901
                    document
902
                )
903
                for node, _ in document_iterator.all_content():
904
                    if (
905
                        isinstance(node, SDocNode)
906
                        and node.node_type == "SECTION"
907
                        and not requirements_query_object.evaluate(node)
908
                    ):
909
                        blacklisted_nodes.add(node)
910
 
911
                        # If the node is the last one, we check if all other
912
                        # nodes are filtered out and if so, mark the parent
913
                        # section node as not whitelisted as well.
914
                        if (
915
                            node.parent.section_contents[
916
                                len(node.parent.section_contents) - 1
917
                            ]
918
                            == node
919
                        ):
920
                            if (
921
                                isinstance(node.parent, SDocNode)
922
                                and node.parent.node_type == "SECTION"
923
                            ):
924
                                cls._blacklist_if_needed(
925
                                    blacklisted_nodes, node.parent
926
                                )
927
 
928
                    elif isinstance(
929
                        node, SDocNode
930
                    ) and not requirements_query_object.evaluate(node):
931
                        blacklisted_nodes.add(node)
932
                        # If the node is the last one, we check if all other
933
                        # nodes are filtered out and if so, mark the parent
934
                        # section node as not whitelisted as well.
935
                        if (
936
                            node.parent.section_contents[
937
                                len(node.parent.section_contents) - 1
938
                            ]
939
                            == node
940
                        ):
941
                            cls._blacklist_if_needed(
942
                                blacklisted_nodes, node.parent
943
                            )
944
 
945
        except (AttributeError, NameError, TypeError) as attribute_error_:
946
            raise StrictDocException(
947
                f"Cannot apply a filter query to a node: {attribute_error_}"
948
            ) from attribute_error_
949
 
950
        return NodeFilter(blacklisted_nodes)
951
 
952
    @classmethod
953
    def _blacklist_if_needed(
954
        cls,
955
        blacklisted_nodes: set[SDocElementIF],
956
        node: SDocElementIF,
957
    ) -> None:
958
        if isinstance(node, SDocDocumentFromFileIF):
959
            return
960
 
961
        if node.section_contents is not None:
962
            for node_ in node.section_contents:
963
                if node_ not in blacklisted_nodes:
964
                    return
965
 
966
        blacklisted_nodes.add(node)
967
 
968
        # If it turns out that all child nodes are blacklisted,
969
        # go up and blacklist the parent node if needed.
970
        if (
971
            isinstance(node, SDocNodeIF)
972
            and node.parent not in blacklisted_nodes
973
        ):
974
            cls._blacklist_if_needed(blacklisted_nodes, node.parent)
975
 
976
    @staticmethod
977
    def source_node_parser_tags(
978
        cfg_entry: SourceNodesEntry, grammar_element: GrammarElement
979
    ) -> set[str]:
980
        tags = set(grammar_element.get_field_titles())
981
        # For remapped fields, don't parse the names from grammar but those from the mapping.
982
        for (
983
            sdoc_field_name,
984
            source_field_name,
985
        ) in cfg_entry.sdoc_to_source_map.items():
986
            tags.remove(sdoc_field_name)
987
            tags.add(source_field_name)
988
        return tags