StrictDoc Documentation
strictdoc/core/traceability_index_builder.py
Source file coverage
Path:
strictdoc/core/traceability_index_builder.py
Lines:
979
Non-empty lines:
889
Non-empty lines covered with requirements:
889 / 889 (100.0%)
Functions:
8
Functions covered by requirements:
8 / 8 (100.0%)
1
"""
2
@relation(SDOC-SRS-28, SDOC-SRS-2, scope=file)
3
"""
4
 
5
import datetime
6
import glob
7
import os
8
import posixpath
9
import sys
10
from typing import Any, Dict, Iterator, List, Optional, Set, Union
11
 
12
from textx import TextXSyntaxError
13
 
14
from strictdoc.backend.markdown.reader import SDMarkdownReader
15
from strictdoc.backend.sdoc.constants import SDocMarkup
16
from strictdoc.backend.sdoc.error_handling import StrictDocSemanticError
17
from strictdoc.backend.sdoc.models.anchor import Anchor
18
from strictdoc.backend.sdoc.models.document import SDocDocument
19
from strictdoc.backend.sdoc.models.document_from_file import DocumentFromFile
20
from strictdoc.backend.sdoc.models.document_grammar import DocumentGrammar
21
from strictdoc.backend.sdoc.models.grammar_element import (
22
    GrammarElement,
23
    ReferenceType,
24
)
25
from strictdoc.backend.sdoc.models.inline_link import InlineLink
26
from strictdoc.backend.sdoc.models.model import (
27
    SDocDocumentFromFileIF,
28
    SDocElementIF,
29
    SDocNodeIF,
30
)
31
from strictdoc.backend.sdoc.models.node import SDocNode
32
from strictdoc.backend.sdoc.models.reference import (
33
    ChildReqReference,
34
    ParentReqReference,
35
)
36
from strictdoc.backend.sdoc.node_filter import NodeFilter
37
from strictdoc.backend.sdoc.validations.sdoc_validator import SDocValidator
38
from strictdoc.backend.sdoc_source_code.caching_reader import (
39
    SourceFileTraceabilityCachingReader,
40
)
41
from strictdoc.core.constants import GraphLinkType
42
from strictdoc.core.document_iterator import SDocDocumentIterator
43
from strictdoc.core.document_tree import DocumentTree
44
from strictdoc.core.file_dependency_manager import FileDependencyManager
45
from strictdoc.core.file_system.document_finder import DocumentFinder
46
from strictdoc.core.file_system.source_files_finder import (
47
    SourceFilesFinder,
48
)
49
from strictdoc.core.file_system.source_tree import SourceFile, SourceTree
50
from strictdoc.core.file_traceability_index import FileTraceabilityIndex
51
from strictdoc.core.graph.many_to_many_set import ManyToManySet
52
from strictdoc.core.graph.one_to_one_dictionary import OneToOneDictionary
53
from strictdoc.core.graph_database import GraphDatabase
54
from strictdoc.core.project_config import (
55
    ProjectConfig,
56
    ProjectFeature,
57
    SourceNodesEntry,
58
)
59
from strictdoc.core.query_engine.query_object import (
60
    QueryNullObject,
61
    QueryObject,
62
)
63
from strictdoc.core.query_engine.query_reader import QueryReader
64
from strictdoc.core.traceability_index import (
65
    TraceabilityIndex,
66
)
67
from strictdoc.core.tree_cycle_detector import TreeCycleDetector
68
from strictdoc.helpers.cast import assert_cast
69
from strictdoc.helpers.deprecation_engine import DEPRECATION_ENGINE
70
from strictdoc.helpers.exception import StrictDocException
71
from strictdoc.helpers.file_modification_time import (
72
    get_file_modification_time,
73
)
74
from strictdoc.helpers.mid import MID
75
from strictdoc.helpers.parallelizer import Parallelizer
76
from strictdoc.helpers.timing import measure_performance, timing_decorator
77
 
78
 
79
class TraceabilityIndexBuilder:
80
    @classmethod
81
    def create(
82
        cls,
83
        *,
84
        project_config: ProjectConfig,
85
        parallelizer: Parallelizer,
86
        skip_source_files: bool = False,
87
    ) -> TraceabilityIndex:
88
        # TODO: It would be great to hide this code behind --development flag.
89
        # There is no need for this to be activated in the Pip-released builds.
90
        strict_own_files_unfiltered: Iterator[str] = glob.iglob(
91
            f"{project_config.get_strictdoc_root_path()}/strictdoc/**/*",
92
            recursive=True,
93
        )
94
        strict_own_files: List[str] = [
95
            f
96
            for f in strict_own_files_unfiltered
97
            if f.endswith(".html")
98
            or f.endswith(".py")
99
            or f.endswith(".jinja")
100
            or f.endswith(".svg")
101
        ]
102
        latest_strictdoc_own_file = (
103
            max(strict_own_files, key=os.path.getctime)
104
            if len(strict_own_files) > 0
105
            else None
106
        )
107
 
108
        strictdoc_last_update: datetime.datetime = (
109
            get_file_modification_time(latest_strictdoc_own_file)
110
            if (latest_strictdoc_own_file is not None)
111
            else datetime.datetime.fromtimestamp(0)
112
        )
113
        if (
114
            project_config.config_last_update is not None
115
            and project_config.config_last_update > strictdoc_last_update
116
        ):
117
            strictdoc_last_update = project_config.config_last_update
118
 
119
        document_tree, asset_manager = DocumentFinder.find_sdoc_content(
120
            project_config=project_config, parallelizer=parallelizer
121
        )
122
 
123
        # TODO: This is rather messy, but it is better than it used to be.
124
        # Currently, the traceability index holds everything that is later used
125
        # by HTML generators:
126
        # - traceability index itself
127
        # - document tree
128
        # - assets
129
        # - runtime configuration.
130
        traceability_index: TraceabilityIndex = (
131
            TraceabilityIndexBuilder.create_from_document_tree(
132
                document_tree,
133
                project_config,
134
            )
135
        )
136
        traceability_index.asset_manager = asset_manager
137
        traceability_index.strictdoc_last_update = strictdoc_last_update
138
 
139
        if node_filter_query := project_config.filter_nodes:
140
            traceability_index.node_filter = cls._create_filter(
141
                traceability_index=traceability_index,
142
                filter_query=node_filter_query,
143
            )
144
 
145
        #
146
        # File traceability-related calculations.
147
        #
148
        if not skip_source_files and project_config.is_feature_activated(
149
            ProjectFeature.REQUIREMENT_TO_SOURCE_TRACEABILITY
150
        ):
151
            file_tracability_index = (
152
                traceability_index.get_file_traceability_index()
153
            )
154
 
155
            with measure_performance("Find source files"):
156
                source_tree: SourceTree = SourceFilesFinder.find_source_files(
157
                    project_config=project_config
158
                )
159
 
160
            source_files = source_tree.source_files
161
            source_file: SourceFile
162
            for source_file in source_files:
163
                with measure_performance(
164
                    f"Reading source: {source_file.in_doctree_source_file_rel_path}"
165
                ):
166
                    source_nodes_cfg_entry = (
167
                        project_config.get_relevant_source_nodes_entry(
168
                            source_file.full_path
169
                        )
170
                    )
171
                    if source_nodes_cfg_entry is not None:
172
                        source_node_grammar_element = (
173
                            traceability_index.get_grammar_element(
174
                                source_nodes_cfg_entry.uid,
175
                                source_nodes_cfg_entry.node_type,
176
                            )
177
                        )
178
                        assert source_node_grammar_element is not None, (
179
                            "Missing grammar element for node: "
180
                            f"{source_nodes_cfg_entry.uid} {source_nodes_cfg_entry.node_type}"
181
                        )
182
                        source_node_tags = (
183
                            TraceabilityIndexBuilder.source_node_parser_tags(
184
                                source_nodes_cfg_entry,
185
                                source_node_grammar_element,
186
                            )
187
                        )
188
                    else:
189
                        source_node_tags = None
190
 
191
                    traceability_info = (
192
                        SourceFileTraceabilityCachingReader.read_from_file(
193
                            source_file.full_path,
194
                            project_config,
195
                            source_node_tags,
196
                        )
197
                    )
198
 
199
                if traceability_info:
200
                    traceability_index.create_traceability_info(
201
                        source_file,
202
                        traceability_info,
203
                    )
204
                    # Is file referenced by backwards links?
205
                    if len(traceability_info.markers) > 0:
206
                        source_file.is_referenced = True
207
 
208
            file_tracability_index.validate_and_resolve(
209
                traceability_index, project_config
210
            )
211
 
212
            # Iterate again to resolve if the file is referenced.
213
            # FIXME: Not great to iterate two times.
214
            for source_file in file_tracability_index.indexed_source_files():
215
                # Is file referenced by forward links?
216
                is_source_file_referenced = (
217
                    traceability_index.has_source_file_reqs(
218
                        source_file.in_doctree_source_file_rel_path_posix
219
                    )
220
                )
221
                if is_source_file_referenced:
222
                    source_file.is_referenced = True
223
 
224
                    source_file_reqs: Optional[List[SDocNode]] = (
225
                        traceability_index.get_source_file_reqs(
226
                            source_file.in_doctree_source_file_rel_path_posix
227
                        )
228
                    )
229
                    if source_file_reqs is None:
230
                        continue
231
 
232
                    for node_ in source_file_reqs:
233
                        node_document = assert_cast(
234
                            node_.get_document(), SDocDocument
235
                        )
236
                        assert node_document.meta is not None
237
 
238
                        traceability_index.file_dependency_manager.add_dependency(
239
                            source_file.full_path,
240
                            source_file.output_file_full_path,
241
                        )
242
                        traceability_index.file_dependency_manager.add_dependency(
243
                            source_file.full_path,
244
                            node_document.meta.output_document_full_path,
245
                        )
246
                        traceability_index.file_dependency_manager.add_dependency(
247
                            node_document.meta.input_doc_full_path,
248
                            source_file.output_file_full_path,
249
                        )
250
 
251
            traceability_index.document_tree.attach_source_tree(source_tree)
252
 
253
        #
254
        # Resolve pending InlineLinks. This depends on UIDs and anchors from
255
        # static documents, generated documents and source nodes.
256
        #
257
        for inline_link in traceability_index.pending_inline_links:
258
            if not traceability_index.graph_database.has_any_link(
259
                link_type=GraphLinkType.UID_TO_NODE,
260
                lhs_node=inline_link.link,
261
            ):
262
                raise StrictDocException(
263
                    "DocumentIndex: "
264
                    "the inline link references an object with an UID "
265
                    "that does not exist: "
266
                    f"{inline_link.link}."
267
                )
268
            traceability_index.create_inline_link(inline_link)
269
        traceability_index.pending_inline_links.clear()
270
 
271
        #
272
        # Resolve all modification dates to support the incremental generation of
273
        # all artifacts.
274
        #
275
 
276
        file_dependency_manager = traceability_index.file_dependency_manager
277
 
278
        file_dependency_manager.resolve_modification_dates(
279
            traceability_index.strictdoc_last_update
280
        )
281
 
282
        if project_config.user_plugin is not None:
283
            project_config.user_plugin.traceability_index_build_finished(
284
                traceability_index
285
            )
286
 
287
        return traceability_index
288
 
289
    @staticmethod
290
    @timing_decorator("Build traceability graph")
291
    def create_from_document_tree(
292
        document_tree: DocumentTree,
293
        project_config: ProjectConfig,
294
    ) -> TraceabilityIndex:
295
        """
296
        @relation(SDOC-SRS-32, SDOC-SRS-102, scope=function)
297
        """
298
 
299
        # FIXME: Too many things going on below. Would be great to simplify this
300
        # workflow.
301
        d_01_document_iterators: Dict[SDocDocument, SDocDocumentIterator] = {}
302
        d_07_file_traceability_index = FileTraceabilityIndex()
303
 
304
        graph_database = GraphDatabase(
305
            [
306
                (
307
                    GraphLinkType.MID_TO_NODE,
308
                    OneToOneDictionary(
309
                        MID,
310
                        (
311
                            SDocNode,
312
                            SDocDocument,
313
                            InlineLink,
314
                            Anchor,
315
                        ),
316
                    ),
317
                ),
318
                (
319
                    GraphLinkType.UID_TO_NODE,
320
                    OneToOneDictionary(str, (SDocDocument, SDocNode, Anchor)),
321
                ),
322
                (
323
                    GraphLinkType.NODE_TO_PARENT_NODES,
324
                    ManyToManySet(SDocNode, SDocNode),
325
                ),
326
                (
327
                    GraphLinkType.NODE_TO_CHILD_NODES,
328
                    ManyToManySet(SDocNode, SDocNode),
329
                ),
330
                (
331
                    GraphLinkType.NODE_TO_INCOMING_LINKS,
332
                    ManyToManySet(MID, InlineLink),
333
                ),
334
                (
335
                    GraphLinkType.DOCUMENT_TO_TAGS,
336
                    OneToOneDictionary(MID, dict),
337
                ),
338
            ]
339
        )
340
 
341
        file_dependency_manager: FileDependencyManager = (
342
            FileDependencyManager.create_from_cache(
343
                project_config=project_config
344
            )
345
        )
346
 
347
        traceability_index = TraceabilityIndex(
348
            document_tree,
349
            d_01_document_iterators,
350
            file_traceability_index=d_07_file_traceability_index,
351
            graph_database=graph_database,
352
            file_dependency_manager=file_dependency_manager,
353
        )
354
 
355
        # It seems to be impossible to accomplish everything in just one for
356
        # loop. One particular problem that requires two passes: it is not
357
        # possible to know after one iteration which of the requirements
358
        # parents do not exist for each given requirement.
359
        #
360
        # Step #1:
361
        # - Collect a dictionary of all requirements in the document tree:
362
        # {req_id: req}  # noqa: ERA001
363
        # - Each requirement's 'parents_uids' is populated with the forward
364
        # declarations of its parents uids.
365
        # - A separate map is created: {req_id: [req_children]}
366
        # At this point some information is in place, but it was not known if
367
        # some UIDs could not be resolved which is the task of the second
368
        # step.
369
        #
370
        # Step #2:
371
        # - Check if each requirement's has valid parent relations.
372
        # - Resolve parent forward declarations
373
        # - Re-assign children declarations
374
        # - Detect cycles
375
        # - Calculate depth of both parent and child relations.
376
        for (
377
            path_to_grammar_,
378
            grammar_from_file_,
379
        ) in document_tree.map_grammars_by_filenames.items():
380
            try:
381
                SDocValidator.validate_grammar_from_file(
382
                    path_to_grammar_, grammar_from_file_
383
                )
384
            except StrictDocSemanticError as exc:
385
                print(exc.to_print_message())  # noqa: T201
386
                sys.exit(1)
387
 
388
        document: SDocDocument
389
        for document in document_tree.document_list:
390
            assert document.grammar is not None
391
            assert document.meta is not None
392
 
393
            traceability_index.file_dependency_manager.add_dependency(
394
                document.meta.input_doc_full_path,
395
                document.meta.output_document_full_path,
396
            )
397
 
398
            if document.config.view_style_tag == "REQUIREMENT_STYLE":
399
                DEPRECATION_ENGINE.add_message(
400
                    "DEPRECATED_REQUIREMENT_STYLE",
401
                    "WARNING: REQUIREMENT_STYLE is deprecated. Replace it to VIEW_STYLE.",
402
                )
403
            if document.config.node_in_toc_tag == "REQUIREMENT_IN_TOC":
404
                DEPRECATION_ENGINE.add_message(
405
                    "DEPRECATED_REQUIREMENT_IN_TOC",
406
                    "WARNING: REQUIREMENT_IN_TOC is deprecated. Replace it to NODE_IN_TOC.",
407
                )
408
 
409
            #
410
            # First, resolve all grammars that are imported from grammar files.
411
            #
412
            if document.grammar.import_from_file is not None:
413
                grammar_path = document.grammar.import_from_file
414
                if grammar_path.startswith("@"):
415
                    grammar_path = project_config.grammars[grammar_path]
416
                else:
417
                    grammar_path = posixpath.join(
418
                        document.meta.input_doc_dir_rel_path.relative_path_posix,
419
                        grammar_path,
420
                    )
421
                document_grammar: Optional[DocumentGrammar] = (
422
                    document_tree.get_grammar_by_filename(grammar_path)
423
                )
424
                if document_grammar is None:
425
                    raise StrictDocException(
426
                        "TraceabilityIndex: "
427
                        f'the document "{document.reserved_title}" '
428
                        "imports a grammar from a file that does not exist: "
429
                        f'"{document.grammar.import_from_file}". One known '
430
                        f"source of this error is when only a single document "
431
                        f"file is provided as input to the export or server "
432
                        f"command, rather than the containing folder. To locate "
433
                        f"the grammar file, StrictDoc needs to be able to "
434
                        f"resolve it relative to the input path."
435
                    )
436
 
437
                document.grammar.update_with_elements(document_grammar.elements)
438
 
439
                # This is for the backward compatibility with the existing users.
440
                # If the included project grammar has no TEXT element defined,
441
                # we add it here automatically.
442
                if not document.grammar.has_text_element():
443
                    document.grammar.add_element_first(
444
                        DocumentGrammar.create_default_text_element(
445
                            document.grammar,
446
                            enable_mid=document.config.enable_mid is True,
447
                        )
448
                    )
449
 
450
            # This is important because due to the difference between the
451
            # normal grammar vs imported grammar, the parent may not be set at
452
            # this point.
453
            document.grammar.parent = document
454
 
455
            if document.config.markup == SDocMarkup.MARKDOWN:
456
                SDMarkdownReader.fixup_composite_nodes(document)
457
 
458
            try:
459
                SDocValidator.validate_document(document)
460
            except StrictDocSemanticError as exc:
461
                print(exc.to_print_message())  # noqa: T201
462
                sys.exit(1)
463
 
464
            if graph_database.has_any_link(
465
                link_type=GraphLinkType.MID_TO_NODE,
466
                lhs_node=document.reserved_mid,
467
            ):
468
                other_document: SDocDocument = graph_database.get_link_value(
469
                    link_type=GraphLinkType.MID_TO_NODE,
470
                    lhs_node=document.reserved_mid,
471
                )
472
                raise StrictDocException(
473
                    "TraceabilityIndex: "
474
                    "the document MID is not unique: "
475
                    f"{document.reserved_mid}. "
476
                    "All machine identifiers (MID) must be unique values. "
477
                    f"Affected documents:\n"
478
                    f"{other_document.get_debug_info()}\n"
479
                    f"and\n"
480
                    f"{document.get_debug_info()}."
481
                )
482
 
483
            graph_database.create_link(
484
                link_type=GraphLinkType.MID_TO_NODE,
485
                lhs_node=document.reserved_mid,
486
                rhs_node=document,
487
            )
488
            if document.uid:
489
                graph_database.create_link(
490
                    link_type=GraphLinkType.UID_TO_NODE,
491
                    lhs_node=document.uid,
492
                    rhs_node=document,
493
                )
494
 
495
            document_tags: Dict[str, int] = {}
496
            graph_database.create_link(
497
                link_type=GraphLinkType.DOCUMENT_TO_TAGS,
498
                lhs_node=document.reserved_mid,
499
                rhs_node=document_tags,
500
            )
501
 
502
            document_iterator = SDocDocumentIterator(document)
503
            d_01_document_iterators[document] = document_iterator
504
 
505
            for node, _ in document_iterator.all_content(
506
                print_fragments=False,
507
            ):
508
                if isinstance(node, SDocNode):
509
                    try:
510
                        assert document.grammar is not None
511
                        SDocValidator.validate_node(
512
                            node,
513
                            document_grammar=document.grammar,
514
                            path_to_sdoc_file=document.meta.input_doc_full_path,
515
                            auto_uid_mode=project_config.auto_uid_mode,
516
                        )
517
                    except StrictDocSemanticError as exc:
518
                        print(exc.to_print_message())  # noqa: T201
519
                        sys.exit(1)
520
 
521
                if graph_database.has_any_link(
522
                    link_type=GraphLinkType.MID_TO_NODE,
523
                    lhs_node=node.reserved_mid,
524
                ):
525
                    other_node: SDocDocument = graph_database.get_link_value(
526
                        link_type=GraphLinkType.MID_TO_NODE,
527
                        lhs_node=node.reserved_mid,
528
                    )
529
                    raise StrictDocException(
530
                        "TraceabilityIndex: "
531
                        "the node MID is not unique: "
532
                        f"{node.reserved_mid}. "
533
                        "All machine identifiers (MID) must be unique values. "
534
                        f"Affected nodes:\n"
535
                        f"{other_node.get_debug_info()}\n"
536
                        f"and\n"
537
                        f"{node.get_debug_info()}."
538
                    )
539
                graph_database.create_link(
540
                    link_type=GraphLinkType.MID_TO_NODE,
541
                    lhs_node=node.reserved_mid,
542
                    rhs_node=node,
543
                )
544
 
545
                if node.reserved_uid is not None:
546
                    # @relation(SDOC-SRS-29, scope=range_start)
547
                    if traceability_index.graph_database.has_any_link(
548
                        link_type=GraphLinkType.UID_TO_NODE,
549
                        lhs_node=node.reserved_uid,
550
                    ):
551
                        already_existing_node: SDocNode = (
552
                            traceability_index.graph_database.get_link_value(
553
                                link_type=GraphLinkType.UID_TO_NODE,
554
                                lhs_node=node.reserved_uid,
555
                            )
556
                        )
557
                        other_req_doc = assert_cast(
558
                            already_existing_node.get_document(), SDocDocument
559
                        )
560
                        if other_req_doc == document:
561
                            print(  # noqa: T201
562
                                "error: DocumentIndex: "
563
                                "two nodes with the same UID "
564
                                "exist in the same document: "
565
                                f'{node.reserved_uid} in "{document.title}".'
566
                            )
567
                        else:
568
                            print(  # noqa: T201
569
                                "error: DocumentIndex: "
570
                                "two nodes with the same UID "
571
                                "exist in two different documents: "
572
                                f'{node.reserved_uid} in "{other_req_doc.title}" '
573
                                f'and "{document.title}".'
574
                            )
575
                        sys.exit(1)
576
                    # @relation(SDOC-SRS-29, scope=range_end)
577
 
578
                    traceability_index.graph_database.create_link(
579
                        link_type=GraphLinkType.UID_TO_NODE,
580
                        lhs_node=node.reserved_uid,
581
                        rhs_node=node,
582
                    )
583
 
584
                if isinstance(node, SDocNode):
585
                    requirement_node: SDocNode = assert_cast(node, SDocNode)
586
                    if requirement_node.reserved_tags is not None:
587
                        for tag in requirement_node.reserved_tags:
588
                            document_tags.setdefault(tag, 0)
589
                            document_tags[tag] += 1
590
                    for node_field_ in node.enumerate_fields():
591
                        for part in node_field_.parts:
592
                            # The inline links are handled at the next big
593
                            # For loop pass because the information about
594
                            # all Nodes and Anchors have not been
595
                            # collected yet at this point.
596
                            # see create_inline_link below.
597
                            if isinstance(part, Anchor):
598
                                graph_database.create_link(
599
                                    link_type=GraphLinkType.MID_TO_NODE,
600
                                    lhs_node=part.mid,
601
                                    rhs_node=part,
602
                                )
603
                                graph_database.create_link(
604
                                    link_type=GraphLinkType.UID_TO_NODE,
605
                                    lhs_node=part.value,
606
                                    rhs_node=part,
607
                                )
608
 
609
        # Now iterate over the requirements again to build an in-depth map of
610
        # parents and children.
611
        requirement: SDocNode
612
 
613
        for document in document_tree.document_list:
614
            assert document.meta is not None
615
 
616
            document_iterator = d_01_document_iterators[document]
617
 
618
            if document.config.custom_metadata is not None:
619
                for metadata_entry_ in document.config.custom_metadata.entries:
620
                    for part in metadata_entry_.parts:
621
                        if isinstance(part, InlineLink):
622
                            traceability_index.pending_inline_links.append(part)
623
 
624
            for node, _ in document_iterator.all_content(
625
                print_fragments=False,
626
            ):
627
                if not isinstance(node, SDocNode):
628
                    continue
629
 
630
                requirement = assert_cast(node, SDocNode)
631
                for node_field_ in requirement.enumerate_fields():
632
                    for part in node_field_.parts:
633
                        if isinstance(part, InlineLink):
634
                            traceability_index.pending_inline_links.append(part)
635
                if requirement.reserved_uid is None:
636
                    continue
637
 
638
                # Now it is possible to resolve parents first checking if they
639
                # indeed exist.
640
                for reference in requirement.relations:
641
                    if reference.ref_type == ReferenceType.FILE:
642
                        d_07_file_traceability_index.create_requirement_with_forward_source_links(
643
                            requirement
644
                        )
645
                    elif reference.ref_type == ReferenceType.PARENT:
646
                        parent_reference: ParentReqReference = assert_cast(
647
                            reference, ParentReqReference
648
                        )
649
                        parent_requirement = traceability_index.graph_database.get_link_value_weak(
650
                            link_type=GraphLinkType.UID_TO_NODE,
651
                            lhs_node=parent_reference.ref_uid,
652
                        )
653
                        if parent_requirement is None:
654
                            raise StrictDocException(
655
                                f"[DocumentIndex.create] "
656
                                f"Requirement {requirement.reserved_uid} "
657
                                f"references "
658
                                f"parent requirement which doesn't exist: "
659
                                f"{parent_reference.ref_uid}."
660
                            )
661
                        traceability_index.graph_database.create_link(
662
                            link_type=GraphLinkType.NODE_TO_PARENT_NODES,
663
                            lhs_node=requirement,
664
                            rhs_node=parent_requirement,
665
                            edge=parent_reference.role,
666
                        )
667
                        traceability_index.graph_database.create_link(
668
                            link_type=GraphLinkType.NODE_TO_CHILD_NODES,
669
                            lhs_node=parent_requirement,
670
                            rhs_node=requirement,
671
                            edge=parent_reference.role,
672
                        )
673
 
674
                        # Set document dependencies.
675
                        parent_document: SDocDocument = assert_cast(
676
                            parent_requirement.get_document(), SDocDocument
677
                        )
678
                        if document != parent_document:
679
                            assert parent_document.meta is not None
680
 
681
                            # This is where we help the incremental generation to
682
                            # understand that the related documents must be
683
                            # re-generated together.
684
                            file_dependency_manager.add_dependency(
685
                                document.meta.input_doc_full_path,
686
                                parent_document.meta.output_document_full_path,
687
                            )
688
                            file_dependency_manager.add_dependency(
689
                                parent_document.meta.input_doc_full_path,
690
                                document.meta.output_document_full_path,
691
                            )
692
                    elif reference.ref_type == ReferenceType.CHILD:
693
                        child_reference: ChildReqReference = assert_cast(
694
                            reference, ChildReqReference
695
                        )
696
                        child_requirement = traceability_index.graph_database.get_link_value_weak(
697
                            link_type=GraphLinkType.UID_TO_NODE,
698
                            lhs_node=child_reference.ref_uid,
699
                        )
700
                        if child_requirement is None:
701
                            raise StrictDocException(
702
                                f"[DocumentIndex.create] "
703
                                f"Requirement {requirement.reserved_uid} "
704
                                f"references a "
705
                                f"child requirement that doesn't exist: "
706
                                f"{child_reference.ref_uid}."
707
                            )
708
                        traceability_index.graph_database.create_link(
709
                            link_type=GraphLinkType.NODE_TO_PARENT_NODES,
710
                            lhs_node=child_requirement,
711
                            rhs_node=requirement,
712
                            edge=child_reference.role,
713
                        )
714
                        traceability_index.graph_database.create_link(
715
                            link_type=GraphLinkType.NODE_TO_CHILD_NODES,
716
                            lhs_node=requirement,
717
                            rhs_node=child_requirement,
718
                            edge=child_reference.role,
719
                        )
720
                        # Set document dependencies.
721
                        child_requirement_document = assert_cast(
722
                            child_requirement.get_document(), SDocDocument
723
                        )
724
                        if document != child_requirement_document:
725
                            assert child_requirement_document.meta is not None
726
 
727
                            # This is where we help the incremental generation to
728
                            # understand that the related documents must be
729
                            # re-generated together.
730
                            file_dependency_manager.add_dependency(
731
                                document.meta.input_doc_full_path,
732
                                child_requirement_document.meta.output_document_full_path,
733
                            )
734
                            file_dependency_manager.add_dependency(
735
                                child_requirement_document.meta.input_doc_full_path,
736
                                document.meta.output_document_full_path,
737
                            )
738
                    else:
739
                        raise AssertionError(reference.ref_type)
740
 
741
        # Iterate for the third time to validate the graph against
742
        # requirement cycles.
743
        parents_cycle_detector = TreeCycleDetector()
744
        children_cycle_detector = TreeCycleDetector()
745
        for document in document_tree.document_list:
746
            document_iterator = d_01_document_iterators[document]
747
 
748
            for node, _ in document_iterator.all_content(
749
                print_fragments=False,
750
            ):
751
                if not isinstance(node, SDocNode):
752
                    continue
753
 
754
                requirement = assert_cast(node, SDocNode)
755
 
756
                if requirement.reserved_uid is None:
757
                    continue
758
 
759
                # @relation(SDOC-SRS-30, scope=range_start)
760
                # Detect cycles
761
                def parent_cycle_traverse_(node_id: str) -> Any:
762
                    current_node = (
763
                        traceability_index.graph_database.get_link_value(
764
                            link_type=GraphLinkType.UID_TO_NODE,
765
                            lhs_node=node_id,
766
                        )
767
                    )
768
                    return list(
769
                        map(
770
                            lambda node_: node_.reserved_uid,
771
                            traceability_index.graph_database.get_link_values(
772
                                link_type=GraphLinkType.NODE_TO_PARENT_NODES,
773
                                lhs_node=current_node,
774
                            ),
775
                        )
776
                    )
777
 
778
                parents_cycle_detector.check_node(
779
                    requirement.reserved_uid,
780
                    parent_cycle_traverse_,
781
                )
782
 
783
                def child_cycle_traverse_(node_id: str) -> Any:
784
                    current_node = (
785
                        traceability_index.graph_database.get_link_value(
786
                            link_type=GraphLinkType.UID_TO_NODE,
787
                            lhs_node=node_id,
788
                        )
789
                    )
790
                    return list(
791
                        map(
792
                            lambda node_: node_.reserved_uid,
793
                            traceability_index.graph_database.get_link_values(
794
                                link_type=GraphLinkType.NODE_TO_CHILD_NODES,
795
                                lhs_node=current_node,
796
                            ),
797
                        )
798
                    )
799
 
800
                children_cycle_detector.check_node(
801
                    requirement.reserved_uid,
802
                    child_cycle_traverse_,
803
                )
804
                # @relation(SDOC-SRS-30, scope=range_end)
805
 
806
        map_documents_by_input_rel_path: Dict[str, SDocDocument] = {}
807
        for document_ in document_tree.document_list:
808
            assert document_.meta is not None
809
 
810
            map_documents_by_input_rel_path[
811
                document_.meta.input_doc_full_path
812
            ] = document_
813
 
814
        # @relation(SDOC-SRS-109, scope=range_start)
815
        unique_document_from_file_occurences: Set[str] = set()
816
        for document_ in document_tree.document_list:
817
            document_from_file_: SDocDocumentFromFileIF
818
            for document_from_file_ in document_.fragments_from_files:
819
                traceability_index.contains_included_documents = True
820
 
821
                assert isinstance(document_from_file_, DocumentFromFile), (
822
                    document_from_file_
823
                )
824
 
825
                assert (
826
                    document_from_file_.resolved_full_path_to_document_file
827
                    is not None
828
                )
829
 
830
                if (
831
                    document_from_file_.resolved_full_path_to_document_file
832
                    not in map_documents_by_input_rel_path
833
                ):
834
                    raise StrictDocException(
835
                        "A document includes contains a link to another document "
836
                        "which is not resolved in the current documentation tree: "
837
                        f"'{document_from_file_.file}'. This can happen if a single "
838
                        f"document path is provided as input to a StrictDoc command. "
839
                        f"Try providing a path to a folder where all documents "
840
                        f"are stored."
841
                    )
842
                resolved_document: SDocDocument = (
843
                    map_documents_by_input_rel_path[
844
                        document_from_file_.resolved_full_path_to_document_file
845
                    ]
846
                )
847
 
848
                if (
849
                    document_from_file_.resolved_full_path_to_document_file
850
                    in unique_document_from_file_occurences
851
                ) and resolved_document.has_any_requirements():
852
                    raise StrictDocException(
853
                        "[DOCUMENT_FROM_FILE]: "
854
                        "A multiple inclusion of a document is detected. "
855
                        "A document that contains requirements or other nodes "
856
                        "can be only included once: "
857
                        f"{document_from_file_.file}."
858
                    )
859
                unique_document_from_file_occurences.add(
860
                    document_from_file_.resolved_full_path_to_document_file
861
                )
862
 
863
                document_from_file_.configure_with_resolved_document(
864
                    resolved_document
865
                )
866
 
867
        # @relation(SDOC-SRS-109, scope=range_end)
868
 
869
        return traceability_index
870
 
871
    @classmethod
872
    def _create_filter(
873
        cls, traceability_index: Any, filter_query: str
874
    ) -> "NodeFilter":
875
        query_reader = QueryReader()
876
        requirements_query_object: Union[QueryObject, QueryNullObject]
877
        try:
878
            requirements_query = query_reader.read(filter_query)
879
            requirements_query_object = QueryObject(
880
                requirements_query, traceability_index
881
            )
882
        except TextXSyntaxError as textx_syntax_error_:
883
            raise StrictDocException(
884
                "Cannot parse filter query."
885
            ) from textx_syntax_error_
886
 
887
        blacklisted_nodes: set[SDocElementIF] = set()
888
 
889
        try:
890
            for document in traceability_index.document_tree.document_list:
891
                document_iterator = traceability_index.get_document_iterator(
892
                    document
893
                )
894
                for node, _ in document_iterator.all_content():
895
                    if (
896
                        isinstance(node, SDocNode)
897
                        and node.node_type == "SECTION"
898
                        and not requirements_query_object.evaluate(node)
899
                    ):
900
                        blacklisted_nodes.add(node)
901
 
902
                        # If the node is the last one, we check if all other
903
                        # nodes are filtered out and if so, mark the parent
904
                        # section node as not whitelisted as well.
905
                        if (
906
                            node.parent.section_contents[
907
                                len(node.parent.section_contents) - 1
908
                            ]
909
                            == node
910
                        ):
911
                            if (
912
                                isinstance(node.parent, SDocNode)
913
                                and node.parent.node_type == "SECTION"
914
                            ):
915
                                cls._blacklist_if_needed(
916
                                    blacklisted_nodes, node.parent
917
                                )
918
 
919
                    elif isinstance(
920
                        node, SDocNode
921
                    ) and not requirements_query_object.evaluate(node):
922
                        blacklisted_nodes.add(node)
923
                        # If the node is the last one, we check if all other
924
                        # nodes are filtered out and if so, mark the parent
925
                        # section node as not whitelisted as well.
926
                        if (
927
                            node.parent.section_contents[
928
                                len(node.parent.section_contents) - 1
929
                            ]
930
                            == node
931
                        ):
932
                            cls._blacklist_if_needed(
933
                                blacklisted_nodes, node.parent
934
                            )
935
 
936
        except (AttributeError, NameError, TypeError) as attribute_error_:
937
            raise StrictDocException(
938
                f"Cannot apply a filter query to a node: {attribute_error_}"
939
            ) from attribute_error_
940
 
941
        return NodeFilter(blacklisted_nodes)
942
 
943
    @classmethod
944
    def _blacklist_if_needed(
945
        cls,
946
        blacklisted_nodes: set[SDocElementIF],
947
        node: SDocElementIF,
948
    ) -> None:
949
        if isinstance(node, SDocDocumentFromFileIF):
950
            return
951
 
952
        if node.section_contents is not None:
953
            for node_ in node.section_contents:
954
                if node_ not in blacklisted_nodes:
955
                    return
956
 
957
        blacklisted_nodes.add(node)
958
 
959
        # If it turns out that all child nodes are blacklisted,
960
        # go up and blacklist the parent node if needed.
961
        if (
962
            isinstance(node, SDocNodeIF)
963
            and node.parent not in blacklisted_nodes
964
        ):
965
            cls._blacklist_if_needed(blacklisted_nodes, node.parent)
966
 
967
    @staticmethod
968
    def source_node_parser_tags(
969
        cfg_entry: SourceNodesEntry, grammar_element: GrammarElement
970
    ) -> set[str]:
971
        tags = set(grammar_element.get_field_titles())
972
        # For remapped fields, don't parse the names from grammar but those from the mapping.
973
        for (
974
            sdoc_field_name,
975
            source_field_name,
976
        ) in cfg_entry.sdoc_to_source_map.items():
977
            tags.remove(sdoc_field_name)
978
            tags.add(source_field_name)
979
        return tags