StrictDoc Documentation
strictdoc/backend/sdoc_source_code/reader_c.py
Source file coverage
Path:
strictdoc/backend/sdoc_source_code/reader_c.py
Lines:
530
Non-empty lines:
474
Non-empty lines covered with requirements:
474 / 474 (100.0%)
Functions:
7
Functions covered by requirements:
7 / 7 (100.0%)
1
"""
2
@relation(SDOC-SRS-142, SDOC-SRS-146, scope=file)
3
"""
4
 
5
from typing import Final, List, Optional, Sequence
6
 
7
import tree_sitter_c
8
import tree_sitter_cpp
9
from tree_sitter import Language, Node, Parser
10
 
11
from strictdoc.backend.sdoc_source_code.constants import FunctionAttribute
12
from strictdoc.backend.sdoc_source_code.marker_parser import MarkerParser
13
from strictdoc.backend.sdoc_source_code.models.language import LanguageItem
14
from strictdoc.backend.sdoc_source_code.models.language_item_marker import (
15
    LanguageItemMarker,
16
    RangeMarkerType,
17
)
18
from strictdoc.backend.sdoc_source_code.models.line_marker import LineMarker
19
from strictdoc.backend.sdoc_source_code.models.range_marker import (
20
    RangeMarker,
21
)
22
from strictdoc.backend.sdoc_source_code.models.source_file_info import (
23
    SourceFileTraceabilityInfo,
24
)
25
from strictdoc.backend.sdoc_source_code.models.source_location import ByteRange
26
from strictdoc.backend.sdoc_source_code.models.source_node import SourceNode
27
from strictdoc.backend.sdoc_source_code.parse_context import ParseContext
28
from strictdoc.backend.sdoc_source_code.processors.general_language_marker_processors import (
29
    language_item_marker_processor,
30
    line_marker_processor,
31
    range_marker_processor,
32
    source_file_traceability_info_processor,
33
)
34
from strictdoc.backend.sdoc_source_code.tree_sitter_helpers import (
35
    traverse_tree,
36
    ts_find_child_node_by_type,
37
    ts_find_child_nodes_by_type,
38
)
39
from strictdoc.helpers.cast import assert_cast
40
from strictdoc.helpers.file_stats import SourceFileStats
41
from strictdoc.helpers.file_system import file_open_read_bytes
42
 
43
KNOWN_FUNCTION_DEFINITION_MACROS: Final[frozenset[str]] = frozenset(
44
    (
45
        # Linux
46
        "COMPAT_SYSCALL_DEFINE0",
47
        "COMPAT_SYSCALL_DEFINE1",
48
        "COMPAT_SYSCALL_DEFINE2",
49
        "COMPAT_SYSCALL_DEFINE3",
50
        "COMPAT_SYSCALL_DEFINE4",
51
        "COMPAT_SYSCALL_DEFINE5",
52
        "COMPAT_SYSCALL_DEFINE6",
53
        "FIXTURE_SETUP",
54
        "FIXTURE_TEARDOWN",
55
        "SYSCALL_DEFINE0",
56
        "SYSCALL_DEFINE1",
57
        "SYSCALL_DEFINE2",
58
        "SYSCALL_DEFINE3",
59
        "SYSCALL_DEFINE4",
60
        "SYSCALL_DEFINE5",
61
        "SYSCALL_DEFINE6",
62
        # Google Benchmark
63
        "BENCHMARK",
64
        "BENCHMARK_F",
65
        "BENCHMARK_DEFINE_F",
66
        "BENCHMARK_TEMPLATE",
67
        "BENCHMARK_TEMPLATE_F",
68
        "BENCHMARK_TEMPLATE_DEFINE_F",
69
        # Google Test and Linux
70
        "TEST",
71
        "TEST_F",
72
        "TEST_P",
73
        "TEST_F_SIGNAL",
74
        "TYPED_TEST",
75
        # Zephyr
76
        "ZTEST_USER",
77
    )
78
)
79
 
80
 
81
class SourceFileTraceabilityReader_C:
82
    @staticmethod
83
    def supported_elements() -> list[str]:
84
        return ["function", "class"]
85
 
86
    def __init__(self, custom_tags: Optional[set[str]] = None) -> None:
87
        self.custom_tags: Optional[set[str]] = custom_tags
88
 
89
    def read(
90
        self,
91
        input_buffer: bytes,
92
        file_path: Optional[str] = None,
93
    ) -> SourceFileTraceabilityInfo:
94
        assert isinstance(input_buffer, bytes)
95
 
96
        file_stats = SourceFileStats.create(input_buffer)
97
        parse_context = ParseContext(file_path, file_stats)
98
 
99
        language_arg: object
100
        if file_path is not None and file_path.endswith(".c"):
101
            language_arg = tree_sitter_c.language()
102
        else:
103
            language_arg = tree_sitter_cpp.language()
104
        py_language = Language(language_arg)
105
        parser = Parser(py_language)
106
 
107
        tree = parser.parse(input_buffer)
108
 
109
        traceability_info = SourceFileTraceabilityInfo([])
110
 
111
        nodes = traverse_tree(tree)
112
 
113
        source_node: Optional[SourceNode]
114
        for node_ in nodes:
115
            function_name: str
116
            language_item_markers: List[LanguageItemMarker]
117
            function_comment_node: Optional[Node]
118
            if node_.type == "translation_unit":
119
                if (
120
                    len(node_.children) > 0
121
                    and node_.children[0].type == "comment"
122
                    and (comment_node := node_.children[0])
123
                ):
124
                    if comment_node.text is not None:
125
                        comment_text = comment_node.text.decode("utf-8")
126
                        source_node = MarkerParser.parse(
127
                            input_string=comment_text,
128
                            line_start=node_.start_point[0] + 1,
129
                            # It is important that +1 is not present here because
130
                            # currently StrictDoc does not display the last empty line (\n is 10).
131
                            line_end=node_.end_point[0]
132
                            if input_buffer[-1] == 10
133
                            else node_.end_point[0] + 1,
134
                            comment_line_start=node_.start_point[0] + 1,
135
                            comment_byte_range=ByteRange.create_from_ts_node(
136
                                comment_node
137
                            ),
138
                            filename=parse_context.filename,
139
                            custom_tags=self.custom_tags,
140
                        )
141
                        for marker_ in source_node.markers:
142
                            if not isinstance(marker_, LanguageItemMarker):
143
                                continue
144
                            # At the top level, only accept the scope=file markers.
145
                            # Everything else will be handled by functions and classes.
146
                            if marker_.scope != RangeMarkerType.FILE:
147
                                continue
148
                            if isinstance(marker_, LanguageItemMarker) and (
149
                                language_item_marker_ := marker_
150
                            ):
151
                                language_item_marker_processor(
152
                                    language_item_marker_, parse_context
153
                                )
154
                                traceability_info.markers.append(
155
                                    language_item_marker_
156
                                )
157
 
158
            elif node_.type in ("declaration", "field_declaration"):
159
                function_declarator_node = ts_find_child_node_by_type(
160
                    node_, "function_declarator"
161
                )
162
 
163
                # C++ reference declaration wrap the function declaration one time.
164
                if function_declarator_node is None:
165
                    # Example: "TrkVertex& operator-=(const TrkVertex& c);".
166
                    reference_declarator_node = ts_find_child_node_by_type(
167
                        node_, "reference_declarator"
168
                    )
169
                    if reference_declarator_node is None:
170
                        continue
171
 
172
                    function_declarator_node = ts_find_child_node_by_type(
173
                        reference_declarator_node, "function_declarator"
174
                    )
175
                    if function_declarator_node is None:
176
                        continue
177
 
178
                # For normal C functions the identifier is "identifier".
179
                # For C++, there are:
180
                # Class function declarations: bool CanSend(const CanFrame &frame);         # noqa: ERA001
181
                # Operators:                   TrkVertex& operator-=(const TrkVertex& c);   # noqa: ERA001
182
                # Destructors:                 ~TrkVertex();                                # noqa: ERA001
183
                function_identifier_node = self._get_function_name_node(
184
                    function_declarator_node,
185
                )
186
                if function_identifier_node is None:
187
                    continue
188
 
189
                if function_identifier_node.text is None:
190
                    continue
191
 
192
                assert function_identifier_node.text is not None, node_.text
193
                function_display_name = function_identifier_node.text.decode(
194
                    "utf8"
195
                )
196
 
197
                assert function_declarator_node.text is not None, node_.text
198
                function_name = function_declarator_node.text.decode("utf8")
199
                assert function_name is not None, node_.text
200
                # Remove extra trailing spaces, newlines etc added by code-formatting or linters
201
                function_name = " ".join(function_name.split())
202
 
203
                parent_names = self.get_node_ns(node_)
204
                if len(parent_names) > 0:
205
                    function_name = (
206
                        f"{'::'.join(parent_names)}::{function_name}"
207
                    )
208
                    function_display_name = (
209
                        f"{'::'.join(parent_names)}::{function_display_name}"
210
                    )
211
 
212
                function_attributes = {FunctionAttribute.DECLARATION}
213
                for specifier_node_ in ts_find_child_nodes_by_type(
214
                    node_, "storage_class_specifier"
215
                ):
216
                    if specifier_node_.text == b"static":
217
                        function_attributes.add(FunctionAttribute.STATIC)
218
 
219
                source_node = None
220
                language_item_markers = []
221
                function_comment_node = None
222
                if (
223
                    node_.prev_sibling is not None
224
                    and node_.prev_sibling.type == "comment"
225
                ):
226
                    function_comment_node = node_.prev_sibling
227
                    assert function_comment_node.text is not None, node_.text
228
                    function_comment_text = function_comment_node.text.decode(
229
                        "utf8"
230
                    )
231
 
232
                    function_last_line = node_.end_point[0] + 1
233
 
234
                    source_node = MarkerParser.parse(
235
                        input_string=function_comment_text,
236
                        line_start=function_comment_node.start_point[0] + 1,
237
                        line_end=function_last_line,
238
                        comment_line_start=function_comment_node.start_point[0]
239
                        + 1,
240
                        comment_byte_range=ByteRange.create_from_ts_node(
241
                            function_comment_node
242
                        ),
243
                        filename=parse_context.filename,
244
                        entity_name=function_display_name,
245
                        custom_tags=self.custom_tags,
246
                    )
247
                    for marker_ in source_node.markers:
248
                        if isinstance(marker_, LanguageItemMarker) and (
249
                            language_item_marker_ := marker_
250
                        ):
251
                            language_item_marker_processor(
252
                                language_item_marker_, parse_context
253
                            )
254
                            traceability_info.markers.append(
255
                                language_item_marker_
256
                            )
257
                            language_item_markers.append(marker_)
258
 
259
                # The function range includes the top comment if it exists.
260
                new_function = LanguageItem(
261
                    parent=traceability_info,
262
                    name=function_name,
263
                    display_name=function_display_name,
264
                    line_begin=function_comment_node.start_point[0] + 1
265
                    if function_comment_node is not None
266
                    else node_.range.start_point[0] + 1,
267
                    line_end=node_.range.end_point[0] + 1,
268
                    code_byte_range=ByteRange.create_from_ts_node(node_),
269
                    child_functions=[],
270
                    markers=language_item_markers,
271
                    attributes=function_attributes,
272
                )
273
                if source_node is not None:
274
                    source_node.function = new_function
275
                traceability_info.functions.append(new_function)
276
 
277
            elif node_.type == "function_definition":
278
                function_name = ""
279
 
280
                try:
281
                    function_declarator_node = ts_find_child_node_by_type(
282
                        node_, "function_declarator", raise_on_error=True
283
                    )
284
                except LookupError:
285
                    # Probably confused by macro, skip node to avoid processing random subtrees.
286
                    continue
287
                # C++ reference declaration wrap the function declaration one time.
288
                if function_declarator_node is None:
289
                    # Example: Foo& Foo::operator+(const Foo& c) { return *this; }
290
                    reference_declarator_node = ts_find_child_node_by_type(
291
                        node_, "reference_declarator"
292
                    )
293
                    if reference_declarator_node is None:
294
                        continue
295
 
296
                    function_declarator_node = ts_find_child_node_by_type(
297
                        reference_declarator_node, "function_declarator"
298
                    )
299
                    if function_declarator_node is None:
300
                        continue
301
 
302
                assert function_declarator_node is not None, node_.text
303
 
304
                assert function_declarator_node.text is not None, node_.text
305
                function_name = function_declarator_node.text.decode("utf8")
306
 
307
                identifier_node = self._get_function_name_node(
308
                    function_declarator_node
309
                )
310
                if identifier_node is None:
311
                    print(  # noqa: T201
312
                        "warning: C/C++ source reader: skipping unsupported "
313
                        "function declarator at "
314
                        f"{parse_context.filename}:"
315
                        f"{function_declarator_node.start_point[0] + 1}: "
316
                        f"{function_declarator_node}"
317
                    )
318
                    continue
319
 
320
                assert identifier_node.text is not None, node_.text
321
                function_display_name = identifier_node.text.decode("utf8")
322
 
323
                assert function_name is not None, node_.text
324
                # Remove extra trailing spaces, newlines etc added by code-formatting or linters
325
                function_name = " ".join(function_name.split())
326
                parent_names = self.get_node_ns(node_)
327
 
328
                # The first if branch handles a special case where selected
329
                # macros are actually function definitions. Typical examples of
330
                # such functions:
331
                # 1) Google Test TEST(...)
332
                # 2) Zephyr RTOS ZTEST_USER(...)
333
                # 3) Linux SYSCALL2_DEFINE etc.
334
                if function_display_name in KNOWN_FUNCTION_DEFINITION_MACROS:
335
                    # Make the display name to include the entire macro/function
336
                    # signature.
337
                    # Example of this case:
338
                    # function_name: ZTEST_USER(semaphore, test_k_sem_correct_count_limit)  # noqa: ERA001
339
                    # function_display_name (before assignment): ZTEST_USER
340
                    function_display_name = function_name
341
                elif len(parent_names) > 0:
342
                    function_name = (
343
                        f"{'::'.join(parent_names)}::{function_name}"
344
                    )
345
                    function_display_name = (
346
                        f"{'::'.join(parent_names)}::{function_display_name}"
347
                    )
348
 
349
                source_node = None
350
                language_item_markers = []
351
                function_comment_node = None
352
                function_comment_text = None
353
 
354
                # In the condition below, it is important that the comment is
355
                # considered a function comment only if it there are no empty
356
                # lines between the comment and function.
357
                if (
358
                    node_.prev_sibling is not None
359
                    and node_.prev_sibling.type == "comment"
360
                    and (node_.prev_sibling.end_point[0] + 1)
361
                    == node_.start_point[0]
362
                ):
363
                    function_comment_node = node_.prev_sibling
364
                    assert function_comment_node.text is not None, node_.text
365
                    function_comment_text = function_comment_node.text.decode(
366
                        "utf8"
367
                    )
368
 
369
                    function_last_line = node_.end_point[0] + 1
370
 
371
                    source_node = MarkerParser.parse(
372
                        input_string=function_comment_text,
373
                        line_start=function_comment_node.start_point[0] + 1,
374
                        line_end=function_last_line,
375
                        comment_line_start=function_comment_node.start_point[0]
376
                        + 1,
377
                        comment_byte_range=ByteRange.create_from_ts_node(
378
                            function_comment_node
379
                        ),
380
                        filename=parse_context.filename,
381
                        entity_name=function_display_name,
382
                        custom_tags=self.custom_tags,
383
                    )
384
 
385
                    traceability_info.source_nodes.append(source_node)
386
                    for marker_ in source_node.markers:
387
                        if isinstance(marker_, LanguageItemMarker):
388
                            language_item_marker_processor(
389
                                marker_, parse_context
390
                            )
391
                            traceability_info.markers.append(marker_)
392
                            language_item_markers.append(marker_)
393
 
394
                # The function range includes the top comment if it exists.
395
                new_function = LanguageItem(
396
                    parent=traceability_info,
397
                    name=function_name,
398
                    display_name=function_display_name,
399
                    line_begin=function_comment_node.start_point[0] + 1
400
                    if function_comment_node is not None
401
                    else node_.range.start_point[0] + 1,
402
                    line_end=node_.range.end_point[0] + 1,
403
                    code_byte_range=ByteRange.create_from_ts_node(node_),
404
                    child_functions=[],
405
                    markers=language_item_markers,
406
                    attributes={FunctionAttribute.DEFINITION},
407
                )
408
                traceability_info.functions.append(new_function)
409
                if len(language_item_markers) > 0:
410
                    traceability_info.ng_map_names_to_markers[function_name] = (
411
                        # FIXME: Cannot win the fight with mypy without assert_cast.
412
                        assert_cast(language_item_markers, list)
413
                    )
414
                    traceability_info.ng_map_names_to_definition_functions[
415
                        function_name
416
                    ] = new_function
417
                if source_node is not None:
418
                    source_node.function = new_function
419
            elif node_.type == "comment":
420
                #
421
                # FIXME: Here parsing of function comments can happen as well
422
                #        but this time the focus is ONLY on range and line markers.
423
                #        The case which is handled here is when a user adds a
424
                #        range_start marker in a function comment.
425
                #        It is not good that parsing of function comments
426
                #        happens twice.
427
                #
428
 
429
                assert node_.text is not None, (
430
                    f"Comment without a text: {node_}"
431
                )
432
 
433
                node_text_string = node_.text.decode("utf8")
434
 
435
                source_node = MarkerParser.parse(
436
                    input_string=node_text_string,
437
                    line_start=node_.start_point[0] + 1,
438
                    line_end=node_.end_point[0] + 1,
439
                    comment_line_start=node_.start_point[0] + 1,
440
                    comment_byte_range=ByteRange.create_from_ts_node(node_),
441
                    filename=parse_context.filename,
442
                    custom_tags=None,
443
                )
444
 
445
                for marker_ in source_node.markers:
446
                    if isinstance(marker_, RangeMarker) and (
447
                        range_marker_ := marker_
448
                    ):
449
                        range_marker_processor(range_marker_, parse_context)
450
                    elif isinstance(marker_, LineMarker) and (
451
                        line_marker_ := marker_
452
                    ):
453
                        line_marker_processor(line_marker_, parse_context)
454
                    else:
455
                        pass
456
            else:
457
                pass
458
 
459
        source_file_traceability_info_processor(
460
            traceability_info, parse_context
461
        )
462
 
463
        traceability_info.ng_map_reqs_to_markers = (
464
            parse_context.map_reqs_to_markers
465
        )
466
 
467
        return traceability_info
468
 
469
    def read_from_file(self, file_path: str) -> SourceFileTraceabilityInfo:
470
        with file_open_read_bytes(file_path) as file:
471
            sdoc_content = file.read()
472
            sdoc = self.read(sdoc_content, file_path=file_path)
473
            return sdoc
474
 
475
    @staticmethod
476
    def _get_function_name_node(
477
        function_declarator_node: Node,
478
    ) -> Optional[Node]:
479
        assert function_declarator_node.type == "function_declarator"
480
        function_identifier_node = ts_find_child_node_by_type(
481
            function_declarator_node,
482
            node_type=(
483
                "identifier",
484
                "field_identifier",
485
                "operator_name",
486
                "destructor_name",
487
                "qualified_identifier",
488
            ),
489
        )
490
        if function_identifier_node is not None:
491
            return function_identifier_node
492
 
493
        nested_function_declarator_node = ts_find_child_node_by_type(
494
            function_declarator_node,
495
            "function_declarator",
496
        )
497
        if nested_function_declarator_node is not None:
498
            return SourceFileTraceabilityReader_C._get_function_name_node(
499
                nested_function_declarator_node
500
            )
501
 
502
        return None
503
 
504
    @staticmethod
505
    def get_node_ns(node: Node) -> Sequence[str]:
506
        """
507
        Walk up the tree and find parent classes.
508
        """
509
        parent_scopes = []
510
        cursor: Optional[Node] = node
511
        while cursor is not None:
512
            if cursor.type == "class_specifier" and len(cursor.children) > 1:
513
                second_node_or_none = cursor.children[1]
514
                if (
515
                    second_node_or_none.type == "type_identifier"
516
                    and second_node_or_none.text is not None
517
                ):
518
                    parent_class_name = second_node_or_none.text.decode("utf8")
519
                    parent_scopes.append(parent_class_name)
520
            elif cursor.type == "namespace_definition":
521
                for c in cursor.children:
522
                    if c.type == "namespace_identifier" and c.text is not None:
523
                        parent_class_name = c.text.decode("utf8")
524
                        parent_scopes.append(parent_class_name)
525
                        break
526
 
527
            cursor = cursor.parent
528
 
529
        parent_scopes.reverse()
530
        return parent_scopes
1 - 530 | entire file