Path:
strictdoc/backend/sdoc_source_code/reader_c.py
Lines:
539
Non-empty lines:
483
Non-empty lines covered with requirements:
483 / 483 (100.0%)
Functions:
8
Functions covered by requirements:
8 / 8 (100.0%)
- "7.2.1. Language-aware parsing of source code" (REQUIREMENT)
- "7.2.2. Language-aware parsing of C/C++ code" (REQUIREMENT)
1
"""2
@relation(SDOC-SRS-142, SDOC-SRS-146, scope=file)3
"""4
5
from typing import Final, List, Optional, Sequence
6
7
import tree_sitter_c
8
import tree_sitter_cpp
9
from tree_sitter import Language, Node, Parser
10
11
from strictdoc.backend.sdoc_source_code.constants import FunctionAttribute
12
from strictdoc.backend.sdoc_source_code.marker_parser import MarkerParser
13
from strictdoc.backend.sdoc_source_code.models.language import LanguageItem
14
from strictdoc.backend.sdoc_source_code.models.language_item_marker import (
15
LanguageItemMarker,
16
RangeMarkerType,
17
)18
from strictdoc.backend.sdoc_source_code.models.line_marker import LineMarker
19
from strictdoc.backend.sdoc_source_code.models.range_marker import (
20
RangeMarker,
21
)22
from strictdoc.backend.sdoc_source_code.models.source_file_info import (
23
SourceFileTraceabilityInfo,
24
)25
from strictdoc.backend.sdoc_source_code.models.source_location import ByteRange
26
from strictdoc.backend.sdoc_source_code.models.source_node import SourceNode
27
from strictdoc.backend.sdoc_source_code.parse_context import ParseContext
28
from strictdoc.backend.sdoc_source_code.processors.general_language_marker_processors import (
29
language_item_marker_processor,
30
line_marker_processor,
31
range_marker_processor,
32
source_file_traceability_info_processor,
33
)34
from strictdoc.backend.sdoc_source_code.tree_sitter_helpers import (
35
traverse_tree,
36
ts_find_child_node_by_type,
37
ts_find_child_nodes_by_type,
38
)39
from strictdoc.helpers.cast import assert_cast
40
from strictdoc.helpers.file_stats import SourceFileStats
41
from strictdoc.helpers.file_system import file_open_read_bytes
42
43
KNOWN_FUNCTION_DEFINITION_MACROS: Final[frozenset[str]] = frozenset(
44
(45
# Linux46
"COMPAT_SYSCALL_DEFINE0",
47
"COMPAT_SYSCALL_DEFINE1",
48
"COMPAT_SYSCALL_DEFINE2",
49
"COMPAT_SYSCALL_DEFINE3",
50
"COMPAT_SYSCALL_DEFINE4",
51
"COMPAT_SYSCALL_DEFINE5",
52
"COMPAT_SYSCALL_DEFINE6",
53
"FIXTURE_SETUP",
54
"FIXTURE_TEARDOWN",
55
"SYSCALL_DEFINE0",
56
"SYSCALL_DEFINE1",
57
"SYSCALL_DEFINE2",
58
"SYSCALL_DEFINE3",
59
"SYSCALL_DEFINE4",
60
"SYSCALL_DEFINE5",
61
"SYSCALL_DEFINE6",
62
# Google Benchmark63
"BENCHMARK",
64
"BENCHMARK_F",
65
"BENCHMARK_DEFINE_F",
66
"BENCHMARK_TEMPLATE",
67
"BENCHMARK_TEMPLATE_F",
68
"BENCHMARK_TEMPLATE_DEFINE_F",
69
# Google Test and Linux70
"TEST",
71
"TEST_F",
72
"TEST_P",
73
"TEST_F_SIGNAL",
74
"TYPED_TEST",
75
# Zephyr76
"ZTEST_USER",
77
)78
)79
80
81
class SourceFileTraceabilityReader_C:
82
@staticmethod83
def supported_elements() -> list[str]:
84
return ["function", "class"]
85
86
def __init__(self, custom_tags: Optional[set[str]] = None) -> None:
87
self.custom_tags: Optional[set[str]] = custom_tags
88
89
def read(
90
self,
91
input_buffer: bytes,
92
file_path: Optional[str] = None,
93
) -> SourceFileTraceabilityInfo:
94
assert isinstance(input_buffer, bytes)
95
96
file_stats = SourceFileStats.create(input_buffer)
97
parse_context = ParseContext(file_path, file_stats)
98
99
language_arg: object
100
if file_path is not None and file_path.endswith(".c"):
101
language_arg = tree_sitter_c.language()
102
else:
103
language_arg = tree_sitter_cpp.language()
104
py_language = Language(language_arg)
105
parser = Parser(py_language)
106
107
tree = parser.parse(input_buffer)
108
109
traceability_info = SourceFileTraceabilityInfo([])
110
111
nodes = traverse_tree(tree)
112
113
source_node: Optional[SourceNode]
114
for node_ in nodes:
115
function_name: str
116
language_item_markers: List[LanguageItemMarker]
117
function_comment_node: Optional[Node]
118
if node_.type == "translation_unit":
119
if (
120
len(node_.children) > 0
121
and node_.children[0].type == "comment"
122
and (comment_node := node_.children[0])
123
):124
if comment_node.text is not None:
125
comment_text = comment_node.text.decode("utf-8")
126
source_node = MarkerParser.parse(
127
input_string=comment_text,
128
line_start=node_.start_point[0] + 1,
129
# It is important that +1 is not present here because130
# currently StrictDoc does not display the last empty line (\n is 10).131
line_end=node_.end_point[0]
132
if input_buffer[-1] == 10
133
else node_.end_point[0] + 1,
134
comment_line_start=node_.start_point[0] + 1,
135
comment_byte_range=ByteRange.create_from_ts_node(
136
comment_node137
),138
filename=parse_context.filename,
139
custom_tags=self.custom_tags,
140
)141
for marker_ in source_node.markers:
142
if not isinstance(marker_, LanguageItemMarker):
143
continue144
# At the top level, only accept the scope=file markers.145
# Everything else will be handled by functions and classes.146
if marker_.scope != RangeMarkerType.FILE:
147
continue148
if isinstance(marker_, LanguageItemMarker) and (
149
language_item_marker_ := marker_
150
):151
language_item_marker_processor(
152
language_item_marker_, parse_context
153
)154
traceability_info.markers.append(
155
language_item_marker_156
)157
158
elif node_.type in ("declaration", "field_declaration"):
159
# Pointer return types ("SomeType *foo();") and C++160
# reference declarations ("TrkVertex& operator-=(...);")161
# wrap the function declarator one or more times.162
function_declarator_node = self._find_function_declarator_node(
163
node_164
)165
if function_declarator_node is None:
166
continue167
168
# For normal C functions the identifier is "identifier".169
# For C++, there are:170
# Class function declarations: bool CanSend(const CanFrame &frame); # noqa: ERA001171
# Operators: TrkVertex& operator-=(const TrkVertex& c); # noqa: ERA001172
# Destructors: ~TrkVertex(); # noqa: ERA001173
function_identifier_node = self._get_function_name_node(
174
function_declarator_node,
175
)176
if function_identifier_node is None:
177
continue178
179
if function_identifier_node.text is None:
180
continue181
182
assert function_identifier_node.text is not None, node_.text
183
function_display_name = function_identifier_node.text.decode(
184
"utf8"185
)186
187
assert function_declarator_node.text is not None, node_.text
188
function_name = function_declarator_node.text.decode("utf8")
189
assert function_name is not None, node_.text
190
# Remove extra trailing spaces, newlines etc added by code-formatting or linters191
function_name = " ".join(function_name.split())
192
193
parent_names = self.get_node_ns(node_)
194
if len(parent_names) > 0:
195
function_name = (
196
f"{'::'.join(parent_names)}::{function_name}"
197
)198
function_display_name = (
199
f"{'::'.join(parent_names)}::{function_display_name}"
200
)201
202
function_attributes = {FunctionAttribute.DECLARATION}
203
for specifier_node_ in ts_find_child_nodes_by_type(
204
node_, "storage_class_specifier"
205
):206
if specifier_node_.text == b"static":
207
function_attributes.add(FunctionAttribute.STATIC)
208
209
source_node = None
210
language_item_markers = []
211
function_comment_node = None
212
if (
213
node_.prev_sibling is not None
214
and node_.prev_sibling.type == "comment"
215
):216
function_comment_node = node_.prev_sibling
217
assert function_comment_node.text is not None, node_.text
218
function_comment_text = function_comment_node.text.decode(
219
"utf8"220
)221
222
function_last_line = node_.end_point[0] + 1
223
224
source_node = MarkerParser.parse(
225
input_string=function_comment_text,
226
line_start=function_comment_node.start_point[0] + 1,
227
line_end=function_last_line,
228
comment_line_start=function_comment_node.start_point[0]
229
+ 1,
230
comment_byte_range=ByteRange.create_from_ts_node(
231
function_comment_node232
),233
filename=parse_context.filename,
234
entity_name=function_display_name,
235
custom_tags=self.custom_tags,
236
)237
for marker_ in source_node.markers:
238
if isinstance(marker_, LanguageItemMarker) and (
239
language_item_marker_ := marker_
240
):241
language_item_marker_processor(
242
language_item_marker_, parse_context
243
)244
traceability_info.markers.append(
245
language_item_marker_246
)247
language_item_markers.append(marker_)
248
249
# The function range includes the top comment if it exists.250
new_function = LanguageItem(
251
parent=traceability_info,
252
name=function_name,
253
display_name=function_display_name,
254
line_begin=function_comment_node.start_point[0] + 1
255
if function_comment_node is not None
256
else node_.range.start_point[0] + 1,
257
line_end=node_.range.end_point[0] + 1,
258
code_byte_range=ByteRange.create_from_ts_node(node_),
259
child_functions=[],
260
markers=language_item_markers,
261
attributes=function_attributes,
262
)263
if source_node is not None:
264
source_node.function = new_function
265
traceability_info.functions.append(new_function)
266
267
elif node_.type == "function_definition":
268
function_name = ""
269
270
# Pointer return types ("SomeType *foo() { ... }") and C++271
# reference declarations ("Foo& Foo::operator+(...) {...}")272
# wrap the function declarator one or more times.273
try:
274
function_declarator_node = (
275
self._find_function_declarator_node(
276
node_, raise_on_error=True
277
)278
)279
except LookupError:
280
# Probably confused by macro, skip node to avoid processing random subtrees.281
continue282
if function_declarator_node is None:
283
continue284
285
assert function_declarator_node is not None, node_.text
286
287
assert function_declarator_node.text is not None, node_.text
288
function_name = function_declarator_node.text.decode("utf8")
289
290
identifier_node = self._get_function_name_node(
291
function_declarator_node292
)293
if identifier_node is None:
294
print( # noqa: T201
295
"warning: C/C++ source reader: skipping unsupported "296
"function declarator at "297
f"{parse_context.filename}:"
298
f"{function_declarator_node.start_point[0] + 1}: "
299
f"{function_declarator_node}"
300
)301
continue302
303
assert identifier_node.text is not None, node_.text
304
function_display_name = identifier_node.text.decode("utf8")
305
306
assert function_name is not None, node_.text
307
# Remove extra trailing spaces, newlines etc added by code-formatting or linters308
function_name = " ".join(function_name.split())
309
parent_names = self.get_node_ns(node_)
310
311
# The first if branch handles a special case where selected312
# macros are actually function definitions. Typical examples of313
# such functions:314
# 1) Google Test TEST(...)315
# 2) Zephyr RTOS ZTEST_USER(...)316
# 3) Linux SYSCALL2_DEFINE etc.317
if function_display_name in KNOWN_FUNCTION_DEFINITION_MACROS:
318
# Make the display name to include the entire macro/function319
# signature.320
# Example of this case:321
# function_name: ZTEST_USER(semaphore, test_k_sem_correct_count_limit) # noqa: ERA001322
# function_display_name (before assignment): ZTEST_USER323
function_display_name = function_name
324
elif len(parent_names) > 0:
325
function_name = (
326
f"{'::'.join(parent_names)}::{function_name}"
327
)328
function_display_name = (
329
f"{'::'.join(parent_names)}::{function_display_name}"
330
)331
332
source_node = None
333
language_item_markers = []
334
function_comment_node = None
335
function_comment_text = None
336
337
# In the condition below, it is important that the comment is338
# considered a function comment only if it there are no empty339
# lines between the comment and function.340
if (
341
node_.prev_sibling is not None
342
and node_.prev_sibling.type == "comment"
343
and (node_.prev_sibling.end_point[0] + 1)
344
== node_.start_point[0]
345
):346
function_comment_node = node_.prev_sibling
347
assert function_comment_node.text is not None, node_.text
348
function_comment_text = function_comment_node.text.decode(
349
"utf8"350
)351
352
function_last_line = node_.end_point[0] + 1
353
354
source_node = MarkerParser.parse(
355
input_string=function_comment_text,
356
line_start=function_comment_node.start_point[0] + 1,
357
line_end=function_last_line,
358
comment_line_start=function_comment_node.start_point[0]
359
+ 1,
360
comment_byte_range=ByteRange.create_from_ts_node(
361
function_comment_node362
),363
filename=parse_context.filename,
364
entity_name=function_display_name,
365
custom_tags=self.custom_tags,
366
)367
368
traceability_info.source_nodes.append(source_node)
369
for marker_ in source_node.markers:
370
if isinstance(marker_, LanguageItemMarker):
371
language_item_marker_processor(
372
marker_, parse_context
373
)374
traceability_info.markers.append(marker_)
375
language_item_markers.append(marker_)
376
377
# The function range includes the top comment if it exists.378
new_function = LanguageItem(
379
parent=traceability_info,
380
name=function_name,
381
display_name=function_display_name,
382
line_begin=function_comment_node.start_point[0] + 1
383
if function_comment_node is not None
384
else node_.range.start_point[0] + 1,
385
line_end=node_.range.end_point[0] + 1,
386
code_byte_range=ByteRange.create_from_ts_node(node_),
387
child_functions=[],
388
markers=language_item_markers,
389
attributes={FunctionAttribute.DEFINITION},
390
)391
traceability_info.functions.append(new_function)
392
if len(language_item_markers) > 0:
393
traceability_info.ng_map_names_to_markers[function_name] = (
394
# FIXME: Cannot win the fight with mypy without assert_cast.395
assert_cast(language_item_markers, list)
396
)397
traceability_info.ng_map_names_to_definition_functions[
398
function_name399
] = new_function
400
if source_node is not None:
401
source_node.function = new_function
402
elif node_.type == "comment":
403
#404
# FIXME: Here parsing of function comments can happen as well405
# but this time the focus is ONLY on range and line markers.406
# The case which is handled here is when a user adds a407
# range_start marker in a function comment.408
# It is not good that parsing of function comments409
# happens twice.410
#411
412
assert node_.text is not None, (
413
f"Comment without a text: {node_}"
414
)415
416
node_text_string = node_.text.decode("utf8")
417
418
source_node = MarkerParser.parse(
419
input_string=node_text_string,
420
line_start=node_.start_point[0] + 1,
421
line_end=node_.end_point[0] + 1,
422
comment_line_start=node_.start_point[0] + 1,
423
comment_byte_range=ByteRange.create_from_ts_node(node_),
424
filename=parse_context.filename,
425
custom_tags=None,
426
)427
428
for marker_ in source_node.markers:
429
if isinstance(marker_, RangeMarker) and (
430
range_marker_ := marker_
431
):432
range_marker_processor(range_marker_, parse_context)
433
elif isinstance(marker_, LineMarker) and (
434
line_marker_ := marker_
435
):436
line_marker_processor(line_marker_, parse_context)
437
else:
438
pass439
else:
440
pass441
442
source_file_traceability_info_processor(
443
traceability_info, parse_context
444
)445
446
traceability_info.ng_map_reqs_to_markers = (
447
parse_context.map_reqs_to_markers
448
)449
450
return traceability_info
451
452
def read_from_file(self, file_path: str) -> SourceFileTraceabilityInfo:
453
with file_open_read_bytes(file_path) as file:
454
sdoc_content = file.read()
455
sdoc = self.read(sdoc_content, file_path=file_path)
456
return sdoc
457
458
@staticmethod459
def _find_function_declarator_node(
460
node: Node, raise_on_error: bool = False
461
) -> Optional[Node]:
462
"""
463
Find a "function_declarator" among the node's children, descending464
through "pointer_declarator" (pointer return types, e.g. "SomeType465
*foo();") and "reference_declarator" (C++ reference return types,466
e.g. "TrkVertex& foo();") wrappers, which may be nested (e.g.467
"SomeType **foo();").468
"""469
function_declarator_node = ts_find_child_node_by_type(
470
node, "function_declarator", raise_on_error=raise_on_error
471
)472
if function_declarator_node is not None:
473
return function_declarator_node
474
475
for wrapper_type_ in ("pointer_declarator", "reference_declarator"):
476
wrapper_node = ts_find_child_node_by_type(node, wrapper_type_)
477
if wrapper_node is not None:
478
return SourceFileTraceabilityReader_C._find_function_declarator_node(
479
wrapper_node, raise_on_error=raise_on_error
480
)481
482
return None
483
484
@staticmethod485
def _get_function_name_node(
486
function_declarator_node: Node,
487
) -> Optional[Node]:
488
assert function_declarator_node.type == "function_declarator"
489
function_identifier_node = ts_find_child_node_by_type(
490
function_declarator_node,
491
node_type=(
492
"identifier",
493
"field_identifier",
494
"operator_name",
495
"destructor_name",
496
"qualified_identifier",
497
),498
)499
if function_identifier_node is not None:
500
return function_identifier_node
501
502
nested_function_declarator_node = ts_find_child_node_by_type(
503
function_declarator_node,
504
"function_declarator",
505
)506
if nested_function_declarator_node is not None:
507
return SourceFileTraceabilityReader_C._get_function_name_node(
508
nested_function_declarator_node509
)510
511
return None
512
513
@staticmethod514
def get_node_ns(node: Node) -> Sequence[str]:
515
"""
516
Walk up the tree and find parent classes.517
"""518
parent_scopes = []
519
cursor: Optional[Node] = node
520
while cursor is not None:
521
if cursor.type == "class_specifier" and len(cursor.children) > 1:
522
second_node_or_none = cursor.children[1]
523
if (
524
second_node_or_none.type == "type_identifier"
525
and second_node_or_none.text is not None
526
):527
parent_class_name = second_node_or_none.text.decode("utf8")
528
parent_scopes.append(parent_class_name)
529
elif cursor.type == "namespace_definition":
530
for c in cursor.children:
531
if c.type == "namespace_identifier" and c.text is not None:
532
parent_class_name = c.text.decode("utf8")
533
parent_scopes.append(parent_class_name)
534
break535
536
cursor = cursor.parent
537
538
parent_scopes.reverse()
539
return parent_scopes