Path:
strictdoc/backend/sdoc_source_code/marker_parser.py
Lines:
325
Non-empty lines:
285
Non-empty lines covered with requirements:
285 / 285 (100.0%)
Functions:
4
Functions covered by requirements:
4 / 4 (100.0%)
- "7.6.1. Relation markers syntax" (REQUIREMENT)
- "7.7.1. Parse nodes from source code" (REQUIREMENT)
1
"""2
@relation(SDOC-SRS-34, SDOC-SRS-141, scope=file)3
"""4
5
from typing import Dict, List, Optional, Tuple, Union
6
7
from lark import ParseTree, Token, Tree
8
9
from strictdoc.backend.sdoc.error_handling import StrictDocSemanticError
10
from strictdoc.backend.sdoc_source_code.comment_parser.marker_lexer import (
11
MarkerLexer,
12
)13
from strictdoc.backend.sdoc_source_code.helpers.comment_preprocessor import (
14
preprocess_source_code_comment,
15
)16
from strictdoc.backend.sdoc_source_code.models.language_item_marker import (
17
LanguageItemMarker,
18
)19
from strictdoc.backend.sdoc_source_code.models.line_marker import LineMarker
20
from strictdoc.backend.sdoc_source_code.models.range_marker import (
21
RangeMarker,
22
)23
from strictdoc.backend.sdoc_source_code.models.requirement_marker import Req
24
from strictdoc.backend.sdoc_source_code.models.source_location import ByteRange
25
from strictdoc.backend.sdoc_source_code.models.source_node import SourceNode
26
27
28
class MarkerParser:
29
@staticmethod30
def parse(
31
*,
32
input_string: str,
33
line_start: int,
34
line_end: int,
35
comment_line_start: int,
36
comment_byte_range: Optional[ByteRange],
37
filename: Optional[str] = None,
38
entity_name: Optional[str] = None,
39
col_offset: int = 0,
40
custom_tags: Optional[set[str]] = None,
41
default_scope: Optional[str] = None,
42
) -> SourceNode:
43
"""
44
Parse source nodes and relation markers from source file comments.45
46
The input_string is parsed for @relation markers. If custom_tags are given,47
input_string is additionally parsed for source nodes and SourceNode.fields_locations48
offsets are calculated relative to input_string. This implies that input_string49
lines must not be pre-stripped by the caller, otherwise offsets would mismatch with50
actual file content and source node write-back would corrupt source files.51
Comment symbols like /** ... */ or /// Doxygen comments or Python comments52
are instead replaced internally with spaces (preserving string length), so that53
all byte offsets remain valid for both parsing and file write-back.54
55
The 1-based line start/end provide hints to the parser for the case markers56
of scope file, class or function are found, in which case the user values are57
set as highlight range. If the parser finds line or range markers, the user58
provided line start/end values are ignored. Should be set to the item definition59
block, *with* leading comment lines if any.60
61
The 1-based comment_line_start parameter is the first actual comment line.62
It is required as a base offset for some parser tokens to determine their63
absolute position in file, as lexing gives only a position relative64
to comment start.65
66
comment_byte_range, if given, enables write-back of modified source nodes.67
Modification happens when a user edits the source node in the web server, or68
when StrictDoc auto-assigns MID or HASH. Values are 0-based byte-offsets69
specifying the exact input_string start-to-end position inside the source file.70
71
custom_tags is a set of valid tags if a comment is expected to contain72
key-value pairs for source node generation. The caller is responsible to determine73
valid custom tags from the grammar element associated with the source code file.74
75
filename should be given if input_string comes from a static source file.76
It will be used to create more helpful parsing error messages.77
78
entity_name is required for language item markers. It's the user-visible79
description of the marked range in the rendered document. Should be equal80
to the related LanguageItem.description for consistency with forward markers.81
82
default_scope should be provided if the caller's language-aware parser83
can infer the scope from the semantic comment position. Think of Rust doc84
comments for example. If given, users are allowed to omit the scope argument85
in a relation marker. A user provided scope argument always takes preference.86
If neither default nor a user provided value is available,87
StrictDocSemanticError will be raised.88
89
The function returns a SourceNode. Note: This is also the case if no custom tags were90
found at all (in which case fields is empty) because SourceNode also acts as a container91
for markers.92
"""93
94
node_fields: Dict[str, str] = {}
95
96
source_node: SourceNode = SourceNode(
97
entity_name=entity_name,
98
comment_byte_range=comment_byte_range,
99
)100
input_string = preprocess_source_code_comment(input_string)
101
102
tree: ParseTree = MarkerLexer.parse(
103
input_string, custom_tags=custom_tags
104
)105
106
for element_ in tree.children:
107
if not isinstance(element_, Tree):
108
continue109
110
if element_.data == "relation_marker":
111
relation_markers = MarkerParser._parse_relation_marker(
112
element_=element_,
113
line_start=line_start,
114
line_end=line_end,
115
comment_line_start=comment_line_start,
116
filename=filename,
117
entity_name=entity_name,
118
col_offset=col_offset,
119
default_scope=default_scope,
120
)121
source_node.markers.extend(relation_markers)
122
123
elif element_.data == "node_field":
124
node_name, node_value = MarkerParser._parse_node_field(
125
element_,
126
)127
node_fields[node_name] = node_value
128
129
source_node.fields_locations[node_name] = (
130
element_.meta.start_pos,
131
element_.meta.end_pos - 1,
132
)133
else:
134
raise AssertionError
135
136
if len(node_fields) > 0:
137
source_node.fields = node_fields
138
139
return source_node
140
141
@staticmethod142
def _parse_relation_marker(
143
*,
144
element_: Tree[Token],
145
line_start: int,
146
line_end: int,
147
comment_line_start: int,
148
filename: Optional[str] = None,
149
entity_name: Optional[str] = None,
150
col_offset: int = 0,
151
default_scope: Optional[str] = None,
152
) -> List[Union[LanguageItemMarker, RangeMarker, LineMarker]]:
153
markers: List[Union[LanguageItemMarker, RangeMarker, LineMarker]] = []
154
155
relation_uid_elements = []
156
relation_scope_element: Optional[Tree[Token]] = None
157
relation_role_element: Optional[Tree[Token]] = None
158
for relation_marker_element_ in element_.children:
159
assert isinstance(relation_marker_element_, Tree)
160
if relation_marker_element_.data == "relation_node_uid":
161
relation_uid_elements.append(relation_marker_element_)
162
elif relation_marker_element_.data == "relation_scope":
163
relation_scope_element = relation_marker_element_
164
elif relation_marker_element_.data == "relation_role":
165
relation_role_element = relation_marker_element_
166
else:
167
raise NotImplementedError
168
169
assert len(relation_uid_elements) > 0
170
171
if relation_scope_element is not None:
172
assert isinstance(relation_scope_element.children[0], Token)
173
relation_scope = relation_scope_element.children[0].value
174
else:
175
relation_scope = default_scope
176
177
relation_role = None
178
if relation_role_element is not None:
179
assert isinstance(relation_role_element.children[0], Token)
180
relation_role = relation_role_element.children[0].value
181
182
requirements = []
183
used_uids = set()
184
185
for relation_uid_token_ in relation_uid_elements:
186
assert isinstance(relation_uid_token_.children[0], Token)
187
assert relation_uid_token_.children[0].line is not None
188
189
relation_uid = relation_uid_token_.children[0].value
190
if relation_uid in used_uids:
191
raise ValueError(
192
f"@relation marker contains duplicate node UIDs: ['{relation_uid}']. "
193
f"Location: {filename}:{relation_uid_token_.children[0].line}."
194
)195
used_uids.add(relation_uid)
196
197
requirement = Req(None, relation_uid)
198
requirement.ng_source_line = (
199
comment_line_start + relation_uid_token_.children[0].line - 1
200
)201
requirement.ng_source_column = relation_uid_token_.children[
202
0203
].column
204
requirements.append(requirement)
205
206
if relation_scope in ("file", "class", "function"):
207
language_item_marker = LanguageItemMarker(
208
None, requirements, scope=relation_scope, role=relation_role
209
)210
language_item_marker.ng_source_line_begin = (
211
comment_line_start + element_.meta.line - 1
212
)213
language_item_marker.ng_source_column_begin = (
214
element_.meta.column + col_offset
215
)216
language_item_marker.ng_range_line_begin = line_start
217
language_item_marker.ng_range_line_end = line_end
218
if relation_scope == "file":
219
language_item_marker.set_description("entire file")
220
elif relation_scope == "function":
221
language_item_marker.set_description(
222
f"function {entity_name}()"
223
)224
elif relation_scope == "class":
225
language_item_marker.set_description(f"class {entity_name}")
226
markers.append(language_item_marker)
227
elif relation_scope in ("range_start", "range_end"):
228
range_marker = RangeMarker(
229
None,
230
requirements,
231
scope=relation_scope,
232
role=relation_role,
233
)234
range_marker.ng_source_line_begin = (
235
comment_line_start + element_.meta.line - 1
236
)237
range_marker.ng_source_column_begin = (
238
element_.meta.column + col_offset
239
)240
range_marker.ng_range_line_begin = (
241
comment_line_start + element_.meta.line - 1
242
)243
range_marker.ng_range_line_end = (
244
comment_line_start + element_.meta.end_line - 1
245
)246
markers.append(range_marker)
247
elif relation_scope == "line":
248
line_marker = LineMarker(None, requirements, role=relation_role)
249
line_marker.ng_source_line_begin = (
250
comment_line_start + element_.meta.line - 1
251
)252
line_marker.ng_source_column_begin = (
253
element_.meta.column + col_offset
254
)255
line_marker.ng_range_line_begin = (
256
comment_line_start + element_.meta.line - 1
257
)258
line_marker.ng_range_line_end = (
259
comment_line_start + element_.meta.end_line
260
)261
markers.append(line_marker)
262
elif relation_scope is None:
263
reqs = ",".join(sorted(used_uids))
264
raise StrictDocSemanticError(
265
title=f"@relation marker for requirements {reqs} misses scope argument.",
266
hint="Scope can only be omitted if supported by language, as e.g. with Rust doc comments.",
267
example=(
268
"Add a scope argument. Example:\n"
269
f"@relation({reqs}, scope=function)"
270
),271
line=comment_line_start + element_.meta.line - 1,
272
filename=filename,
273
)274
else:
275
raise NotImplementedError
276
277
return markers
278
279
@staticmethod280
def _parse_node_field(
281
element_: Tree[Token],
282
) -> Tuple[str, str]:
283
node_name_node = element_.children[0]
284
assert isinstance(node_name_node, Tree)
285
assert node_name_node.data == "node_name"
286
assert isinstance(node_name_node.children[0], Token)
287
node_name = node_name_node.children[0].value
288
289
node_value_node = element_.children[1]
290
assert isinstance(node_value_node, Tree)
291
assert node_value_node.data == "node_multiline_value"
292
293
# Find minimal indent in lines 1..n. It will be used to dedent the block.294
dedent = None
295
if len(node_value_node.children) > 1:
296
for node_value_component_ in node_value_node.children[1:]:
297
assert isinstance(node_value_component_, Token)
298
if node_value_component_.type == "NEWLINE":
299
continue300
line_value = node_value_component_.value
301
non_ws_len = len(line_value.lstrip(" "))
302
this_dedent = len(line_value) - non_ws_len
303
if dedent is None:
304
dedent = this_dedent
305
elif non_ws_len > 0:
306
dedent = min(this_dedent, dedent)
307
if dedent is None:
308
dedent = 0
309
310
# Join and dedent.311
node_value = ""
312
for i, node_value_component_ in enumerate(node_value_node.children):
313
assert isinstance(node_value_component_, Token)
314
line_value = node_value_component_.value
315
if (
316
i > 0
317
and node_value_component_.type != "NEWLINE"
318
and dedent is not None
319
):320
line_value = line_value[min(dedent, len(line_value)) :]
321
node_value += line_value
322
323
node_value = node_value.rstrip()
324
325
return node_name, node_value