Path:
tests/unit/strictdoc/backend/sdoc_source_code/test_marker_lexer.py
Lines:
612
Non-empty lines:
455
Non-empty lines covered with requirements:
455 / 455 (100.0%)
Functions:
25
Functions covered by requirements:
25 / 25 (100.0%)
- "7.6.1. Relation markers syntax" (REQUIREMENT)
- "7.7.1. Parse nodes from source code" (REQUIREMENT)
1
"""2
@relation(SDOC-SRS-34, SDOC-SRS-141, scope=file)3
"""4
5
from typing import Any, List, Optional
6
7
from lark import Token, Tree
8
9
from strictdoc.backend.sdoc_source_code.comment_parser.marker_lexer import (
10
MarkerLexer,
11
)12
13
14
def lark_tree_find_child_trees(tree: Any) -> List[Any]:
15
return list(filter(lambda child_: isinstance(child_, Tree), tree.children))
16
17
18
def assert_relation_marker(
19
node: Any, node_uids: List[str], scope: str, role: Optional[str] = None
20
) -> None:
21
relation_marker_trees = lark_tree_find_child_trees(node)
22
for idx_ in range(len(node_uids)):
23
assert relation_marker_trees[idx_].data == "relation_node_uid"
24
assert relation_marker_trees[idx_].children[0].value == node_uids[idx_]
25
26
assert relation_marker_trees[len(node_uids)].data == "relation_scope"
27
assert relation_marker_trees[len(node_uids)].children[0].value == scope
28
29
if role is not None:
30
assert relation_marker_trees[len(node_uids) + 1].data == "relation_role"
31
assert (
32
relation_marker_trees[len(node_uids) + 1].children[0].value == role
33
)34
35
36
def assert_node_field(
37
node: Any, field_name: str, expected_field_value: List[str]
38
) -> None:
39
node_name = next(node.find_data("node_name"))
40
assert node_name.children[0].value == field_name
41
42
node_values = node.find_data("node_multiline_value")
43
node_value = next(node_values)
44
45
values = list(
46
node_value.scan_values(
47
lambda t: (
48
t.type
49
in ("NODE_FIRST_STRING_VALUE", "NODE_STRING_VALUE", "NEWLINE")
50
)51
)52
)53
for idx in range(len(expected_field_value)):
54
assert values[idx] == expected_field_value[idx]
55
56
57
def test_01_basic_nominal() -> None:
58
tree = MarkerLexer.parse("")
59
assert tree.data == "start"
60
61
62
def test_02_single_marker() -> None:
63
input_strings = [
64
("REQ-1", "REQ-2"),
65
("REQ_1", "REQ_2"),
66
("REQ.1", "REQ.2"),
67
("REQ/1", "REQ/2"),
68
]69
for node_1_, node_2_ in input_strings:
70
input_string = f"@relation({node_1_}, {node_2_}, scope=function)"
71
tree = MarkerLexer.parse(input_string)
72
assert tree.data == "start"
73
74
relation_markers = tree.find_data("relation_marker")
75
relation_marker = next(relation_markers)
76
assert relation_marker.data == "relation_marker"
77
78
assert_relation_marker(relation_marker, [node_1_, node_2_], "function")
79
80
81
def test_03_single_marker_with_role() -> None:
82
tree = MarkerLexer.parse(
83
"@relation(REQ-1, scope=function, role=Implementation)"84
)85
assert tree.data == "start"
86
87
relation_markers = tree.find_data("relation_marker")
88
relation_marker = next(relation_markers)
89
assert relation_marker.data == "relation_marker"
90
91
assert_relation_marker(
92
relation_marker, ["REQ-1"], "function", "Implementation"
93
)94
95
96
def test_04_skip_markers() -> None:
97
tree = MarkerLexer.parse("@relation(skip, scope=file)")
98
assert tree.data == "start"
99
100
relation_markers = tree.find_data("relation_marker")
101
relation_marker = next(relation_markers)
102
assert relation_marker.data == "relation_marker"
103
104
assert_relation_marker(
105
relation_marker,
106
["skip"],
107
"file",
108
)109
110
111
def test_10_single_marker_with_newline() -> None:
112
input_string = "@relation(REQ-1, scope=function)\n"
113
114
tree = MarkerLexer.parse(input_string)
115
assert tree.data == "start"
116
117
relation_markers = tree.find_data("relation_marker")
118
relation_marker = next(relation_markers)
119
assert relation_marker.data == "relation_marker"
120
121
assert_relation_marker(
122
relation_marker,
123
["REQ-1"],
124
"function",
125
)126
127
128
def test_11_single_marker_with_newline() -> None:
129
input_string = """\
130
@relation(131
REQ-1,132
scope=function133
)134
"""135
136
tree = MarkerLexer.parse(input_string)
137
assert tree.data == "start"
138
139
relation_markers = tree.find_data("relation_marker")
140
relation_marker = next(relation_markers)
141
assert relation_marker.data == "relation_marker"
142
143
assert_relation_marker(
144
relation_marker,
145
["REQ-1"],
146
"function",
147
)148
149
150
def test_12_python_preprocessed_input() -> None:
151
input_string = """\
152
@relation(REQ-001, REQ-002, REQ-003, scope=range_start)153
"""154
155
tree = MarkerLexer.parse(input_string)
156
assert tree.data == "start"
157
158
relation_markers = tree.find_data("relation_marker")
159
relation_marker = next(relation_markers)
160
assert relation_marker.data == "relation_marker"
161
162
assert_relation_marker(
163
relation_marker,
164
["REQ-001", "REQ-002", "REQ-003"],
165
"range_start",
166
)167
168
169
def test_13_python_preprocessed_input() -> None:
170
input_string = """\
171
@relation(REQ-001, REQ-002, REQ-003, scope=range_start)172
print("Hello world")173
@relation(REQ-001, REQ-002, REQ-003, scope=range_end)174
""".lstrip()
175
176
tree = MarkerLexer.parse(input_string)
177
assert tree.data == "start"
178
179
relation_markers = tree.find_data("relation_marker")
180
relation_marker_0 = next(relation_markers)
181
relation_marker_1 = next(relation_markers)
182
183
assert_relation_marker(
184
relation_marker_0,
185
["REQ-001", "REQ-002", "REQ-003"],
186
"range_start",
187
)188
assert_relation_marker(
189
relation_marker_1,
190
["REQ-001", "REQ-002", "REQ-003"],
191
"range_end",
192
)193
194
195
def test_20_single_marker_and_normal_line() -> None:
196
input_string = """\
197
FOOBAR198
199
@relation(200
REQ-1,201
scope=function202
)203
204
FOOBAR205
"""206
207
tree = MarkerLexer.parse(input_string)
208
assert tree.data == "start"
209
210
relation_markers = tree.find_data("relation_marker")
211
relation_marker = next(relation_markers)
212
assert_relation_marker(
213
relation_marker,
214
["REQ-1"],
215
"function",
216
)217
218
219
def test_30_relation_and_field() -> None:
220
input_string = """\
221
FOOBAR222
223
@relation(224
REQ-1,225
scope=function226
)227
228
STATEMENT: When C,229
The system A shall do B230
231
STATEMENT: When Z,232
The system X shall do Y233
234
STATEMENT: When 1, The system 2 shall do 3235
236
STATEMENT: When 1, The system 2 shall do 3237
238
Additionally, the system 3 shall do 4.239
"""240
241
tree = MarkerLexer.parse(input_string, custom_tags={"STATEMENT"})
242
assert tree.data == "start"
243
244
relation_markers = tree.find_data("relation_marker")
245
relation_marker = next(relation_markers)
246
247
assert_relation_marker(
248
relation_marker,
249
["REQ-1"],
250
"function",
251
)252
253
node_fields = list(tree.find_data("node_field"))
254
255
assert_node_field(
256
node_fields[0],
257
"STATEMENT",
258
["When C,", "\n", " The system A shall do B"],
259
)260
261
assert_node_field(
262
node_fields[1],
263
"STATEMENT",
264
["When Z,", "\n", " The system X shall do Y"],
265
)266
267
assert_node_field(
268
node_fields[2],
269
"STATEMENT",
270
["When 1, The system 2 shall do 3"],
271
)272
273
assert_node_field(
274
node_fields[3],
275
"STATEMENT",
276
[277
"When 1, The system 2 shall do 3",
278
"\n\n",
279
"Additionally, the system 3 shall do 4.",
280
],281
)282
283
284
def test_31_single_node_field() -> None:
285
"""
286
Ensure that a single field can be parsed.287
288
It turns out that this particular case is pretty sensitive with respect to289
how the grammar is constructed.290
"""291
292
input_string = """
293
STATEMENT: This can likely replace _weak below with no problem.294
"""295
296
tree = MarkerLexer.parse(input_string, custom_tags={"STATEMENT"})
297
assert tree.data == "start"
298
299
node_fields = list(tree.find_data("node_field"))
300
301
assert_node_field(
302
node_fields[0],
303
"STATEMENT",
304
["This can likely replace _weak below with no problem."],
305
)306
307
308
def test_31B_single_node_field() -> None:
309
"""
310
Ensure that a single field can be parsed.311
312
It turns out that this particular case is pretty sensitive with respect to313
how the grammar is constructed.314
315
NOTE: The input below contains random whitespace.316
"""317
318
input_string = """\
319
#include <stdio.h>320
321
322
Some text.323
324
INTENTION: Intention A.325
326
327
""" # noqa: W293
328
329
tree = MarkerLexer.parse(input_string, custom_tags={"INTENTION"})
330
assert tree.data == "start"
331
332
node_fields = list(tree.find_data("node_field"))
333
334
assert_node_field(
335
node_fields[0],
336
"INTENTION",
337
["Intention A."],
338
)339
340
341
def test_31C_single_node_field() -> None:
342
"""
343
Ensure that a single field can be parsed.344
345
It turns out that this particular case is pretty sensitive with respect to346
how the grammar is constructed.347
348
NOTE: The input below contains random whitespace.349
"""350
351
input_string = """\
352
#include <stdio.h>353
354
Some text.355
356
INTENTION: Intention A.357
358
@relation(REQ-1, scope=function)359
360
void hello_world(void) {361
print("hello world\\n");
362
}363
""" # noqa: W293
364
365
tree = MarkerLexer.parse(input_string, custom_tags={"INTENTION"})
366
assert tree.data == "start"
367
368
node_fields = list(tree.find_data("node_field"))
369
370
assert_node_field(
371
node_fields[0],
372
"INTENTION",
373
["Intention A."],
374
)375
376
377
def test_32_two_single_line_fields() -> None:
378
"""
379
Ensure that a single field can be parsed.380
381
It turns out that this particular case is pretty sensitive with respect to382
how the grammar is constructed.383
"""384
385
input_string = """
386
STATEMENT: This can likely replace _weak below with no problem.387
388
STATEMENT: This can likely replace _weak below with no problem.389
"""390
391
tree = MarkerLexer.parse(input_string, custom_tags={"STATEMENT"})
392
assert tree.data == "start"
393
394
node_fields = list(tree.find_data("node_field"))
395
396
assert_node_field(
397
node_fields[0],
398
"STATEMENT",
399
["This can likely replace _weak below with no problem."],
400
)401
assert_node_field(
402
node_fields[1],
403
"STATEMENT",
404
["This can likely replace _weak below with no problem."],
405
)406
407
408
def test_32B_two_single_line_fields_consecutive() -> None:
409
"""
410
Ensure that two consecutive fields can be parsed.411
"""412
413
input_string = """
414
STATEMENT: This can likely replace _weak below with no problem.415
STATEMENTT: This can likely replace _weak below with no problem.416
"""417
418
tree = MarkerLexer.parse(
419
input_string, custom_tags={"STATEMENT", "STATEMENTT"}
420
)421
422
assert tree.data == "start"
423
424
node_fields = list(tree.find_data("node_field"))
425
426
assert_node_field(
427
node_fields[0],
428
"STATEMENT",
429
["This can likely replace _weak below with no problem."],
430
)431
assert_node_field(
432
node_fields[1],
433
"STATEMENTT",
434
["This can likely replace _weak below with no problem."],
435
)436
437
438
def test_33_multiline_and_multiparagraph_fields() -> None:
439
input_string = """\
440
FOOBAR441
442
STATEMENT: This443
444
is445
446
how we do paragraphs.447
"""448
449
tree = MarkerLexer.parse(input_string, custom_tags={"STATEMENT"})
450
assert tree.data == "start"
451
452
node_fields = list(tree.find_data("node_field"))
453
454
assert_node_field(
455
node_fields[0],
456
"STATEMENT",
457
[458
"This",
459
"\n\n",
460
" is",
461
"\n\n",
462
" how we do paragraphs.",
463
],464
)465
466
467
def test_34_node_text_starting_below() -> None:
468
"""
469
Ensure that first text can start somewhere in the next lines after a field,470
and that the result keeps starting newlines as separate NEWLINE-symbol.471
"""472
input_string = """\
473
FIELD1:474
475
Text starting far off from tag.476
"""477
tree = MarkerLexer.parse(input_string, custom_tags={"FIELD1"})
478
assert tree.data == "start"
479
480
node_fields = list(tree.find_data("node_field"))
481
assert len(node_fields) == 1
482
483
assert_node_field(
484
node_fields[0],
485
"FIELD1",
486
[487
"\n\n",
488
" Text starting far off from tag.",
489
"\n",
490
],491
)492
493
494
def test_35a_node_value_newline_lf() -> None:
495
"""Verify that LF goes into a separate NEWLINE token."""
496
input_string = "FIELD: value1\nvalue2\n"
497
tree = MarkerLexer.parse(input_string, custom_tags={"FIELD"})
498
499
node_fields = list(tree.find_data("node_field"))
500
501
assert_node_field(
502
node_fields[0],
503
"FIELD",
504
[505
Token("NODE_STRING_VALUE", "value1"),
506
Token("NEWLINE", "\n"),
507
Token("NODE_STRING_VALUE", "value2"),
508
Token("NEWLINE", "\n"),
509
],510
)511
512
513
def test_35b_node_value_newline_crlf() -> None:
514
"""Verify that CR LF goes into a separate NEWLINE token."""
515
input_string = "FIELD: value1\r\nvalue2\r\n"
516
tree = MarkerLexer.parse(input_string, custom_tags={"FIELD"})
517
518
node_fields = list(tree.find_data("node_field"))
519
520
assert_node_field(
521
node_fields[0],
522
"FIELD",
523
[524
Token("NODE_STRING_VALUE", "value1"),
525
Token("NEWLINE", "\r\n"),
526
Token("NODE_STRING_VALUE", "value2"),
527
# The implicit \r\n => \n conversion at EOF is not nice, but doesn't hurt (yet).528
# We need to improve EOF handling in lark grammar to get rid of it.529
Token("NEWLINE", "\n"),
530
],531
)532
533
534
def test_60_exclude_reserved_keywords() -> None:
535
input_string = """
536
FIXME: This can likely replace _weak below with no problem.537
538
TODO: This can likely replace _weak below with no problem.539
"""540
541
tree = MarkerLexer.parse(input_string)
542
assert tree.data == "start"
543
544
node_fields = list(tree.find_data("node_field"))
545
assert len(node_fields) == 0
546
547
548
def test_70_exclude_similar_but_not_in_grammar() -> None:
549
input_string = """
550
Note: This is ordinary comment text.551
552
STATEMENT: This can likely replace _weak below with no problem.553
FYI: More ordinary comment text.554
555
TEST: This can likely replace _weak below with no problem.556
557
Hint: Again, ordinary comment text.558
"""559
560
tree = MarkerLexer.parse(input_string, custom_tags={"STATEMENT", "TEST"})
561
assert tree.data == "start"
562
563
node_fields = list(tree.find_data("node_field"))
564
assert len(node_fields) == 2
565
566
assert_node_field(
567
node_fields[0],
568
"STATEMENT",
569
["This can likely replace _weak below with no problem."],
570
)571
assert_node_field(
572
node_fields[1],
573
"TEST",
574
["This can likely replace _weak below with no problem."],
575
)576
577
578
def test_80_linux_spdx_like_identifiers() -> None:
579
input_string = """\
580
SPDX-ID: REQ-1581
582
SPDX-Text: This583
is584
a statement585
586
And this is the same statement's another paragraph.587
"""588
589
tree = MarkerLexer.parse(input_string, custom_tags={"SPDX-ID", "SPDX-Text"})
590
assert tree.data == "start"
591
592
node_fields = list(tree.find_data("node_field"))
593
assert len(node_fields) == 2
594
595
assert_node_field(
596
node_fields[0],
597
"SPDX-ID",
598
["REQ-1"],
599
)600
assert_node_field(
601
node_fields[1],
602
"SPDX-Text",
603
[604
"This",
605
"\n",
606
" is",
607
"\n",
608
" a statement",
609
"\n\n",
610
" And this is the same statement's another paragraph.",
611
],612
)