StrictDoc Documentation
tests/unit/strictdoc/backend/sdoc_source_code/test_marker_lexer.py
Source file coverage
Path:
tests/unit/strictdoc/backend/sdoc_source_code/test_marker_lexer.py
Lines:
612
Non-empty lines:
455
Non-empty lines covered with requirements:
455 / 455 (100.0%)
Functions:
25
Functions covered by requirements:
25 / 25 (100.0%)
1
"""
2
@relation(SDOC-SRS-34, SDOC-SRS-141, scope=file)
3
"""
4
 
5
from typing import Any, List, Optional
6
 
7
from lark import Token, Tree
8
 
9
from strictdoc.backend.sdoc_source_code.comment_parser.marker_lexer import (
10
    MarkerLexer,
11
)
12
 
13
 
14
def lark_tree_find_child_trees(tree: Any) -> List[Any]:
15
    return list(filter(lambda child_: isinstance(child_, Tree), tree.children))
16
 
17
 
18
def assert_relation_marker(
19
    node: Any, node_uids: List[str], scope: str, role: Optional[str] = None
20
) -> None:
21
    relation_marker_trees = lark_tree_find_child_trees(node)
22
    for idx_ in range(len(node_uids)):
23
        assert relation_marker_trees[idx_].data == "relation_node_uid"
24
        assert relation_marker_trees[idx_].children[0].value == node_uids[idx_]
25
 
26
    assert relation_marker_trees[len(node_uids)].data == "relation_scope"
27
    assert relation_marker_trees[len(node_uids)].children[0].value == scope
28
 
29
    if role is not None:
30
        assert relation_marker_trees[len(node_uids) + 1].data == "relation_role"
31
        assert (
32
            relation_marker_trees[len(node_uids) + 1].children[0].value == role
33
        )
34
 
35
 
36
def assert_node_field(
37
    node: Any, field_name: str, expected_field_value: List[str]
38
) -> None:
39
    node_name = next(node.find_data("node_name"))
40
    assert node_name.children[0].value == field_name
41
 
42
    node_values = node.find_data("node_multiline_value")
43
    node_value = next(node_values)
44
 
45
    values = list(
46
        node_value.scan_values(
47
            lambda t: (
48
                t.type
49
                in ("NODE_FIRST_STRING_VALUE", "NODE_STRING_VALUE", "NEWLINE")
50
            )
51
        )
52
    )
53
    for idx in range(len(expected_field_value)):
54
        assert values[idx] == expected_field_value[idx]
55
 
56
 
57
def test_01_basic_nominal() -> None:
58
    tree = MarkerLexer.parse("")
59
    assert tree.data == "start"
60
 
61
 
62
def test_02_single_marker() -> None:
63
    input_strings = [
64
        ("REQ-1", "REQ-2"),
65
        ("REQ_1", "REQ_2"),
66
        ("REQ.1", "REQ.2"),
67
        ("REQ/1", "REQ/2"),
68
    ]
69
    for node_1_, node_2_ in input_strings:
70
        input_string = f"@relation({node_1_}, {node_2_}, scope=function)"
71
        tree = MarkerLexer.parse(input_string)
72
        assert tree.data == "start"
73
 
74
        relation_markers = tree.find_data("relation_marker")
75
        relation_marker = next(relation_markers)
76
        assert relation_marker.data == "relation_marker"
77
 
78
        assert_relation_marker(relation_marker, [node_1_, node_2_], "function")
79
 
80
 
81
def test_03_single_marker_with_role() -> None:
82
    tree = MarkerLexer.parse(
83
        "@relation(REQ-1, scope=function, role=Implementation)"
84
    )
85
    assert tree.data == "start"
86
 
87
    relation_markers = tree.find_data("relation_marker")
88
    relation_marker = next(relation_markers)
89
    assert relation_marker.data == "relation_marker"
90
 
91
    assert_relation_marker(
92
        relation_marker, ["REQ-1"], "function", "Implementation"
93
    )
94
 
95
 
96
def test_04_skip_markers() -> None:
97
    tree = MarkerLexer.parse("@relation(skip, scope=file)")
98
    assert tree.data == "start"
99
 
100
    relation_markers = tree.find_data("relation_marker")
101
    relation_marker = next(relation_markers)
102
    assert relation_marker.data == "relation_marker"
103
 
104
    assert_relation_marker(
105
        relation_marker,
106
        ["skip"],
107
        "file",
108
    )
109
 
110
 
111
def test_10_single_marker_with_newline() -> None:
112
    input_string = "@relation(REQ-1, scope=function)\n"
113
 
114
    tree = MarkerLexer.parse(input_string)
115
    assert tree.data == "start"
116
 
117
    relation_markers = tree.find_data("relation_marker")
118
    relation_marker = next(relation_markers)
119
    assert relation_marker.data == "relation_marker"
120
 
121
    assert_relation_marker(
122
        relation_marker,
123
        ["REQ-1"],
124
        "function",
125
    )
126
 
127
 
128
def test_11_single_marker_with_newline() -> None:
129
    input_string = """\
130
@relation(
131
    REQ-1,
132
    scope=function
133
)
134
"""
135
 
136
    tree = MarkerLexer.parse(input_string)
137
    assert tree.data == "start"
138
 
139
    relation_markers = tree.find_data("relation_marker")
140
    relation_marker = next(relation_markers)
141
    assert relation_marker.data == "relation_marker"
142
 
143
    assert_relation_marker(
144
        relation_marker,
145
        ["REQ-1"],
146
        "function",
147
    )
148
 
149
 
150
def test_12_python_preprocessed_input() -> None:
151
    input_string = """\
152
  @relation(REQ-001, REQ-002, REQ-003, scope=range_start)
153
"""
154
 
155
    tree = MarkerLexer.parse(input_string)
156
    assert tree.data == "start"
157
 
158
    relation_markers = tree.find_data("relation_marker")
159
    relation_marker = next(relation_markers)
160
    assert relation_marker.data == "relation_marker"
161
 
162
    assert_relation_marker(
163
        relation_marker,
164
        ["REQ-001", "REQ-002", "REQ-003"],
165
        "range_start",
166
    )
167
 
168
 
169
def test_13_python_preprocessed_input() -> None:
170
    input_string = """\
171
  @relation(REQ-001, REQ-002, REQ-003, scope=range_start)
172
  print("Hello world")
173
  @relation(REQ-001, REQ-002, REQ-003, scope=range_end)
174
""".lstrip()
175
 
176
    tree = MarkerLexer.parse(input_string)
177
    assert tree.data == "start"
178
 
179
    relation_markers = tree.find_data("relation_marker")
180
    relation_marker_0 = next(relation_markers)
181
    relation_marker_1 = next(relation_markers)
182
 
183
    assert_relation_marker(
184
        relation_marker_0,
185
        ["REQ-001", "REQ-002", "REQ-003"],
186
        "range_start",
187
    )
188
    assert_relation_marker(
189
        relation_marker_1,
190
        ["REQ-001", "REQ-002", "REQ-003"],
191
        "range_end",
192
    )
193
 
194
 
195
def test_20_single_marker_and_normal_line() -> None:
196
    input_string = """\
197
FOOBAR
198
 
199
@relation(
200
    REQ-1,
201
    scope=function
202
)
203
 
204
FOOBAR
205
"""
206
 
207
    tree = MarkerLexer.parse(input_string)
208
    assert tree.data == "start"
209
 
210
    relation_markers = tree.find_data("relation_marker")
211
    relation_marker = next(relation_markers)
212
    assert_relation_marker(
213
        relation_marker,
214
        ["REQ-1"],
215
        "function",
216
    )
217
 
218
 
219
def test_30_relation_and_field() -> None:
220
    input_string = """\
221
FOOBAR
222
 
223
@relation(
224
    REQ-1,
225
    scope=function
226
)
227
 
228
STATEMENT: When C,
229
           The system A shall do B
230
 
231
STATEMENT: When Z,
232
           The system X shall do Y
233
 
234
STATEMENT: When 1, The system 2 shall do 3
235
 
236
STATEMENT: When 1, The system 2 shall do 3
237
 
238
Additionally, the system 3 shall do 4.
239
"""
240
 
241
    tree = MarkerLexer.parse(input_string, custom_tags={"STATEMENT"})
242
    assert tree.data == "start"
243
 
244
    relation_markers = tree.find_data("relation_marker")
245
    relation_marker = next(relation_markers)
246
 
247
    assert_relation_marker(
248
        relation_marker,
249
        ["REQ-1"],
250
        "function",
251
    )
252
 
253
    node_fields = list(tree.find_data("node_field"))
254
 
255
    assert_node_field(
256
        node_fields[0],
257
        "STATEMENT",
258
        ["When C,", "\n", "           The system A shall do B"],
259
    )
260
 
261
    assert_node_field(
262
        node_fields[1],
263
        "STATEMENT",
264
        ["When Z,", "\n", "           The system X shall do Y"],
265
    )
266
 
267
    assert_node_field(
268
        node_fields[2],
269
        "STATEMENT",
270
        ["When 1, The system 2 shall do 3"],
271
    )
272
 
273
    assert_node_field(
274
        node_fields[3],
275
        "STATEMENT",
276
        [
277
            "When 1, The system 2 shall do 3",
278
            "\n\n",
279
            "Additionally, the system 3 shall do 4.",
280
        ],
281
    )
282
 
283
 
284
def test_31_single_node_field() -> None:
285
    """
286
    Ensure that a single field can be parsed.
287
 
288
    It turns out that this particular case is pretty sensitive with respect to
289
    how the grammar is constructed.
290
    """
291
 
292
    input_string = """
293
        STATEMENT: This can likely replace _weak below with no problem.
294
    """
295
 
296
    tree = MarkerLexer.parse(input_string, custom_tags={"STATEMENT"})
297
    assert tree.data == "start"
298
 
299
    node_fields = list(tree.find_data("node_field"))
300
 
301
    assert_node_field(
302
        node_fields[0],
303
        "STATEMENT",
304
        ["This can likely replace _weak below with no problem."],
305
    )
306
 
307
 
308
def test_31B_single_node_field() -> None:
309
    """
310
    Ensure that a single field can be parsed.
311
 
312
    It turns out that this particular case is pretty sensitive with respect to
313
    how the grammar is constructed.
314
 
315
    NOTE: The input below contains random whitespace.
316
    """
317
 
318
    input_string = """\
319
#include <stdio.h>
320
 
321
   
322
   Some text.
323
  
324
   INTENTION: Intention A.
325
  
326
  
327
    """  # noqa: W293
328
 
329
    tree = MarkerLexer.parse(input_string, custom_tags={"INTENTION"})
330
    assert tree.data == "start"
331
 
332
    node_fields = list(tree.find_data("node_field"))
333
 
334
    assert_node_field(
335
        node_fields[0],
336
        "INTENTION",
337
        ["Intention A."],
338
    )
339
 
340
 
341
def test_31C_single_node_field() -> None:
342
    """
343
    Ensure that a single field can be parsed.
344
 
345
    It turns out that this particular case is pretty sensitive with respect to
346
    how the grammar is constructed.
347
 
348
    NOTE: The input below contains random whitespace.
349
    """
350
 
351
    input_string = """\
352
#include <stdio.h>
353
   
354
    Some text.
355
  
356
   INTENTION: Intention A.
357
   
358
   @relation(REQ-1, scope=function)
359
   
360
void hello_world(void) {
361
    print("hello world\\n");
362
}
363
"""  # noqa: W293
364
 
365
    tree = MarkerLexer.parse(input_string, custom_tags={"INTENTION"})
366
    assert tree.data == "start"
367
 
368
    node_fields = list(tree.find_data("node_field"))
369
 
370
    assert_node_field(
371
        node_fields[0],
372
        "INTENTION",
373
        ["Intention A."],
374
    )
375
 
376
 
377
def test_32_two_single_line_fields() -> None:
378
    """
379
    Ensure that a single field can be parsed.
380
 
381
    It turns out that this particular case is pretty sensitive with respect to
382
    how the grammar is constructed.
383
    """
384
 
385
    input_string = """
386
        STATEMENT: This can likely replace _weak below with no problem.
387
 
388
        STATEMENT: This can likely replace _weak below with no problem.
389
    """
390
 
391
    tree = MarkerLexer.parse(input_string, custom_tags={"STATEMENT"})
392
    assert tree.data == "start"
393
 
394
    node_fields = list(tree.find_data("node_field"))
395
 
396
    assert_node_field(
397
        node_fields[0],
398
        "STATEMENT",
399
        ["This can likely replace _weak below with no problem."],
400
    )
401
    assert_node_field(
402
        node_fields[1],
403
        "STATEMENT",
404
        ["This can likely replace _weak below with no problem."],
405
    )
406
 
407
 
408
def test_32B_two_single_line_fields_consecutive() -> None:
409
    """
410
    Ensure that two consecutive fields can be parsed.
411
    """
412
 
413
    input_string = """
414
        STATEMENT: This can likely replace _weak below with no problem.
415
        STATEMENTT: This can likely replace _weak below with no problem.
416
    """
417
 
418
    tree = MarkerLexer.parse(
419
        input_string, custom_tags={"STATEMENT", "STATEMENTT"}
420
    )
421
 
422
    assert tree.data == "start"
423
 
424
    node_fields = list(tree.find_data("node_field"))
425
 
426
    assert_node_field(
427
        node_fields[0],
428
        "STATEMENT",
429
        ["This can likely replace _weak below with no problem."],
430
    )
431
    assert_node_field(
432
        node_fields[1],
433
        "STATEMENTT",
434
        ["This can likely replace _weak below with no problem."],
435
    )
436
 
437
 
438
def test_33_multiline_and_multiparagraph_fields() -> None:
439
    input_string = """\
440
FOOBAR
441
 
442
STATEMENT: This
443
 
444
           is
445
 
446
           how we do paragraphs.
447
"""
448
 
449
    tree = MarkerLexer.parse(input_string, custom_tags={"STATEMENT"})
450
    assert tree.data == "start"
451
 
452
    node_fields = list(tree.find_data("node_field"))
453
 
454
    assert_node_field(
455
        node_fields[0],
456
        "STATEMENT",
457
        [
458
            "This",
459
            "\n\n",
460
            "           is",
461
            "\n\n",
462
            "           how we do paragraphs.",
463
        ],
464
    )
465
 
466
 
467
def test_34_node_text_starting_below() -> None:
468
    """
469
    Ensure that first text can start somewhere in the next lines after a field,
470
    and that the result keeps starting newlines as separate NEWLINE-symbol.
471
    """
472
    input_string = """\
473
FIELD1:
474
 
475
   Text starting far off from tag.
476
"""
477
    tree = MarkerLexer.parse(input_string, custom_tags={"FIELD1"})
478
    assert tree.data == "start"
479
 
480
    node_fields = list(tree.find_data("node_field"))
481
    assert len(node_fields) == 1
482
 
483
    assert_node_field(
484
        node_fields[0],
485
        "FIELD1",
486
        [
487
            "\n\n",
488
            "   Text starting far off from tag.",
489
            "\n",
490
        ],
491
    )
492
 
493
 
494
def test_35a_node_value_newline_lf() -> None:
495
    """Verify that LF goes into a separate NEWLINE token."""
496
    input_string = "FIELD: value1\nvalue2\n"
497
    tree = MarkerLexer.parse(input_string, custom_tags={"FIELD"})
498
 
499
    node_fields = list(tree.find_data("node_field"))
500
 
501
    assert_node_field(
502
        node_fields[0],
503
        "FIELD",
504
        [
505
            Token("NODE_STRING_VALUE", "value1"),
506
            Token("NEWLINE", "\n"),
507
            Token("NODE_STRING_VALUE", "value2"),
508
            Token("NEWLINE", "\n"),
509
        ],
510
    )
511
 
512
 
513
def test_35b_node_value_newline_crlf() -> None:
514
    """Verify that CR LF goes into a separate NEWLINE token."""
515
    input_string = "FIELD: value1\r\nvalue2\r\n"
516
    tree = MarkerLexer.parse(input_string, custom_tags={"FIELD"})
517
 
518
    node_fields = list(tree.find_data("node_field"))
519
 
520
    assert_node_field(
521
        node_fields[0],
522
        "FIELD",
523
        [
524
            Token("NODE_STRING_VALUE", "value1"),
525
            Token("NEWLINE", "\r\n"),
526
            Token("NODE_STRING_VALUE", "value2"),
527
            # The implicit \r\n => \n conversion at EOF is not nice, but doesn't hurt (yet).
528
            # We need to improve EOF handling in lark grammar to get rid of it.
529
            Token("NEWLINE", "\n"),
530
        ],
531
    )
532
 
533
 
534
def test_60_exclude_reserved_keywords() -> None:
535
    input_string = """
536
        FIXME: This can likely replace _weak below with no problem.
537
 
538
        TODO: This can likely replace _weak below with no problem.
539
    """
540
 
541
    tree = MarkerLexer.parse(input_string)
542
    assert tree.data == "start"
543
 
544
    node_fields = list(tree.find_data("node_field"))
545
    assert len(node_fields) == 0
546
 
547
 
548
def test_70_exclude_similar_but_not_in_grammar() -> None:
549
    input_string = """
550
        Note: This is ordinary comment text.
551
 
552
        STATEMENT: This can likely replace _weak below with no problem.
553
        FYI: More ordinary comment text.
554
 
555
        TEST: This can likely replace _weak below with no problem.
556
 
557
        Hint: Again, ordinary comment text.
558
    """
559
 
560
    tree = MarkerLexer.parse(input_string, custom_tags={"STATEMENT", "TEST"})
561
    assert tree.data == "start"
562
 
563
    node_fields = list(tree.find_data("node_field"))
564
    assert len(node_fields) == 2
565
 
566
    assert_node_field(
567
        node_fields[0],
568
        "STATEMENT",
569
        ["This can likely replace _weak below with no problem."],
570
    )
571
    assert_node_field(
572
        node_fields[1],
573
        "TEST",
574
        ["This can likely replace _weak below with no problem."],
575
    )
576
 
577
 
578
def test_80_linux_spdx_like_identifiers() -> None:
579
    input_string = """\
580
SPDX-ID: REQ-1
581
 
582
SPDX-Text: This
583
           is
584
           a statement
585
 
586
           And this is the same statement's another paragraph.
587
"""
588
 
589
    tree = MarkerLexer.parse(input_string, custom_tags={"SPDX-ID", "SPDX-Text"})
590
    assert tree.data == "start"
591
 
592
    node_fields = list(tree.find_data("node_field"))
593
    assert len(node_fields) == 2
594
 
595
    assert_node_field(
596
        node_fields[0],
597
        "SPDX-ID",
598
        ["REQ-1"],
599
    )
600
    assert_node_field(
601
        node_fields[1],
602
        "SPDX-Text",
603
        [
604
            "This",
605
            "\n",
606
            "           is",
607
            "\n",
608
            "           a statement",
609
            "\n\n",
610
            "           And this is the same statement's another paragraph.",
611
        ],
612
    )