StrictDoc Documentation
strictdoc/backend/sdoc_source_code/comment_parser/marker_lexer.py
Source file coverage
Path:
strictdoc/backend/sdoc_source_code/comment_parser/marker_lexer.py
Lines:
104
Non-empty lines:
79
Non-empty lines covered with requirements:
79 / 79 (100.0%)
Functions:
3
Functions covered by requirements:
3 / 3 (100.0%)
1
"""
2
@relation(SDOC-SRS-34, SDOC-SRS-141, scope=file)
3
"""
4
 
5
from string import Template
6
from typing import Optional
7
 
8
from lark import Lark, ParseTree, UnexpectedToken
9
 
10
from strictdoc.backend.sdoc_source_code.constants import (
11
    REGEX_REQ,
12
    RESERVED_KEYWORDS,
13
)
14
 
15
 
16
class GrammarTemplate(Template):
17
    delimiter = "##"
18
 
19
 
20
RELATION_MARKER_START = r"@relation[\(\{]"
21
 
22
NODE_GRAMMAR_EXTENSION = GrammarTemplate("""
23
node_field: node_name ":" node_multiline_value
24
node_name: /##CUSTOM_TAGS/
25
node_multiline_value: (_WS_INLINE? | (_WS_INLINE NODE_STRING_VALUE)) NEWLINE (NODE_STRING_VALUE NEWLINE)*
26
 
27
NODE_STRING_VALUE.2: /(?![ ]*##RELATION_MARKER_START)(?!\\s*(##CUSTOM_TAGS):\\s)(?!\\s*##NODE_FIELD_END_MARKER)[^\\n\\r]+/
28
 
29
_NORMAL_STRING_NO_MARKER_NO_NODE: /(?!\\s*##RELATION_MARKER_START)((?!\\s*(##CUSTOM_TAGS):\\s)|(##RESERVED_KEYWORDS)).+/
30
""")
31
 
32
GRAMMAR = GrammarTemplate("""
33
start: ##START
34
 
35
relation_marker: _RELATION_MARKER_START _WS? relation_node_uid (_SEP _WS relation_node_uid)* (_SEP _WS "scope=" relation_scope)? (_SEP _WS "role=" relation_role)? _WS? _BRACE_RIGHT
36
 
37
_RELATION_MARKER_START: /##RELATION_MARKER_START/
38
relation_node_uid: /##REGEX_REQ/
39
relation_scope: /file|class|function|line|range_start|range_end/
40
relation_role: ALPHANUMERIC_WORD
41
 
42
##GRAMMAR_EXTENSION
43
 
44
_NORMAL_STRING_NO_MARKER: /(?!\\s*##RELATION_MARKER_START).+/
45
 
46
_BRACE_LEFT: /[\\(\\{{]/
47
_BRACE_RIGHT: /[\\)\\}}]/
48
 
49
_SEP: ","
50
 
51
_NL : (CR? LF)
52
 
53
_WS : WS
54
_WS_INLINE : WS_INLINE
55
 
56
ALPHANUMERIC_WORD: /[a-zA-Z0-9_]+/
57
 
58
%import common.WS -> WS
59
%import common.CR -> CR
60
%import common.LF -> LF
61
%import common.WS_INLINE -> WS_INLINE
62
%import common.NEWLINE -> NEWLINE
63
""")
64
 
65
 
66
class MarkerLexer:
67
    @staticmethod
68
    def parse(
69
        source_input: str, custom_tags: Optional[set[str]] = None
70
    ) -> ParseTree:
71
        if custom_tags is not None:
72
            grammar_extension = NODE_GRAMMAR_EXTENSION.substitute(
73
                CUSTOM_TAGS="|".join(f"{tag}(?=:)" for tag in custom_tags),
74
                RESERVED_KEYWORDS=RESERVED_KEYWORDS,
75
                RELATION_MARKER_START=RELATION_MARKER_START,
76
                NODE_FIELD_END_MARKER="SPDX-Req-End",
77
            )
78
            start = "(relation_marker | node_field | _NORMAL_STRING_NO_MARKER_NO_NODE | _WS)*"
79
        else:
80
            grammar_extension = ""
81
            start = "(relation_marker | _NORMAL_STRING_NO_MARKER | _WS)*"
82
 
83
        grammar = GRAMMAR.substitute(
84
            GRAMMAR_EXTENSION=grammar_extension,
85
            RELATION_MARKER_START=RELATION_MARKER_START,
86
            REGEX_REQ=REGEX_REQ,
87
            START=start,
88
        )
89
        parser: Lark = Lark(
90
            grammar, parser="lalr", cache=True, propagate_positions=True
91
        )
92
 
93
        try:
94
            # FIXME: Without rstrip, there is an edge case where the parser
95
            #        breaks when resolving conflicts between multiline node
96
            #        fields and normal strings.
97
            #        See also test: test_31_single_node_field.
98
            tree: ParseTree = parser.parse(source_input.rstrip() + "\n")
99
        except UnexpectedToken as exception_:  # pragma: no cover
100
            print(  # noqa: T201
101
                "error: could not parse source comment:\n" + source_input
102
            )
103
            raise exception_
104
        return tree
1 - 104 | entire file