StrictDoc Documentation
strictdoc/backend/excel/import_/excel_to_sdoc_converter.py
Source file coverage
Path:
strictdoc/backend/excel/import_/excel_to_sdoc_converter.py
Lines:
288
Non-empty lines:
260
Non-empty lines covered with requirements:
260 / 260 (100.0%)
Functions:
7
Functions covered by requirements:
7 / 7 (100.0%)
1
"""
2
@relation(SDOC-SRS-152, scope=file)
3
"""
4
 
5
import os
6
from dataclasses import dataclass
7
from typing import List, Optional, Tuple, Union
8
 
9
from strictdoc.backend.excel.import_.excel_sheet_proxy import ExcelSheetProxy
10
from strictdoc.backend.sdoc.models.document import SDocDocument
11
from strictdoc.backend.sdoc.models.document_config import DocumentConfig
12
from strictdoc.backend.sdoc.models.document_grammar import (
13
    DocumentGrammar,
14
)
15
from strictdoc.backend.sdoc.models.grammar_element import (
16
    GrammarElement,
17
    GrammarElementFieldString,
18
)
19
from strictdoc.backend.sdoc.models.node import SDocNode, SDocNodeField
20
from strictdoc.backend.sdoc.models.object_factory import SDocObjectFactory
21
from strictdoc.backend.sdoc.models.reference import ParentReqReference
22
from strictdoc.helpers.string import ensure_newline
23
 
24
 
25
@dataclass
26
class ExcelSheetSchema:
27
    uid_column_idx: Optional[int]
28
    title_column_idx: Optional[int]
29
    statement_column_idx: Optional[int]
30
    comment_column_idx: Optional[int]
31
    parent_column_idx: Optional[int]
32
    extra_header_pairs: List[Tuple[int, str]]
33
    column_types: dict[int, bool]
34
 
35
    def is_column_multiline(self, column_idx: int) -> bool:
36
        return self.column_types.get(column_idx, False)
37
 
38
 
39
def safe_name(dangerous_name: str) -> str:
40
    dangerous_name = dangerous_name.splitlines()[0]
41
    dangerous_name = dangerous_name.strip()
42
    dangerous_name = dangerous_name.upper()
43
    dangerous_name = dangerous_name.replace(":", "")
44
    dangerous_name = dangerous_name.replace("+", "")
45
    dangerous_name = dangerous_name.replace(",", "_")
46
    dangerous_name = dangerous_name.replace(" ", "_")
47
    dangerous_name = dangerous_name.replace("-", "_")
48
    dangerous_name = dangerous_name.replace("/", "_OR_")
49
    return dangerous_name
50
 
51
 
52
class ExcelToSDocConverter:
53
    @staticmethod
54
    def convert(
55
        excel_file: str, title: Union[str, None] = None
56
    ) -> SDocDocument:
57
        """
58
        Convert a provided Excel file to an SDoc document.
59
 
60
        Optional argument title is present for external scripts to assign a title.
61
        """
62
        sheet = ExcelSheetProxy(excel_file)
63
 
64
        excel_file_name = os.path.basename(excel_file)
65
        if title is None:
66
            title = excel_file_name + " sheet " + sheet.name
67
 
68
        header_column_indexes = list(range(sheet.ncols))
69
 
70
        # The first 16 rows should do ¯\_(ツ)_/¯.
71
        for i in range(16):
72
            if sheet.row_values(i)[0].strip() != "":
73
                header_row_idx = i
74
                break
75
        else:
76
            raise NotImplementedError
77
 
78
        header_row = sheet.row_values(header_row_idx)
79
        safe_header_row = [safe_name(x) for x in header_row]
80
 
81
        statement_column_idx = None
82
        uid_column_idx = None
83
        comment_column_idx = None
84
        title_column_idx = None
85
        parent_column_idx = None
86
        for header_column_idx_, header_column_title_ in enumerate(
87
            safe_header_row
88
        ):
89
            # Detect reserved field columns. The rest will be treated as extra
90
            # fields.
91
            if header_column_title_ in ("REQUIREMENT", "STATEMENT"):
92
                statement_column_idx = header_column_idx_
93
            elif header_column_title_ in (
94
                "REF",
95
                "REF #",
96
                "REF_#",
97
                "REFDES",
98
                "ID",
99
                "UID",
100
            ):
101
                uid_column_idx = header_column_idx_
102
            elif header_column_title_ in ("REMARKS", "COMMENT"):
103
                comment_column_idx = header_column_idx_
104
            elif header_column_title_ in ("TITLE", "NAME"):
105
                title_column_idx = header_column_idx_
106
            elif header_column_title_ in ("PARENT", "PARENT_REF", "PARENT_UID"):
107
                parent_column_idx = header_column_idx_
108
            else:
109
                continue
110
            header_column_indexes.remove(header_column_idx_)
111
 
112
        assert statement_column_idx is not None, (
113
            "Couldn't detect a column for requirement statements among the headers: "
114
            + ", ".join(safe_header_row)
115
        )
116
 
117
        extra_header_pairs: List[Tuple[int, str]] = list(
118
            map(lambda x: (x, safe_header_row[x]), header_column_indexes)
119
        )
120
 
121
        # For each column, check if it contains multiline values.
122
        # All rows must be checked, otherwise we might miss multiline values
123
        # that are only present in some rows. This is needed to determine
124
        # whether to treat the field as multiline in the SDoc document.
125
        column_types: dict[int, bool] = {}
126
        for i in range(header_row_idx + 1, sheet.nrows):
127
            row_values = sheet.row_values(i)
128
            for header_column_idx in header_column_indexes:
129
                value = str(row_values[header_column_idx]).strip()
130
                if (
131
                    isinstance(value, str)
132
                    and "\n" in value
133
                    or len(str(value)) > 60
134
                ):
135
                    column_types[header_column_idx] = True
136
                elif header_column_idx not in column_types:
137
                    column_types[header_column_idx] = False
138
 
139
        schema = ExcelSheetSchema(
140
            uid_column_idx=uid_column_idx,
141
            title_column_idx=title_column_idx,
142
            statement_column_idx=statement_column_idx,
143
            comment_column_idx=comment_column_idx,
144
            parent_column_idx=parent_column_idx,
145
            extra_header_pairs=extra_header_pairs,
146
            column_types=column_types,
147
        )
148
        document = ExcelToSDocConverter.create_document(
149
            title, extra_header_pairs
150
        )
151
        for i in range(header_row_idx + 1, sheet.nrows):
152
            row_values = sheet.row_values(i)
153
            requirement = ExcelToSDocConverter.create_requirement(
154
                row_values, document, schema
155
            )
156
            document.section_contents.append(requirement)
157
 
158
        return document
159
 
160
    @staticmethod
161
    def create_document(
162
        title: Optional[str], extra_header_pairs: List[Tuple[int, str]]
163
    ) -> SDocDocument:
164
        document_config = DocumentConfig.default_config(None)
165
        document_title = title if title else "<No title>"
166
        document = SDocDocument(
167
            mid=None,
168
            title=document_title,
169
            config=document_config,
170
            view=None,
171
            grammar=None,
172
            section_contents=[],
173
        )
174
 
175
        # FIXME: This is becoming very limiting. It must work against a complete grammar.
176
        fields = list(
177
            DocumentGrammar.create_default(document)
178
            .elements_by_type["REQUIREMENT"]
179
            .fields
180
        )
181
        field_titles = [f.title for f in fields]
182
        for _, name in extra_header_pairs:
183
            if name not in field_titles:
184
                fields.extend(
185
                    [
186
                        GrammarElementFieldString(
187
                            parent=None,
188
                            title=name,
189
                            human_title=None,
190
                            required="False",
191
                        ),
192
                    ]
193
                )
194
 
195
        requirements_element = GrammarElement(
196
            parent=None,
197
            tag="REQUIREMENT",
198
            property_is_composite="",
199
            property_prefix="",
200
            property_view_style="",
201
            fields=fields,
202
            relations=[],
203
        )
204
        elements = [requirements_element]
205
        grammar = DocumentGrammar(parent=document, elements=elements)
206
        document.grammar = grammar
207
        return document
208
 
209
    @staticmethod
210
    def create_requirement(
211
        row_values: List[str],
212
        document: SDocDocument,
213
        schema: ExcelSheetSchema,
214
    ) -> SDocNode:
215
        assert schema.statement_column_idx is not None
216
        statement = row_values[schema.statement_column_idx].strip()
217
        uid = None
218
        if schema.uid_column_idx is not None:
219
            uid = row_values[schema.uid_column_idx].strip()
220
        title = None
221
        if schema.title_column_idx is not None:
222
            title = row_values[schema.title_column_idx].strip()
223
        comments = None
224
        if schema.comment_column_idx is not None:
225
            comment = row_values[schema.comment_column_idx].strip()
226
            if comment in ("", "-"):
227
                comments = None
228
            else:
229
                comments = [ensure_newline(comment)]
230
        parent_uid = None
231
        if schema.parent_column_idx is not None:
232
            parent_uid = row_values[schema.parent_column_idx].strip()
233
            if len(parent_uid) == 0:
234
                parent_uid = None
235
 
236
        template_requirement = SDocObjectFactory.create_requirement(
237
            parent=document,
238
            node_type="REQUIREMENT",
239
            title=title,
240
            uid=uid,
241
            level=None,
242
            statement=None,
243
            statement_multiline=ensure_newline(statement),
244
            rationale=None,
245
            rationale_multiline=None,
246
            tags=None,
247
            comments=comments,
248
        )
249
        for column_idx_, column_name_ in schema.extra_header_pairs:
250
            # For a field that looks like a decimal (e.g. LEVEL: 3.1), it gets
251
            # treated as a float and then this fails without string casting.
252
            # Cast to string and strip whitespace.
253
            value = str(row_values[column_idx_]).strip()
254
            if value == "":
255
                continue
256
 
257
            # Only treat field as a multiline if necessary.
258
            # This fixed a weird bug where the LEVEL field included
259
            # a carriage return, which then got oddly scooped into
260
            # the headings in the HTML view.
261
            if schema.is_column_multiline(column_idx_):
262
                multiline = True
263
                field_value = ensure_newline(value)
264
            else:
265
                multiline = False
266
                field_value = value
267
            template_requirement.ordered_fields_lookup[column_name_] = [
268
                SDocNodeField.create_from_string(
269
                    parent=None,
270
                    field_name=column_name_,
271
                    field_value=field_value,
272
                    multiline=multiline,
273
                )
274
            ]
275
 
276
        if parent_uid is not None:
277
            reference = ParentReqReference(
278
                template_requirement, parent_uid, role=None
279
            )
280
            template_requirement.relations = [reference]
281
        requirement = SDocNode(
282
            parent=template_requirement.parent,
283
            node_type=template_requirement.node_type,
284
            fields=list(template_requirement.enumerate_fields()),
285
            relations=template_requirement.relations,
286
        )
287
 
288
        return requirement
1 - 288 | entire file