Path:
strictdoc/backend/sdoc_source_code/test_reports/junit_xml_reader.py
Lines:
584
Non-empty lines:
542
Non-empty lines covered with requirements:
542 / 542 (100.0%)
Functions:
9
Functions covered by requirements:
9 / 9 (100.0%)
1
"""2
@relation(SDOC-SRS-143, scope=file)3
"""4
5
import os
6
import re
7
from enum import IntEnum
8
from typing import Any, Dict, List, Optional, Set, Type
9
10
import bs4
11
from bs4 import BeautifulSoup
12
13
from strictdoc.backend.sdoc.document_reference import DocumentReference
14
from strictdoc.backend.sdoc.models.document import SDocDocument
15
from strictdoc.backend.sdoc.models.document_grammar import DocumentGrammar
16
from strictdoc.backend.sdoc.models.grammar_element import (
17
GrammarElementFieldString,
18
)19
from strictdoc.backend.sdoc.models.node import SDocNode, SDocNodeField
20
from strictdoc.backend.sdoc.models.reference import (
21
FileEntry,
22
FileEntryFormat,
23
FileReference,
24
)25
from strictdoc.core.file_system.file_tree import File
26
from strictdoc.core.project_config import ProjectConfig
27
from strictdoc.helpers.cast import assert_cast, assert_optional_cast
28
from strictdoc.helpers.file_system import file_open_read_utf8
29
from strictdoc.helpers.paths import path_to_posix_path
30
from strictdoc.helpers.string import is_uppercase_underscore_string
31
32
33
class JUnitXMLDialect(IntEnum):
34
LLVM_LIT = 1
35
CTEST = 2
36
GOOGLE_TEST = 3
37
PYTEST = 4
38
CARGO_NEXTEST = 5
39
40
@staticmethod41
def create_from_path(file_path: str) -> "JUnitXMLDialect":
42
if file_path.endswith(".lit.junit.xml"):
43
return JUnitXMLDialect.LLVM_LIT
44
if file_path.endswith(".ctest.junit.xml"):
45
return JUnitXMLDialect.CTEST
46
if file_path.endswith(".gtest.junit.xml"):
47
return JUnitXMLDialect.GOOGLE_TEST
48
if file_path.endswith(".pytest.junit.xml"):
49
return JUnitXMLDialect.PYTEST
50
if file_path.endswith(".nextest.junit.xml"):
51
return JUnitXMLDialect.CARGO_NEXTEST
52
raise NotImplementedError(file_path)
53
54
55
class JUnitXMLReader:
56
_SAFE_SDOC_FIELD_NAME_REGEX: re.Pattern[str] = re.compile(r"[^A-Za-z0-9]")
57
_TEST_RESULT_RESERVED_FIELD_NAMES: Set[str] = {
58
"UID",
59
"TEST_PATH",
60
"TEST_FUNCTION",
61
"DURATION",
62
"STATUS",
63
"TITLE",
64
"STATEMENT",
65
}66
67
@classmethod68
def read_from_file(
69
cls: Type["JUnitXMLReader"],
70
doc_file: File,
71
project_config: ProjectConfig,
72
) -> SDocDocument:
73
with file_open_read_utf8(doc_file.full_path) as file:
74
content = file.read()
75
return cls.read_from_string(content, doc_file, project_config)
76
77
@classmethod78
def read_from_string(
79
cls: Type["JUnitXMLReader"],
80
content: str,
81
doc_file: File,
82
project_config: ProjectConfig,
83
) -> SDocDocument:
84
if len(content) == 0:
85
raise RuntimeError(
86
"Document is empty, line 1, column 1 (<string>, line 1)"87
)88
try:
89
soup = BeautifulSoup(content, "xml")
90
91
except Exception as exception: # pylint: disable=broad-except
92
raise RuntimeError(str(exception)) from None
93
94
xml_format = JUnitXMLDialect.create_from_path(doc_file.full_path)
95
96
document = SDocDocument(
97
mid=None,
98
title="Test report",
99
config=None,
100
view=None,
101
grammar=None,
102
section_contents=[],
103
autogen=True,
104
)105
document.ng_including_document_reference = DocumentReference()
106
xml_testsuite_list: List[bs4.element.Tag]
107
108
xml_testsuites: Optional[bs4.element.Tag] = assert_optional_cast(
109
soup.find("testsuites", recursive=False), bs4.element.Tag
110
)111
112
# Some tools, e.g., LLVM LIT, produce JUnit XML files with a top-level113
# <testsuites> tag which allows containing multiple <testsuite> tags.114
# Some tools, e.g., Google Test, produce only a single <testsuite>.115
if xml_testsuites is not None:
116
assert isinstance(xml_testsuites, bs4.element.Tag)
117
xml_testsuite_list = xml_testsuites.find_all(
118
"testsuite", recursive=False
119
)120
else:
121
xml_testsuite: bs4.element.Tag = assert_cast(
122
soup.find("testsuite", recursive=False), bs4.element.Tag
123
)124
xml_testsuite_list = [xml_testsuite]
125
if len(xml_testsuite_list) == 1:
126
assert isinstance(xml_testsuite_list[0], bs4.element.Tag)
127
document.title = "Test report: " + assert_cast(
128
xml_testsuite_list[0]["name"], str
129
)130
131
property_field_names: Dict[str, str] = (
132
cls._collect_property_field_names(xml_testsuite_list)
133
)134
additional_fields: List[GrammarElementFieldString] = [
135
GrammarElementFieldString(
136
parent=None,
137
title=field_name_,
138
human_title=property_name_,
139
required="False",
140
)141
for property_name_, field_name_ in property_field_names.items()
142
]143
grammar = DocumentGrammar.create_for_test_report(
144
document, additional_fields=additional_fields
145
)146
document.grammar = grammar
147
document.config.requirement_style = "Table"
148
149
for xml_testsuite_ in xml_testsuite_list:
150
assert isinstance(xml_testsuite_, bs4.element.Tag)
151
title = assert_cast(xml_testsuite_["name"], str)
152
total_tests: int = int(str(xml_testsuite_["tests"]))
153
total_failures: int = int(str(xml_testsuite_["failures"]))
154
# The "skipped" attribute is optional on <testsuite>. cargo-nextest155
# for instance does not emit it.156
total_skipped: int = int(str(xml_testsuite_.get("skipped", "0")))
157
total_success: int = total_tests - total_failures - total_skipped
158
159
test_suite_section = SDocNode.create_section(
160
parent=document,
161
document=document,
162
title=title,
163
)164
document.section_contents.append(test_suite_section)
165
166
summary_table = f"""\
167
.. list-table:: Test suite summary168
:widths: 25 10169
:header-rows: 0170
171
* - **Number of tests:**172
- {total_tests}
173
* - **Number of successful tests:**174
- {total_success}
175
* - **Number of failed tests:**176
- {total_failures}
177
* - **Number of skipped tests:**178
- {total_skipped}
179
"""180
181
testcase_node = SDocNode(
182
parent=test_suite_section,
183
node_type="TEXT",
184
fields=[],
185
relations=[],
186
)187
testcase_node.ng_document_reference = DocumentReference()
188
testcase_node.ng_document_reference.set_document(document)
189
testcase_node.ng_including_document_reference = DocumentReference()
190
testcase_node.set_field_value(
191
field_name="STATEMENT",
192
form_field_index=0,
193
value=SDocNodeField(
194
parent=testcase_node,
195
field_name="STATEMENT",
196
parts=[summary_table],
197
multiline__="True",
198
),199
)200
test_suite_section.section_contents.append(testcase_node)
201
202
#203
# Parse individual <testcase> elements.204
#205
206
xml_testcase_list: List[Any] = xml_testsuite_.find_all(
207
"testcase", recursive=False
208
)209
for xml_testcase_ in xml_testcase_list:
210
assert isinstance(xml_testcase_, bs4.element.Tag)
211
212
test_case_node_uid: str
213
test_case_node_title: str
214
test_case_node_status: str = "PASSED"
215
test_case_node_test_path: Optional[str] = None
216
test_case_node_test_function: Optional[str] = None
217
218
xml_testcase_name: str = assert_cast(xml_testcase_["name"], str)
219
xml_testcase_classname: str = assert_cast(
220
xml_testcase_["classname"], str
221
)222
xml_testcase_time: str = assert_cast(xml_testcase_["time"], str)
223
xml_error_or_none: Optional[bs4.element.Tag] = (
224
assert_optional_cast(
225
xml_testcase_.find("error"), bs4.element.Tag
226
)227
)228
xml_failure_or_none: Optional[bs4.element.Tag] = (
229
assert_optional_cast(
230
xml_testcase_.find("failure"), bs4.element.Tag
231
)232
)233
xml_skipped_or_none: Optional[bs4.element.Tag] = (
234
assert_optional_cast(
235
xml_testcase_.find("skipped"), bs4.element.Tag
236
)237
)238
239
if xml_error_or_none is not None:
240
raise RuntimeError(
241
"JUnit XML contains a test that failed due to an error: "242
f"{xml_testcase_name}"
243
)244
245
if xml_failure_or_none is not None:
246
test_case_node_status = "FAILED"
247
elif xml_skipped_or_none is not None:
248
test_case_node_status = "SKIPPED"
249
250
#251
# Different tools produce different outputs when it comes to how252
# the test names and paths are stored. Each tool's output is253
# handled separately below.254
#255
if xml_format == JUnitXMLDialect.LLVM_LIT:
256
#257
# Example produced by LLVM LIT:258
# <testcase classname="StrictDoc integration tests.tests/integration" name="test.ignored.itest" time="5.50"/>259
#260
261
rel_path_to_test_suite = (
262
project_config.test_report_root_dict.get(
263
doc_file.rel_path.relative_path_posix, None
264
)265
)266
267
if rel_path_to_test_suite is None:
268
raise RuntimeError(
269
"The relative path to the test suite must be "270
"registered in the strictdoc.toml config under the "271
"'test_report_root_dict' option: "272
f"{doc_file.rel_path.relative_path_posix}"
273
)274
275
# Relative path to test is a combination of the classname and name,276
# but we must remove the test suite name prefix.277
rel_path_to_test: str = os.path.join(
278
xml_testcase_classname, xml_testcase_name
279
)280
_, _, rel_path_to_test = rel_path_to_test.partition(".")
281
282
rel_path_to_test = os.path.join(
283
rel_path_to_test_suite, rel_path_to_test
284
)285
rel_path_to_test = path_to_posix_path(rel_path_to_test)
286
287
assert project_config.source_root_path is not None
288
full_path_to_test = os.path.join(
289
project_config.source_root_path, rel_path_to_test
290
)291
assert os.path.isfile(full_path_to_test), full_path_to_test
292
293
test_case_node_uid = rel_path_to_test
294
test_case_node_title = rel_path_to_test
295
test_case_node_duration = xml_testcase_time
296
test_case_node_test_path = rel_path_to_test
297
elif xml_format == JUnitXMLDialect.CTEST:
298
test_case_node_uid = xml_testcase_name
299
test_case_node_title = xml_testcase_name
300
test_case_node_duration = xml_testcase_time
301
test_case_node_test_function = "#GTEST#" + xml_testcase_name
302
elif xml_format == JUnitXMLDialect.GOOGLE_TEST:
303
google_test_name = (
304
xml_testcase_classname + "." + xml_testcase_name
305
)306
test_case_node_uid = google_test_name
307
test_case_node_title = google_test_name
308
test_case_node_duration = xml_testcase_time
309
test_case_node_test_function = "#GTEST#" + google_test_name
310
elif xml_format == JUnitXMLDialect.CARGO_NEXTEST:
311
# cargo-nextest records no source location, so the312
# classname and name are forwarded as a "#NEXTEST#" marker313
# for FileTraceabilityIndex to resolve by forward lookup.314
# See the cargo-nextest section of the user guide and315
# convert_nextest_test_to_rust_canonical_paths() for the316
# JUnit-XML layout and resolution rules.317
nextest_full_name = (
318
xml_testcase_classname + "::" + xml_testcase_name
319
)320
test_case_node_uid = nextest_full_name
321
test_case_node_title = nextest_full_name
322
test_case_node_duration = xml_testcase_time
323
test_case_node_test_function = (
324
"#NEXTEST#"325
+ xml_testcase_classname
326
+ "|"
327
+ xml_testcase_name
328
)329
elif xml_format == JUnitXMLDialect.PYTEST:
330
xml_testcase_path_parts = xml_testcase_classname
331
xml_testcase_class = ""
332
# Heuristic: if last part of the classname attribute starts with "Test" and is333
# in CamelCase, then we assume it is the test class name:334
# - Remove it from the path.335
# - Prepend it to the function name.336
parts = xml_testcase_classname.split(".")
337
if (
338
parts339
and parts[-1].startswith("Test")
340
and parts[-1][1:].lower() != parts[-1][1:]
341
):342
xml_testcase_path_parts = ".".join(parts[:-1])
343
xml_testcase_class = parts[-1] + "."
344
test_case_node_uid = (
345
xml_testcase_classname + "." + xml_testcase_name
346
)347
test_case_node_duration = xml_testcase_time
348
test_case_node_test_path = (
349
xml_testcase_path_parts.replace(".", os.path.sep)
350
+ ".py"
351
)352
test_case_node_title = xml_testcase_classname
353
test_case_node_test_function = (
354
xml_testcase_class + xml_testcase_name
355
)356
else:
357
raise NotImplementedError("Unsupported JUnit XML format")
358
359
testcase_node = SDocNode(
360
parent=test_suite_section,
361
node_type="TEST_RESULT",
362
fields=[],
363
relations=[],
364
)365
testcase_node.ng_document_reference = DocumentReference()
366
testcase_node.ng_document_reference.set_document(document)
367
testcase_node.ng_including_document_reference = (
368
DocumentReference()
369
)370
if xml_skipped_or_none is None:
371
testcase_node.set_field_value(
372
field_name="UID",
373
form_field_index=0,
374
value=SDocNodeField(
375
parent=testcase_node,
376
field_name="UID",
377
parts=[test_case_node_uid],
378
multiline__=None,
379
),380
)381
if test_case_node_test_path is not None:
382
testcase_node.set_field_value(
383
field_name="TEST_PATH",
384
form_field_index=0,
385
value=SDocNodeField(
386
parent=testcase_node,
387
field_name="TEST_PATH",
388
parts=[
389
path_to_posix_path(test_case_node_test_path)
390
],391
multiline__=None,
392
),393
)394
if test_case_node_test_function is not None:
395
testcase_node.set_field_value(
396
field_name="TEST_FUNCTION",
397
form_field_index=0,
398
value=SDocNodeField(
399
parent=testcase_node,
400
field_name="TEST_FUNCTION",
401
parts=[test_case_node_test_function],
402
multiline__=None,
403
),404
)405
testcase_node.set_field_value(
406
field_name="DURATION",
407
form_field_index=0,
408
value=SDocNodeField(
409
parent=testcase_node,
410
field_name="DURATION",
411
parts=[test_case_node_duration],
412
multiline__=None,
413
),414
)415
testcase_node.set_field_value(
416
field_name="STATUS",
417
form_field_index=0,
418
value=SDocNodeField(
419
parent=testcase_node,
420
field_name="STATUS",
421
parts=[test_case_node_status],
422
multiline__=None,
423
),424
)425
for xml_property_ in cls._get_xml_property_list(xml_testcase_):
426
property_name: str = cls._get_xml_property_attribute(
427
xml_property_, "name"
428
)429
property_value: str = cls._get_xml_property_attribute(
430
xml_property_, "value"
431
)432
property_field_name: str = property_field_names[
433
property_name434
]435
testcase_node.set_field_value(
436
field_name=property_field_name,
437
form_field_index=0,
438
value=SDocNodeField(
439
parent=testcase_node,
440
field_name=property_field_name,
441
parts=[property_value],
442
multiline__=None,
443
),444
)445
testcase_node.set_field_value(
446
field_name="TITLE",
447
form_field_index=0,
448
value=SDocNodeField(
449
parent=testcase_node,
450
field_name="TITLE",
451
parts=[test_case_node_title],
452
multiline__=None,
453
),454
)455
if xml_skipped_or_none is None:
456
if test_case_node_test_path is not None:
457
testcase_node.relations.append(
458
FileReference(
459
parent=testcase_node,
460
g_file_entry=FileEntry(
461
parent=None,
462
g_file_format=FileEntryFormat.SOURCECODE,
463
g_file_path=test_case_node_test_path,
464
g_line_range=None,
465
element="function",
466
id=test_case_node_test_function,
467
),468
)469
)470
elif test_case_node_test_function is not None:
471
testcase_node.relations.append(
472
FileReference(
473
parent=testcase_node,
474
g_file_entry=FileEntry(
475
parent=None,
476
g_file_format=FileEntryFormat.SOURCECODE,
477
g_file_path="#FORWARD#",
478
g_line_range=None,
479
element="function",
480
id=test_case_node_test_function,
481
),482
)483
)484
test_suite_section.section_contents.append(testcase_node)
485
486
return document
487
488
@classmethod489
def _collect_property_field_names(
490
cls: Type["JUnitXMLReader"],
491
xml_testsuite_list: List[bs4.element.Tag],
492
) -> Dict[str, str]:
493
property_field_names: Dict[str, str] = {}
494
property_names_by_field_name: Dict[str, str] = {}
495
496
for xml_testsuite_ in xml_testsuite_list:
497
xml_testcase_list: List[Any] = xml_testsuite_.find_all(
498
"testcase", recursive=False
499
)500
for xml_testcase_ in xml_testcase_list:
501
assert isinstance(xml_testcase_, bs4.element.Tag)
502
property_names_for_testcase: Set[str] = set()
503
for xml_property_ in cls._get_xml_property_list(xml_testcase_):
504
property_name: str = cls._get_xml_property_attribute(
505
xml_property_, "name"
506
)507
if property_name in property_names_for_testcase:
508
raise RuntimeError(
509
"JUnit testcase contains duplicate property name: "510
f"{property_name!r}."
511
)512
property_names_for_testcase.add(property_name)
513
field_name: str = cls._create_sdoc_field_name(property_name)
514
if not is_uppercase_underscore_string(field_name):
515
raise RuntimeError(
516
"JUnit property name cannot be mapped to a "517
"StrictDoc field name: "518
f"{property_name!r}."
519
)520
if field_name in cls._TEST_RESULT_RESERVED_FIELD_NAMES:
521
raise RuntimeError(
522
"JUnit property name maps to a reserved "523
"TEST_RESULT field: "524
f"{property_name!r} -> {field_name!r}."
525
)526
existing_property_name: Optional[str] = (
527
property_names_by_field_name.get(field_name)
528
)529
if (
530
existing_property_name is not None
531
and existing_property_name != property_name
532
):533
raise RuntimeError(
534
"JUnit property names map to the same StrictDoc "535
"field: "536
f"{existing_property_name!r}, {property_name!r} "
537
f"-> {field_name!r}."
538
)539
property_names_by_field_name[field_name] = property_name
540
property_field_names[property_name] = field_name
541
542
return property_field_names
543
544
@staticmethod545
def _get_xml_property_list(
546
xml_testcase: bs4.element.Tag,
547
) -> List[bs4.element.Tag]:
548
xml_properties_or_none: Optional[bs4.element.Tag] = (
549
assert_optional_cast(
550
xml_testcase.find("properties", recursive=False),
551
bs4.element.Tag,
552
)553
)554
if xml_properties_or_none is None:
555
return []
556
return [
557
assert_cast(xml_property_, bs4.element.Tag)
558
for xml_property_ in xml_properties_or_none.find_all(
559
"property", recursive=False
560
)561
]562
563
@staticmethod564
def _get_xml_property_attribute(
565
xml_property: bs4.element.Tag, attribute_name: str
566
) -> str:
567
attribute_value: Any = xml_property.get(attribute_name)
568
if attribute_value is None:
569
raise RuntimeError(
570
"JUnit property is missing required "571
f"{attribute_name!r} attribute."
572
)573
if not isinstance(attribute_value, str):
574
raise RuntimeError(
575
"JUnit property attribute must be a string: "576
f"{attribute_name!r}."
577
)578
return attribute_value
579
580
@classmethod581
def _create_sdoc_field_name(
582
cls: Type["JUnitXMLReader"], property_name: str
583
) -> str:
584
return cls._SAFE_SDOC_FIELD_NAME_REGEX.sub("_", property_name).upper()