Path:
strictdoc/features/tree_map/generator.py
Lines:
595
Non-empty lines:
510
Non-empty lines covered with requirements:
510 / 510 (100.0%)
Functions:
17
Functions covered by requirements:
17 / 17 (100.0%)
1
"""2
Generate HTML graphs with documentation tree information.3
4
Uses Plotly.js for generating tree map graphs.5
6
@relation(SDOC-SRS-157, scope=file)7
"""8
9
import os
10
import textwrap
11
from copy import deepcopy
12
from dataclasses import dataclass
13
from typing import Any, Dict, List, Optional, Union
14
15
import pandas as pd
16
import plotly.express as px
17
import plotly.io as pio
18
19
from strictdoc.backend.sdoc.models.document import SDocDocument
20
from strictdoc.backend.sdoc.models.node import SDocNode
21
from strictdoc.core.document_iterator import SDocDocumentIterator
22
from strictdoc.core.project_config import ProjectConfig
23
from strictdoc.core.traceability_index import TraceabilityIndex
24
from strictdoc.export.html.document_type import DocumentType
25
from strictdoc.export.html.html_templates import HTMLTemplates
26
from strictdoc.export.html.renderers.link_renderer import LinkRenderer
27
from strictdoc.features.tree_map.helpers import get_color
28
from strictdoc.features.tree_map.view_object import TreeMapViewObject
29
from strictdoc.helpers.timing import timing_decorator
30
31
32
@dataclass33
class GraphSection:
34
title: str
35
description: str
36
graph_content: str
37
38
def get_html(self) -> str:
39
return f"""
40
<h2 class="section">{self.title}</h2>
41
42
<p class="section_description">{self.description}</p>
43
44
{self.graph_content}
45
"""46
47
48
@dataclass49
class NodeStats:
50
child_nodes: int = 0
51
child_nodes_with_links_to_source_files: int = 0
52
child_nodes_with_links_to_test_files: int = 0
53
54
@staticmethod55
def create_child_node_without_stats() -> "NodeStats":
56
return NodeStats(
57
child_nodes=0,
58
child_nodes_with_links_to_source_files=0,
59
child_nodes_with_links_to_test_files=0,
60
)61
62
def add_child_stats(self, child_node_stats: "NodeStats") -> None:
63
self.child_nodes += child_node_stats.child_nodes
64
self.child_nodes_with_links_to_source_files += (
65
child_node_stats.child_nodes_with_links_to_source_files
66
)67
self.child_nodes_with_links_to_test_files += (
68
child_node_stats.child_nodes_with_links_to_test_files
69
)70
71
def get_code_coverage_ratio(self) -> float:
72
covered = self.child_nodes_with_links_to_source_files
73
total = self.child_nodes
74
ratio = max(0.0, min(1.0, covered / total))
75
return ratio
76
77
def get_test_coverage_ratio(self) -> float:
78
covered = self.child_nodes_with_links_to_test_files
79
total = self.child_nodes
80
ratio = max(0.0, min(1.0, covered / total))
81
return ratio
82
83
84
class PlotlyDataFrameColumn:
85
COLOR_SOURCE = "_COLOR_SOURCE"
86
COLOR_TEST = "_COLOR_TEST"
87
LEVEL = "_LEVEL"
88
PARENT_MID = "_PARENT_MID"
89
WEIGHT = "_WEIGHT"
90
IS_NORMATIVE = "_IS_NORMATIVE"
91
92
93
# Without this, Plotly shrinks each box's title to fit, down to94
# sub-pixel sizes that render as an illegible blur on small boxes.95
# uniformtext instead forces one font size across the whole chart and96
# hides labels that don't fit at "minsize", so titles are always97
# legible or absent.98
# minsize is a tuned tradeoff: too low brings back blur, too high99
# hides titles on more (still readable) boxes.100
TREE_MAP_UNIFORMTEXT = {"minsize": 10, "mode": "hide"}
101
102
103
class TreeMapGenerator:
104
@staticmethod105
@timing_decorator("Export tree map visualizations")
106
def export(
107
project_config: ProjectConfig,
108
traceability_index: TraceabilityIndex,
109
html_templates: HTMLTemplates,
110
) -> None:
111
data = []
112
113
for document_ in traceability_index.document_tree.document_list:
114
assert document_.meta is not None
115
if traceability_index.file_dependency_manager.must_generate(
116
document_.meta.output_document_full_path
117
):118
break119
else:
120
print( # noqa: T201
121
"All documents are up-to-date. "122
"Skipping the generation of the tree map screen."123
)124
return125
126
documents_with_requirements = set()
127
128
map_node_to_coverage: Dict[
129
Union[SDocNode, SDocDocument], NodeStats
130
] = {}
131
132
link_renderer = LinkRenderer(
133
root_path="", static_path=project_config.dir_for_sdoc_assets
134
)135
136
def create_node_link(node__: Union[SDocDocument, SDocNode]) -> str:
137
href = link_renderer.render_node_link(
138
node__,
139
context_document=None,
140
document_type=DocumentType.DOCUMENT,
141
)142
return f'<a href="{href}">Open in document</a>'
143
144
def get_node_stats(
145
node_: Union[SDocNode, SDocDocument],
146
) -> NodeStats:
147
if node_ in map_node_to_coverage:
148
return map_node_to_coverage[node_]
149
150
if not node_.section_contents:
151
if (
152
not isinstance(node_, SDocNode)
153
or not node_.is_normative_node()
154
or node_.reserved_uid is None
155
):156
return NodeStats.create_child_node_without_stats()
157
158
node_stats = NodeStats(child_nodes=1)
159
160
children = traceability_index.get_children_requirements(node_)
161
162
has_children_all_covered_with_code = len(children) > 0
163
has_children_all_covered_with_test = len(children) > 0
164
165
for child_node_ in children:
166
child_node_stats = get_node_stats(child_node_)
167
if (
168
child_node_stats.child_nodes_with_links_to_source_files
169
== 0
170
):171
has_children_all_covered_with_code = False
172
173
if (
174
child_node_stats.child_nodes_with_links_to_test_files
175
== 0
176
):177
has_children_all_covered_with_test = False
178
179
node_stats.child_nodes_with_links_to_source_files = int(
180
has_children_all_covered_with_code181
)182
node_stats.child_nodes_with_links_to_test_files = int(
183
has_children_all_covered_with_test184
)185
186
source_files = traceability_index.get_file_traceability_index().get_requirement_file_links(
187
node_188
)189
for source_file_tuple_ in source_files:
190
if "tests/" in source_file_tuple_[0]:
191
node_stats.child_nodes_with_links_to_test_files = 1
192
else:
193
node_stats.child_nodes_with_links_to_source_files = 1
194
195
return node_stats
196
197
this_node_counters = NodeStats()
198
199
for sub_node_ in node_.section_contents:
200
if not isinstance(sub_node_, SDocNode):
201
continue202
if sub_node_.node_type == "TEXT":
203
continue204
sub_node_stats = get_node_stats(sub_node_)
205
map_node_to_coverage[sub_node_] = sub_node_stats
206
207
this_node_counters.add_child_stats(sub_node_stats)
208
209
return this_node_counters
210
211
for document_ in traceability_index.document_tree.document_list:
212
if document_.document_is_included():
213
continue214
215
map_node_to_coverage[document_] = get_node_stats(document_)
216
217
document_iterator = SDocDocumentIterator(document_)
218
for node_, _ in document_iterator.all_content(
219
print_fragments=False
220
):221
if not isinstance(node_, SDocNode):
222
continue223
224
if node_.is_normative_node():
225
documents_with_requirements.add(document_)
226
map_node_to_coverage[node_] = get_node_stats(node_)
227
228
root_node_title = project_config.project_title
229
230
for document_ in traceability_index.document_tree.document_list:
231
if document_.document_is_included():
232
continue233
234
color_code = "white"
235
color_test = "white"
236
237
if document_ in documents_with_requirements:
238
node_stats = get_node_stats(document_)
239
if node_stats.child_nodes > 0:
240
color_code = get_color(node_stats.get_code_coverage_ratio())
241
color_test = get_color(node_stats.get_test_coverage_ratio())
242
243
document_total_size = document_.get_total_size()[0]
244
document_normative_total_size = document_.get_total_size()[1]
245
246
title = document_.reserved_title
247
title_normative = title
248
title += " (" + str(document_total_size) + ")"
249
title_normative += " (" + str(document_normative_total_size) + ")"
250
251
data.append(
252
{253
PlotlyDataFrameColumn.WEIGHT: document_total_size
254
if document_total_size > 10
255
else 10,
256
"MID": document_.reserved_mid,
257
"UID": document_.reserved_uid
258
if document_.reserved_uid is not None
259
else "",
260
"TITLE": title,
261
"TITLE_NORMATIVE": title_normative,
262
"STATEMENT": " (" + str(document_total_size) + ")",
263
"_LINK": create_node_link(document_),
264
PlotlyDataFrameColumn.LEVEL: 0,
265
PlotlyDataFrameColumn.PARENT_MID: root_node_title,
266
PlotlyDataFrameColumn.COLOR_SOURCE: color_code,
267
PlotlyDataFrameColumn.COLOR_TEST: color_test,
268
PlotlyDataFrameColumn.IS_NORMATIVE: document_
269
in documents_with_requirements,
270
},271
)272
273
document_iterator = SDocDocumentIterator(document_)
274
for node, context_ in document_iterator.all_content(
275
print_fragments=False
276
):277
if not isinstance(node, SDocNode):
278
continue279
280
is_normative = node.is_normative_node() or (
281
node.node_type == "SECTION" and node.ng_has_requirements
282
)283
284
node_total_size = node.get_total_size()[0]
285
node_normative_total_size = node.get_total_size()[1]
286
287
parent_mid = node.parent.reserved_mid
288
289
if node.reserved_title is not None:
290
title = node.reserved_title
291
else:
292
title = "[TEXT] node"
293
title_normative = title
294
295
if (
296
node.section_contents is not None
297
and len(node.section_contents) > 0
298
):299
title += " (" + str(node_total_size) + ")"
300
title_normative += (
301
" (" + str(node_normative_total_size) + ")"
302
)303
304
statement = (
305
node.reserved_statement
306
if node.reserved_statement is not None
307
else ""
308
)309
if len(statement) > 120:
310
statement = statement[:120] + "..."
311
312
color_code = "white"
313
color_test = "white"
314
315
if (
316
node.node_type != "TEXT"
317
and document_ in documents_with_requirements
318
):319
if document_ in documents_with_requirements:
320
node_stats = get_node_stats(node)
321
if node_stats.child_nodes > 0:
322
color_code = get_color(
323
node_stats.get_code_coverage_ratio()
324
)325
color_test = get_color(
326
node_stats.get_test_coverage_ratio()
327
)328
329
data.append(
330
{331
PlotlyDataFrameColumn.WEIGHT: node_total_size
332
if node_total_size > 10
333
else 10,
334
"MID": node.reserved_mid,
335
"UID": node.reserved_uid
336
if node.reserved_uid is not None
337
else "",
338
"TITLE": title,
339
"TITLE_NORMATIVE": title_normative,
340
"STATEMENT": statement,
341
"_LINK": create_node_link(
342
node,
343
),344
PlotlyDataFrameColumn.PARENT_MID: parent_mid,
345
PlotlyDataFrameColumn.LEVEL: context_.get_level(),
346
PlotlyDataFrameColumn.COLOR_SOURCE: color_code,
347
PlotlyDataFrameColumn.COLOR_TEST: color_test,
348
PlotlyDataFrameColumn.IS_NORMATIVE: is_normative,
349
},350
)351
352
root_row = {
353
"MID": root_node_title,
354
"UID": "",
355
"TITLE": root_node_title,
356
"TITLE_NORMATIVE": root_node_title,
357
"STATEMENT": "",
358
PlotlyDataFrameColumn.WEIGHT: 0,
359
"_SHORT_LABEL": "",
360
"_LONG_LABEL": "",
361
"_IS_LEAF": False,
362
"_LINK": "",
363
PlotlyDataFrameColumn.PARENT_MID: "",
364
PlotlyDataFrameColumn.LEVEL: 0,
365
PlotlyDataFrameColumn.COLOR_SOURCE: "white",
366
PlotlyDataFrameColumn.COLOR_TEST: "white",
367
PlotlyDataFrameColumn.IS_NORMATIVE: True,
368
}369
data.append(root_row)
370
371
df = pd.DataFrame(data)
372
373
df["_IS_LEAF"] = ~df["MID"].isin(df[PlotlyDataFrameColumn.PARENT_MID])
374
375
def short_label(row_: Any) -> Any:
376
return row_["TITLE"]
377
378
def short_label_normative(row_: Any) -> Any:
379
return row_["TITLE_NORMATIVE"]
380
381
df["_SHORT_LABEL"] = df.apply(short_label, axis=1)
382
df["_SHORT_LABEL_NORMATIVE"] = df.apply(short_label_normative, axis=1)
383
384
def wrap_text(text: Optional[str], width: int = 80) -> str:
385
if not text:
386
return ""
387
return "<br>".join(textwrap.wrap(text, width=width))
388
389
def long_or_short_label(row_: Any) -> Any:
390
if row_["_IS_LEAF"] and row_["STATEMENT"]:
391
statement = row_["STATEMENT"]
392
statement = statement.replace("\n", "<br>")
393
statement = wrap_text(statement, 80)
394
395
return f"<b>{row_['TITLE']}</b><br><br>{statement}<br><br>{row_['_LINK']}"
396
title = row_["TITLE"]
397
if len(title) > 30:
398
title = (
399
"<br>".join(title.split(" ")) + f"<br><br>{row_['_LINK']}"
400
)401
return title
402
403
df["_LONG_LABEL"] = df.apply(long_or_short_label, axis=1)
404
405
def hover_(row_: Any) -> Any:
406
mid = row_["MID"]
407
uid = row_["UID"]
408
title = row_["TITLE"]
409
statement = row_["STATEMENT"]
410
411
return f"""\
412
<b>MID</b>: {mid}<br>
413
<b>UID</b>: {uid}<br>
414
<b>TITLE</b>: {title}<br>
415
<b>STATEMENT</b>: {statement}<br>
416
"""417
418
df["_HOVER"] = df.apply(hover_, axis=1)
419
420
parts: List[GraphSection] = []
421
422
# FIGURE: Document tree map.423
fig = px.treemap(
424
df,
425
names="_SHORT_LABEL",
426
ids="MID",
427
parents=PlotlyDataFrameColumn.PARENT_MID,
428
values=PlotlyDataFrameColumn.WEIGHT,
429
custom_data=[
430
"_HOVER",
431
"_SHORT_LABEL",
432
"_LONG_LABEL",
433
"_IS_LEAF",
434
PlotlyDataFrameColumn.LEVEL,
435
],436
)437
fig.update_layout(
438
margin={"t": 25, "l": 25, "r": 25, "b": 25},
439
height=800,
440
uniformtext=TREE_MAP_UNIFORMTEXT,
441
)442
fig.update_traces(
443
root_color="lightgray",
444
marker={
445
"colors": ["white"] * len(df),
446
},447
marker_line_color="#ddd",
448
marker_line_width=1,
449
textposition="middle center",
450
texttemplate="%{customdata[0]}",
451
hoverinfo="skip",
452
hovertemplate=None,
453
)454
parts.append(
455
GraphSection(
456
title="Document tree map",
457
description="""\
458
This is a general representation of a document tree. All nodes are included,459
both normative (e.g., REQUIREMENT) and non-normative (e.g., TEXT). The numbers460
indicate how many nodes each section or node contains.461
""",
462
graph_content=pio.to_html(
463
fig,
464
full_html=False,
465
include_plotlyjs=True,
466
),467
)468
)469
470
# FIGURE: Document tree: Requirements coverage with source.471
df = df[df[PlotlyDataFrameColumn.IS_NORMATIVE]]
472
fig = px.treemap(
473
df,
474
names="_SHORT_LABEL_NORMATIVE",
475
ids="MID",
476
parents=PlotlyDataFrameColumn.PARENT_MID,
477
values=PlotlyDataFrameColumn.WEIGHT,
478
custom_data=[
479
"_HOVER",
480
"_SHORT_LABEL",
481
"_LONG_LABEL",
482
"_IS_LEAF",
483
PlotlyDataFrameColumn.LEVEL,
484
],485
)486
fig.update_layout(
487
margin={"t": 25, "l": 25, "r": 25, "b": 25},
488
height=800,
489
uniformtext=TREE_MAP_UNIFORMTEXT,
490
)491
fig.update_traces(
492
root_color="lightgray",
493
marker={
494
"colors": df[PlotlyDataFrameColumn.COLOR_SOURCE],
495
},496
marker_line_color="#ddd",
497
marker_line_width=1,
498
textposition="middle center",
499
texttemplate="%{customdata[0]}",
500
hoverinfo="skip",
501
hovertemplate=None,
502
)503
parts.append(
504
GraphSection(
505
title="Requirements coverage with source",
506
description="""\
507
This graph shows which requirements are covered by at least one source file.508
A requirement is also considered covered if it has child requirements that are509
themselves covered by source files.510
511
<ul>512
<li>513
<span style="background-color: #AAFFAA;">Green</span> –514
Requirement/section is fully covered with one or more source file.515
</li>516
<li>517
<span style="background-color: #FFFFAA;">Yellow</span> –518
Section is partially covered with one or more source file.519
</li>520
<li>521
<span style="background-color: #FFAAAA;">Red</span> –522
Requirement/section is not covered by any source files.523
</li>524
</ul>525
""",
526
graph_content=pio.to_html(
527
fig,
528
full_html=False,
529
include_plotlyjs=False,
530
),531
)532
)533
534
# FIGURE: Document tree: Requirements coverage with test.535
fig = deepcopy(fig)
536
fig.update_traces(
537
root_color="lightgray",
538
marker={
539
"colors": df[PlotlyDataFrameColumn.COLOR_TEST],
540
},541
marker_line_color="#ddd",
542
marker_line_width=1,
543
textposition="middle center",
544
texttemplate="%{customdata[0]}",
545
hoverinfo="skip",
546
hovertemplate=None,
547
)548
parts.append(
549
GraphSection(
550
title="Requirements coverage with test",
551
description="""\
552
This graph shows which requirements are covered by at least one test.553
A requirement is also considered covered if it has child requirements that are554
themselves covered by tests. A source file is considered a test file if its path555
contains "tests/".556
557
<ul>558
<li>559
<span style="background-color: #AAFFAA;">Green</span> –560
Requirement/section is fully covered with one or more tests.561
</li>562
<li>563
<span style="background-color: #FFFFAA;">Yellow</span> –564
Section is partially covered with one or more test.565
</li>566
<li>567
<span style="background-color: #FFAAAA;">Red</span> –568
Requirement/section is not covered with by any tests.569
</li>570
</ul>571
""",
572
graph_content=pio.to_html(
573
fig,
574
full_html=False,
575
include_plotlyjs=False,
576
),577
)578
)579
580
body = "".join(part_.get_html() for part_ in parts)
581
582
view_object = TreeMapViewObject(
583
traceability_index=traceability_index,
584
project_config=project_config,
585
body=body,
586
)587
html = view_object.render_screen(html_templates.jinja_environment())
588
589
output_html = os.path.join(
590
project_config.export_output_html_root,
591
"tree_map.html",
592
)593
594
with open(output_html, "w", encoding="utf-8") as file_:
595
file_.write(html)