Path:
strictdoc/features/tree_map/generator.py
Lines:
594
Non-empty lines:
511
Non-empty lines covered with requirements:
511 / 511 (100.0%)
Functions:
17
Functions covered by requirements:
17 / 17 (100.0%)
1
"""2
Generate HTML graphs with documentation tree information.3
4
Uses Plotly.js for generating tree map graphs.5
6
@relation(SDOC-SRS-157, scope=file)7
"""8
9
import os
10
import textwrap
11
from copy import deepcopy
12
from dataclasses import dataclass
13
from typing import Any, Dict, List, Optional, Union
14
15
import pandas as pd
16
import plotly.express as px
17
import plotly.io as pio
18
19
from strictdoc.backend.sdoc.models.document import SDocDocument
20
from strictdoc.backend.sdoc.models.node import SDocNode
21
from strictdoc.core.document_iterator import SDocDocumentIterator
22
from strictdoc.core.project_config import ProjectConfig
23
from strictdoc.core.traceability_index import TraceabilityIndex
24
from strictdoc.export.html.document_type import DocumentType
25
from strictdoc.export.html.html_templates import HTMLTemplates
26
from strictdoc.export.html.renderers.link_renderer import LinkRenderer
27
from strictdoc.features.tree_map.helpers import (
28
get_color,
29
split_into_max_n_lines,
30
)31
from strictdoc.features.tree_map.view_object import TreeMapViewObject
32
from strictdoc.helpers.timing import timing_decorator
33
34
35
@dataclass36
class GraphSection:
37
title: str
38
description: str
39
graph_content: str
40
41
def get_html(self) -> str:
42
return f"""
43
<h2 class="section">{self.title}</h2>
44
45
<p class="section_description">{self.description}</p>
46
47
{self.graph_content}
48
"""49
50
51
@dataclass52
class NodeStats:
53
child_nodes: int = 0
54
child_nodes_with_links_to_source_files: int = 0
55
child_nodes_with_links_to_test_files: int = 0
56
57
@staticmethod58
def create_child_node_without_stats() -> "NodeStats":
59
return NodeStats(
60
child_nodes=0,
61
child_nodes_with_links_to_source_files=0,
62
child_nodes_with_links_to_test_files=0,
63
)64
65
def add_child_stats(self, child_node_stats: "NodeStats") -> None:
66
self.child_nodes += child_node_stats.child_nodes
67
self.child_nodes_with_links_to_source_files += (
68
child_node_stats.child_nodes_with_links_to_source_files
69
)70
self.child_nodes_with_links_to_test_files += (
71
child_node_stats.child_nodes_with_links_to_test_files
72
)73
74
def get_code_coverage_ratio(self) -> float:
75
covered = self.child_nodes_with_links_to_source_files
76
total = self.child_nodes
77
ratio = max(0.0, min(1.0, covered / total))
78
return ratio
79
80
def get_test_coverage_ratio(self) -> float:
81
covered = self.child_nodes_with_links_to_test_files
82
total = self.child_nodes
83
ratio = max(0.0, min(1.0, covered / total))
84
return ratio
85
86
87
class PlotlyDataFrameColumn:
88
COLOR_SOURCE = "_COLOR_SOURCE"
89
COLOR_TEST = "_COLOR_TEST"
90
LEVEL = "_LEVEL"
91
PARENT_MID = "_PARENT_MID"
92
WEIGHT = "_WEIGHT"
93
IS_NORMATIVE = "_IS_NORMATIVE"
94
95
96
class TreeMapGenerator:
97
@staticmethod98
@timing_decorator("Export tree map visualizations")
99
def export(
100
project_config: ProjectConfig,
101
traceability_index: TraceabilityIndex,
102
html_templates: HTMLTemplates,
103
) -> None:
104
data = []
105
106
for document_ in traceability_index.document_tree.document_list:
107
assert document_.meta is not None
108
if traceability_index.file_dependency_manager.must_generate(
109
document_.meta.output_document_full_path
110
):111
break112
else:
113
print( # noqa: T201
114
"All documents are up-to-date. "115
"Skipping the generation of the tree map screen."116
)117
return118
119
documents_with_requirements = set()
120
121
map_node_to_coverage: Dict[
122
Union[SDocNode, SDocDocument], NodeStats
123
] = {}
124
125
link_renderer = LinkRenderer(
126
root_path="", static_path=project_config.dir_for_sdoc_assets
127
)128
129
def create_node_link(node__: Union[SDocDocument, SDocNode]) -> str:
130
href = link_renderer.render_node_link(
131
node__,
132
context_document=None,
133
document_type=DocumentType.DOCUMENT,
134
)135
return f'<a href="{href}">Open in document</a>'
136
137
def get_node_stats(
138
node_: Union[SDocNode, SDocDocument],
139
) -> NodeStats:
140
if node_ in map_node_to_coverage:
141
return map_node_to_coverage[node_]
142
143
if not node_.section_contents:
144
if (
145
not isinstance(node_, SDocNode)
146
or not node_.is_normative_node()
147
or node_.reserved_uid is None
148
):149
return NodeStats.create_child_node_without_stats()
150
151
node_stats = NodeStats(child_nodes=1)
152
153
children = traceability_index.get_children_requirements(node_)
154
155
has_children_all_covered_with_code = len(children) > 0
156
has_children_all_covered_with_test = len(children) > 0
157
158
for child_node_ in children:
159
child_node_stats = get_node_stats(child_node_)
160
if (
161
child_node_stats.child_nodes_with_links_to_source_files
162
== 0
163
):164
has_children_all_covered_with_code = False
165
166
if (
167
child_node_stats.child_nodes_with_links_to_test_files
168
== 0
169
):170
has_children_all_covered_with_test = False
171
172
node_stats.child_nodes_with_links_to_source_files = int(
173
has_children_all_covered_with_code174
)175
node_stats.child_nodes_with_links_to_test_files = int(
176
has_children_all_covered_with_test177
)178
179
source_files = traceability_index.get_file_traceability_index().get_requirement_file_links(
180
node_181
)182
for source_file_tuple_ in source_files:
183
if "tests/" in source_file_tuple_[0]:
184
node_stats.child_nodes_with_links_to_test_files = 1
185
else:
186
node_stats.child_nodes_with_links_to_source_files = 1
187
188
return node_stats
189
190
this_node_counters = NodeStats()
191
192
for sub_node_ in node_.section_contents:
193
if not isinstance(sub_node_, SDocNode):
194
continue195
if sub_node_.node_type == "TEXT":
196
continue197
sub_node_stats = get_node_stats(sub_node_)
198
map_node_to_coverage[sub_node_] = sub_node_stats
199
200
this_node_counters.add_child_stats(sub_node_stats)
201
202
return this_node_counters
203
204
for document_ in traceability_index.document_tree.document_list:
205
if document_.document_is_included():
206
continue207
208
map_node_to_coverage[document_] = get_node_stats(document_)
209
210
document_iterator = SDocDocumentIterator(document_)
211
for node_, _ in document_iterator.all_content(
212
print_fragments=False
213
):214
if not isinstance(node_, SDocNode):
215
continue216
217
if node_.is_normative_node():
218
documents_with_requirements.add(document_)
219
map_node_to_coverage[node_] = get_node_stats(node_)
220
221
root_node_title = project_config.project_title
222
223
for document_ in traceability_index.document_tree.document_list:
224
if document_.document_is_included():
225
continue226
227
color_code = "white"
228
color_test = "white"
229
230
if document_ in documents_with_requirements:
231
node_stats = get_node_stats(document_)
232
if node_stats.child_nodes > 0:
233
color_code = get_color(node_stats.get_code_coverage_ratio())
234
color_test = get_color(node_stats.get_test_coverage_ratio())
235
236
document_total_size = document_.get_total_size()[0]
237
document_normative_total_size = document_.get_total_size()[1]
238
239
title = document_.reserved_title
240
title_normative = title
241
title += " (" + str(document_total_size) + ")"
242
title_normative += " (" + str(document_normative_total_size) + ")"
243
244
data.append(
245
{246
PlotlyDataFrameColumn.WEIGHT: document_total_size
247
if document_total_size > 10
248
else 10,
249
"MID": document_.reserved_mid,
250
"UID": document_.reserved_uid
251
if document_.reserved_uid is not None
252
else "",
253
"TITLE": title,
254
"TITLE_NORMATIVE": title_normative,
255
"STATEMENT": " (" + str(document_total_size) + ")",
256
"_LINK": create_node_link(document_),
257
PlotlyDataFrameColumn.LEVEL: 0,
258
PlotlyDataFrameColumn.PARENT_MID: root_node_title,
259
PlotlyDataFrameColumn.COLOR_SOURCE: color_code,
260
PlotlyDataFrameColumn.COLOR_TEST: color_test,
261
PlotlyDataFrameColumn.IS_NORMATIVE: document_
262
in documents_with_requirements,
263
},264
)265
266
document_iterator = SDocDocumentIterator(document_)
267
for node, context_ in document_iterator.all_content(
268
print_fragments=False
269
):270
if not isinstance(node, SDocNode):
271
continue272
273
is_normative = node.is_normative_node() or (
274
node.node_type == "SECTION" and node.ng_has_requirements
275
)276
277
node_total_size = node.get_total_size()[0]
278
node_normative_total_size = node.get_total_size()[1]
279
280
parent_mid = node.parent.reserved_mid
281
282
if node.reserved_title is not None:
283
title = node.reserved_title
284
else:
285
title = "[TEXT] node"
286
title_normative = title
287
288
if (
289
node.section_contents is not None
290
and len(node.section_contents) > 0
291
):292
title += " (" + str(node_total_size) + ")"
293
title_normative += (
294
" (" + str(node_normative_total_size) + ")"
295
)296
297
statement = (
298
node.reserved_statement
299
if node.reserved_statement is not None
300
else ""
301
)302
if len(statement) > 120:
303
statement = statement[:120] + "..."
304
305
color_code = "white"
306
color_test = "white"
307
308
if (
309
node.node_type != "TEXT"
310
and document_ in documents_with_requirements
311
):312
if document_ in documents_with_requirements:
313
node_stats = get_node_stats(node)
314
if node_stats.child_nodes > 0:
315
color_code = get_color(
316
node_stats.get_code_coverage_ratio()
317
)318
color_test = get_color(
319
node_stats.get_test_coverage_ratio()
320
)321
322
data.append(
323
{324
PlotlyDataFrameColumn.WEIGHT: node_total_size
325
if node_total_size > 10
326
else 10,
327
"MID": node.reserved_mid,
328
"UID": node.reserved_uid
329
if node.reserved_uid is not None
330
else "",
331
"TITLE": title,
332
"TITLE_NORMATIVE": title_normative,
333
"STATEMENT": statement,
334
"_LINK": create_node_link(
335
node,
336
),337
PlotlyDataFrameColumn.PARENT_MID: parent_mid,
338
PlotlyDataFrameColumn.LEVEL: context_.get_level(),
339
PlotlyDataFrameColumn.COLOR_SOURCE: color_code,
340
PlotlyDataFrameColumn.COLOR_TEST: color_test,
341
PlotlyDataFrameColumn.IS_NORMATIVE: is_normative,
342
},343
)344
345
root_row = {
346
"MID": root_node_title,
347
"UID": "",
348
"TITLE": root_node_title,
349
"TITLE_NORMATIVE": root_node_title,
350
"STATEMENT": "",
351
PlotlyDataFrameColumn.WEIGHT: 0,
352
"_SHORT_LABEL": "",
353
"_LONG_LABEL": "",
354
"_IS_LEAF": False,
355
"_LINK": "",
356
PlotlyDataFrameColumn.PARENT_MID: "",
357
PlotlyDataFrameColumn.LEVEL: 0,
358
PlotlyDataFrameColumn.COLOR_SOURCE: "white",
359
PlotlyDataFrameColumn.COLOR_TEST: "white",
360
PlotlyDataFrameColumn.IS_NORMATIVE: True,
361
}362
data.append(root_row)
363
364
df = pd.DataFrame(data)
365
366
df["_IS_LEAF"] = ~df["MID"].isin(df[PlotlyDataFrameColumn.PARENT_MID])
367
368
def short_label(row_: Any) -> Any:
369
title = row_["TITLE"]
370
# FIXME: Move this reasoning to JS based on zoom depth.371
if len(title) > 20:
372
title = split_into_max_n_lines(title, max_lines=2)
373
return title
374
375
def short_label_normative(row_: Any) -> Any:
376
title = row_["TITLE_NORMATIVE"]
377
# FIXME: Move this reasoning to JS based on zoom depth.378
if len(title) > 20:
379
title = split_into_max_n_lines(title, max_lines=2)
380
return title
381
382
df["_SHORT_LABEL"] = df.apply(short_label, axis=1)
383
df["_SHORT_LABEL_NORMATIVE"] = df.apply(short_label_normative, axis=1)
384
385
def wrap_text(text: Optional[str], width: int = 80) -> str:
386
if not text:
387
return ""
388
return "<br>".join(textwrap.wrap(text, width=width))
389
390
def long_or_short_label(row_: Any) -> Any:
391
if row_["_IS_LEAF"] and row_["STATEMENT"]:
392
statement = row_["STATEMENT"]
393
statement = statement.replace("\n", "<br>")
394
statement = wrap_text(statement, 80)
395
396
return f"<b>{row_['TITLE']}</b><br><br>{statement}<br><br>{row_['_LINK']}"
397
title = row_["TITLE"]
398
if len(title) > 30:
399
title = (
400
"<br>".join(title.split(" ")) + f"<br><br>{row_['_LINK']}"
401
)402
return title
403
404
df["_LONG_LABEL"] = df.apply(long_or_short_label, axis=1)
405
406
def hover_(row_: Any) -> Any:
407
mid = row_["MID"]
408
uid = row_["UID"]
409
title = row_["TITLE"]
410
statement = row_["STATEMENT"]
411
412
return f"""\
413
<b>MID</b>: {mid}<br>
414
<b>UID</b>: {uid}<br>
415
<b>TITLE</b>: {title}<br>
416
<b>STATEMENT</b>: {statement}<br>
417
"""418
419
df["_HOVER"] = df.apply(hover_, axis=1)
420
421
parts: List[GraphSection] = []
422
423
# FIGURE: Document tree map.424
fig = px.treemap(
425
df,
426
names="_SHORT_LABEL",
427
ids="MID",
428
parents=PlotlyDataFrameColumn.PARENT_MID,
429
values=PlotlyDataFrameColumn.WEIGHT,
430
custom_data=[
431
"_HOVER",
432
"_SHORT_LABEL",
433
"_LONG_LABEL",
434
"_IS_LEAF",
435
PlotlyDataFrameColumn.LEVEL,
436
],437
)438
fig.update_layout(
439
margin={"t": 25, "l": 25, "r": 25, "b": 25},
440
height=800,
441
)442
fig.update_traces(
443
root_color="lightgray",
444
marker={
445
"colors": ["white"] * len(df),
446
},447
marker_line_color="#ddd",
448
marker_line_width=1,
449
textposition="middle center",
450
texttemplate="%{customdata[0]}",
451
hoverinfo="skip",
452
hovertemplate=None,
453
)454
parts.append(
455
GraphSection(
456
title="Document tree map",
457
description="""\
458
This is a general representation of a document tree. All nodes are included,459
both normative (e.g., REQUIREMENT) and non-normative (e.g., TEXT). The numbers460
indicate how many nodes each section or node contains.461
""",
462
graph_content=pio.to_html(
463
fig,
464
full_html=False,
465
include_plotlyjs=True,
466
),467
)468
)469
470
# FIGURE: Document tree: Requirements coverage with source.471
df = df[df[PlotlyDataFrameColumn.IS_NORMATIVE]]
472
fig = px.treemap(
473
df,
474
names="_SHORT_LABEL_NORMATIVE",
475
ids="MID",
476
parents=PlotlyDataFrameColumn.PARENT_MID,
477
values=PlotlyDataFrameColumn.WEIGHT,
478
custom_data=[
479
"_HOVER",
480
"_SHORT_LABEL",
481
"_LONG_LABEL",
482
"_IS_LEAF",
483
PlotlyDataFrameColumn.LEVEL,
484
],485
)486
fig.update_layout(
487
margin={"t": 25, "l": 25, "r": 25, "b": 25},
488
height=800,
489
)490
fig.update_traces(
491
root_color="lightgray",
492
marker={
493
"colors": df[PlotlyDataFrameColumn.COLOR_SOURCE],
494
},495
marker_line_color="#ddd",
496
marker_line_width=1,
497
textposition="middle center",
498
texttemplate="%{customdata[0]}",
499
hoverinfo="skip",
500
hovertemplate=None,
501
)502
parts.append(
503
GraphSection(
504
title="Requirements coverage with source",
505
description="""\
506
This graph shows which requirements are covered by at least one source file.507
A requirement is also considered covered if it has child requirements that are508
themselves covered by source files.509
510
<ul>511
<li>512
<span style="background-color: #AAFFAA;">Green</span> –513
Requirement/section is fully covered with one or more source file.514
</li>515
<li>516
<span style="background-color: #FFFFAA;">Yellow</span> –517
Section is partially covered with one or more source file.518
</li>519
<li>520
<span style="background-color: #FFAAAA;">Red</span> –521
Requirement/section is not covered by any source files.522
</li>523
</ul>524
""",
525
graph_content=pio.to_html(
526
fig,
527
full_html=False,
528
include_plotlyjs=False,
529
),530
)531
)532
533
# FIGURE: Document tree: Requirements coverage with test.534
fig = deepcopy(fig)
535
fig.update_traces(
536
root_color="lightgray",
537
marker={
538
"colors": df[PlotlyDataFrameColumn.COLOR_TEST],
539
},540
marker_line_color="#ddd",
541
marker_line_width=1,
542
textposition="middle center",
543
texttemplate="%{customdata[0]}",
544
hoverinfo="skip",
545
hovertemplate=None,
546
)547
parts.append(
548
GraphSection(
549
title="Requirements coverage with test",
550
description="""\
551
This graph shows which requirements are covered by at least one test.552
A requirement is also considered covered if it has child requirements that are553
themselves covered by tests. A source file is considered a test file if its path554
contains "tests/".555
556
<ul>557
<li>558
<span style="background-color: #AAFFAA;">Green</span> –559
Requirement/section is fully covered with one or more tests.560
</li>561
<li>562
<span style="background-color: #FFFFAA;">Yellow</span> –563
Section is partially covered with one or more test.564
</li>565
<li>566
<span style="background-color: #FFAAAA;">Red</span> –567
Requirement/section is not covered with by any tests.568
</li>569
</ul>570
""",
571
graph_content=pio.to_html(
572
fig,
573
full_html=False,
574
include_plotlyjs=False,
575
),576
)577
)578
579
body = "".join(part_.get_html() for part_ in parts)
580
581
view_object = TreeMapViewObject(
582
traceability_index=traceability_index,
583
project_config=project_config,
584
body=body,
585
)586
html = view_object.render_screen(html_templates.jinja_environment())
587
588
output_html = os.path.join(
589
project_config.export_output_html_root,
590
"tree_map.html",
591
)592
593
with open(output_html, "w", encoding="utf-8") as file_:
594
file_.write(html)