Path:
strictdoc/backend/rst/rst_to_html_fragment_writer.py
Lines:
292
Non-empty lines:
259
Non-empty lines covered with requirements:
259 / 259 (100.0%)
Functions:
6
Functions covered by requirements:
6 / 6 (100.0%)
1
"""2
@relation(SDOC-SRS-3, scope=file)3
"""4
5
import hashlib
6
import io
7
import os
8
import re
9
import time
10
import uuid
11
from pathlib import Path
12
from typing import Optional, Tuple
13
14
from docutils.core import publish_parts
15
from docutils.parsers.rst import directives, roles
16
from docutils.utils import SystemMessage
17
from markupsafe import Markup
18
from pygments.lexers import _load_lexers
19
20
from strictdoc.backend.rst.directives.raw_html_role import raw_html_role
21
from strictdoc.backend.rst.directives.sphinx_style_math import (
22
MathDirective,
23
MathDirectiveForServer,
24
eq_role,
25
eq_role_for_server,
26
math_role,
27
math_role_for_server,
28
)29
from strictdoc.backend.rst.directives.wildcard_enhanced_image import (
30
STRICTDOC_FLAT_ASSETS_SETTING,
31
STRICTDOC_REFERENCE_PATH_SETTING,
32
WildcardEnhancedImage,
33
)34
from strictdoc.backend.sdoc.models.document import SDocDocument
35
from strictdoc.core.project_config import ProjectConfig, ProjectFeature
36
from strictdoc.helpers.file_system import file_open_read_bytes
37
38
MAX_RETRIES_FOR_CACHE_FILESYSTEM_LOCKING = 3
39
40
41
class RstToHtmlFragmentWriter:
42
directives.register_directive("image", WildcardEnhancedImage)
43
44
roles.register_local_role("rawhtml", raw_html_role)
45
46
# FIXME: It doesn't feel right to load lexers like this.47
_load_lexers("strictdoc.backend.rst.strictdoc_lexer")
48
49
BASE_SETTINGS = {
50
# This is important for code syntax highlighting. The setting of51
# "short" is coupled to the CSS file that we auto-generated using Pygments:52
# strictdoc/export/html/_static/pygments.css53
"syntax_highlight": "short",
54
"syntax_highlight_opts": {
55
"linenos": "inline", # "table"
56
},57
}58
59
def __init__(
60
self,
61
*,
62
project_config: ProjectConfig,
63
context_document: Optional[SDocDocument],
64
flat_assets: bool = False,
65
reference_path_override: Optional[str] = None,
66
):67
self.source_path: str
68
path_to_output_dir_md5: str = hashlib.md5(
69
project_config.output_dir.encode("utf-8")
70
).hexdigest()
71
72
path_to_tmp_dir = project_config.get_path_to_cache_dir()
73
self.path_to_rst_cache_dir = os.path.join(
74
path_to_tmp_dir, "rst", path_to_output_dir_md5
75
)76
self.reference_path = os.getcwd()
77
78
if reference_path_override is not None:
79
self.reference_path = reference_path_override
80
# See the comment below in the elif branch for why source_path is81
# set to a file inside the reference directory.82
self.source_path = os.path.join(
83
reference_path_override, "STRICTDOC-FRAGMENT.rst"
84
)85
elif context_document is not None:
86
assert context_document.meta is not None
87
self.reference_path = (
88
context_document.meta.output_document_dir_full_path
89
)90
91
# This is a delicate move. Based on a user report and our findings,92
# the csv-table RST directive relies on the 'source path' to93
# calculate paths to CSV files.94
# Our case is, however, special: we do not render RST files but95
# rather RST fragments in memory, and because of that we don't have96
# RST files to point to with 'source_path=' below.97
# At the same time, passing the output folder of the document works98
# because this RST-to-HTML writer resolves path to CSV assets99
# that are copied to that output folder by StrictDoc.100
# See CSVTable().get_csv_data() where the source_path is used.101
self.source_path = os.path.join(
102
context_document.meta.output_document_dir_full_path,
103
"STRICTDOC-FRAGMENT.rst",
104
)105
else:
106
self.source_path = "<string>"
107
self.context_document: Optional[SDocDocument] = context_document
108
109
self.flat_assets: bool = flat_assets
110
111
if project_config.is_feature_activated(ProjectFeature.MATHJAX):
112
if project_config.is_running_on_server:
113
roles.register_canonical_role("eq", eq_role_for_server)
114
roles.register_canonical_role("math", math_role_for_server)
115
directives.register_directive("math", MathDirectiveForServer)
116
else:
117
roles.register_canonical_role("eq", eq_role)
118
roles.register_canonical_role("math", math_role)
119
directives.register_directive("math", MathDirective)
120
121
def write(self, rst_fragment: str, use_cache: bool = True) -> Markup:
122
assert isinstance(rst_fragment, str), rst_fragment
123
124
# Do not try to cache very small fragments.125
if len(rst_fragment) < 40:
126
return Markup(self._write_no_cache(rst_fragment))
127
128
path_to_rst_fragment_bucket_dir = os.path.join(
129
self.path_to_rst_cache_dir, str(len(rst_fragment))
130
)131
fragment_md5 = hashlib.md5(rst_fragment.encode("utf-8")).hexdigest()
132
# flat_assets mode produces different HTML (rebased image paths) for the133
# same RST fragment, so it needs its own cache entry.134
cache_key = fragment_md5 + ("_flat" if self.flat_assets else "")
135
path_to_cached_fragment = os.path.join(
136
path_to_rst_fragment_bucket_dir, cache_key
137
)138
if use_cache and os.path.isdir(path_to_rst_fragment_bucket_dir):
139
if os.path.isfile(path_to_cached_fragment):
140
with file_open_read_bytes(
141
path_to_cached_fragment142
) as cached_fragment_file_:
143
return Markup(cached_fragment_file_.read().decode("UTF-8"))
144
else:
145
Path(path_to_rst_fragment_bucket_dir).mkdir(
146
parents=True, exist_ok=True
147
)148
149
rendered_html: str = self._write_no_cache(rst_fragment)
150
rendered_html_bytes = rendered_html.encode("UTF-8")
151
152
if use_cache:
153
# Thread-safe cache update strategy:154
# 1) write bytes to a unique temp file, then155
# 2) atomically replace the target cache file with os.replace().156
# This ensures that concurrent readers either see the old complete157
# file or the new complete file, but never a partially written file.158
# (os.replace is atomic when source and destination are on the same159
# filesystem, which is true because both paths are in one cache dir.)160
tmp_path_to_cached_fragment = (
161
f"{path_to_cached_fragment}.{uuid.uuid4().hex}.tmp"
162
)163
with open(
164
tmp_path_to_cached_fragment, "wb"
165
) as cached_fragment_file_:
166
cached_fragment_file_.write(rendered_html_bytes)
167
# On Windows in particular, we might get interference from Windows Defender168
# for obtaining the file system locks. As a work-around, we try multiple times...169
for attempt in range(MAX_RETRIES_FOR_CACHE_FILESYSTEM_LOCKING):
170
try:
171
os.replace(
172
tmp_path_to_cached_fragment, path_to_cached_fragment
173
)174
break # Success!
175
except PermissionError as e:
176
if attempt < MAX_RETRIES_FOR_CACHE_FILESYSTEM_LOCKING - 1:
177
# Wait 100ms, then 200ms, etc., to let Windows Defender release the lock178
time.sleep(0.1 * (attempt + 1))
179
else:
180
# Surface the original error181
raise e
182
183
return Markup(rendered_html)
184
185
def _write_no_cache(self, rst_fragment: str) -> str:
186
assert isinstance(rst_fragment, str), rst_fragment
187
188
# How do I convert a docutils document tree into an HTML string?189
# https://stackoverflow.com/a/32168938/598057190
# Use a io.StringIO as the warning stream to prevent warnings from191
# being printed to sys.stderr.192
# https://www.programcreek.com/python/example/88126/docutils.core.publish_parts193
warning_stream = io.StringIO()
194
settings = {
195
**self.BASE_SETTINGS,
196
"warning_stream": warning_stream,
197
STRICTDOC_REFERENCE_PATH_SETTING: self.reference_path,
198
STRICTDOC_FLAT_ASSETS_SETTING: self.flat_assets,
199
}200
201
output = publish_parts(
202
rst_fragment,
203
writer="html",
204
settings_overrides=settings,
205
source_path=self.source_path,
206
)207
208
if warning_stream.tell() > 0:
209
warnings = warning_stream.getvalue().rstrip("\n")
210
# A typical RST warning:211
# """212
# path-to-output-folder/file.rst:4: (WARNING/2) Bullet list ends213
# without a blank line; unexpected unindent.214
# """215
match = re.search(
216
r".*:(?P<line>\d+): \(.*\) (?P<message>.*)", warnings
217
)218
if match is not None:
219
error_message = (
220
f"RST markup syntax error on line {match.group('line')}: "
221
f"{match.group('message')}"
222
)223
else:
224
error_message = f"RST markup syntax error: {warnings}"
225
final_message = (
226
f"problems when converting RST to HTML: {error_message}\n"
227
"RST fragment: >>>\n"
228
f"{rst_fragment}"
229
"<<<"230
)231
raise RuntimeError(final_message)
232
233
html: str = output["html_body"]
234
235
return html
236
237
def write_with_validation(
238
self, rst_fragment: str
239
) -> Tuple[Optional[str], Optional[str]]:
240
# How do I convert a docutils document tree into an HTML string?241
# https://stackoverflow.com/a/32168938/598057242
# Use a io.StringIO as the warning stream to prevent warnings from243
# being printed to sys.stderr.244
# https://www.programcreek.com/python/example/88126/docutils.core.publish_parts245
warning_stream = io.StringIO()
246
settings = {
247
**self.BASE_SETTINGS,
248
"warning_stream": warning_stream,
249
STRICTDOC_REFERENCE_PATH_SETTING: self.reference_path,
250
STRICTDOC_FLAT_ASSETS_SETTING: self.flat_assets,
251
}252
253
try:
254
output = publish_parts(
255
rst_fragment, writer="html", settings_overrides=settings
256
)257
warnings = (
258
warning_stream.getvalue().rstrip("\n")
259
if warning_stream.tell() > 0
260
else None
261
)262
except SystemMessage as exception:
263
output = None
264
warnings = str(exception)
265
266
if warnings is not None and len(warnings) > 0:
267
# A typical RST warning:268
# """269
# <string>:4: (WARNING/2) Bullet list ends without a blank line;270
# unexpected unindent.271
# """272
match = re.search(
273
r".*<.*>:(?P<line>\d+): \(.*\) (?P<message>.*)", warnings
274
)275
if match is not None:
276
error_message = (
277
f"RST markup syntax error on line {match.group('line')}: "
278
f"{match.group('message')}"
279
)280
else:
281
error_message = f"RST markup syntax error: {warnings}"
282
return None, error_message
283
284
html = output["html_body"]
285
286
return html, None
287
288
@staticmethod289
def write_anchor_link(title: str, href: str) -> str:
290
return f"""\
291
:rawhtml:`<a href="{href}">🔗 {title}</a>`\
292
"""