StrictDoc Documentation
strictdoc/backend/rst/rst_to_html_fragment_writer.py
Source file coverage
Path:
strictdoc/backend/rst/rst_to_html_fragment_writer.py
Lines:
292
Non-empty lines:
259
Non-empty lines covered with requirements:
259 / 259 (100.0%)
Functions:
6
Functions covered by requirements:
6 / 6 (100.0%)
1
"""
2
@relation(SDOC-SRS-3, scope=file)
3
"""
4
 
5
import hashlib
6
import io
7
import os
8
import re
9
import time
10
import uuid
11
from pathlib import Path
12
from typing import Optional, Tuple
13
 
14
from docutils.core import publish_parts
15
from docutils.parsers.rst import directives, roles
16
from docutils.utils import SystemMessage
17
from markupsafe import Markup
18
from pygments.lexers import _load_lexers
19
 
20
from strictdoc.backend.rst.directives.raw_html_role import raw_html_role
21
from strictdoc.backend.rst.directives.sphinx_style_math import (
22
    MathDirective,
23
    MathDirectiveForServer,
24
    eq_role,
25
    eq_role_for_server,
26
    math_role,
27
    math_role_for_server,
28
)
29
from strictdoc.backend.rst.directives.wildcard_enhanced_image import (
30
    STRICTDOC_FLAT_ASSETS_SETTING,
31
    STRICTDOC_REFERENCE_PATH_SETTING,
32
    WildcardEnhancedImage,
33
)
34
from strictdoc.backend.sdoc.models.document import SDocDocument
35
from strictdoc.core.project_config import ProjectConfig, ProjectFeature
36
from strictdoc.helpers.file_system import file_open_read_bytes
37
 
38
MAX_RETRIES_FOR_CACHE_FILESYSTEM_LOCKING = 3
39
 
40
 
41
class RstToHtmlFragmentWriter:
42
    directives.register_directive("image", WildcardEnhancedImage)
43
 
44
    roles.register_local_role("rawhtml", raw_html_role)
45
 
46
    # FIXME: It doesn't feel right to load lexers like this.
47
    _load_lexers("strictdoc.backend.rst.strictdoc_lexer")
48
 
49
    BASE_SETTINGS = {
50
        # This is important for code syntax highlighting. The setting of
51
        # "short" is coupled to the CSS file that we auto-generated using Pygments:
52
        # strictdoc/export/html/_static/pygments.css
53
        "syntax_highlight": "short",
54
        "syntax_highlight_opts": {
55
            "linenos": "inline",  # "table"
56
        },
57
    }
58
 
59
    def __init__(
60
        self,
61
        *,
62
        project_config: ProjectConfig,
63
        context_document: Optional[SDocDocument],
64
        flat_assets: bool = False,
65
        reference_path_override: Optional[str] = None,
66
    ):
67
        self.source_path: str
68
        path_to_output_dir_md5: str = hashlib.md5(
69
            project_config.output_dir.encode("utf-8")
70
        ).hexdigest()
71
 
72
        path_to_tmp_dir = project_config.get_path_to_cache_dir()
73
        self.path_to_rst_cache_dir = os.path.join(
74
            path_to_tmp_dir, "rst", path_to_output_dir_md5
75
        )
76
        self.reference_path = os.getcwd()
77
 
78
        if reference_path_override is not None:
79
            self.reference_path = reference_path_override
80
            # See the comment below in the elif branch for why source_path is
81
            # set to a file inside the reference directory.
82
            self.source_path = os.path.join(
83
                reference_path_override, "STRICTDOC-FRAGMENT.rst"
84
            )
85
        elif context_document is not None:
86
            assert context_document.meta is not None
87
            self.reference_path = (
88
                context_document.meta.output_document_dir_full_path
89
            )
90
 
91
            # This is a delicate move. Based on a user report and our findings,
92
            # the csv-table RST directive relies on the 'source path' to
93
            # calculate paths to CSV files.
94
            # Our case is, however, special: we do not render RST files but
95
            # rather RST fragments in memory, and because of that we don't have
96
            # RST files to point to with 'source_path=' below.
97
            # At the same time, passing the output folder of the document works
98
            # because this RST-to-HTML writer resolves path to CSV assets
99
            # that are copied to that output folder by StrictDoc.
100
            # See CSVTable().get_csv_data() where the source_path is used.
101
            self.source_path = os.path.join(
102
                context_document.meta.output_document_dir_full_path,
103
                "STRICTDOC-FRAGMENT.rst",
104
            )
105
        else:
106
            self.source_path = "<string>"
107
        self.context_document: Optional[SDocDocument] = context_document
108
 
109
        self.flat_assets: bool = flat_assets
110
 
111
        if project_config.is_feature_activated(ProjectFeature.MATHJAX):
112
            if project_config.is_running_on_server:
113
                roles.register_canonical_role("eq", eq_role_for_server)
114
                roles.register_canonical_role("math", math_role_for_server)
115
                directives.register_directive("math", MathDirectiveForServer)
116
            else:
117
                roles.register_canonical_role("eq", eq_role)
118
                roles.register_canonical_role("math", math_role)
119
                directives.register_directive("math", MathDirective)
120
 
121
    def write(self, rst_fragment: str, use_cache: bool = True) -> Markup:
122
        assert isinstance(rst_fragment, str), rst_fragment
123
 
124
        # Do not try to cache very small fragments.
125
        if len(rst_fragment) < 40:
126
            return Markup(self._write_no_cache(rst_fragment))
127
 
128
        path_to_rst_fragment_bucket_dir = os.path.join(
129
            self.path_to_rst_cache_dir, str(len(rst_fragment))
130
        )
131
        fragment_md5 = hashlib.md5(rst_fragment.encode("utf-8")).hexdigest()
132
        # flat_assets mode produces different HTML (rebased image paths) for the
133
        # same RST fragment, so it needs its own cache entry.
134
        cache_key = fragment_md5 + ("_flat" if self.flat_assets else "")
135
        path_to_cached_fragment = os.path.join(
136
            path_to_rst_fragment_bucket_dir, cache_key
137
        )
138
        if use_cache and os.path.isdir(path_to_rst_fragment_bucket_dir):
139
            if os.path.isfile(path_to_cached_fragment):
140
                with file_open_read_bytes(
141
                    path_to_cached_fragment
142
                ) as cached_fragment_file_:
143
                    return Markup(cached_fragment_file_.read().decode("UTF-8"))
144
        else:
145
            Path(path_to_rst_fragment_bucket_dir).mkdir(
146
                parents=True, exist_ok=True
147
            )
148
 
149
        rendered_html: str = self._write_no_cache(rst_fragment)
150
        rendered_html_bytes = rendered_html.encode("UTF-8")
151
 
152
        if use_cache:
153
            # Thread-safe cache update strategy:
154
            # 1) write bytes to a unique temp file, then
155
            # 2) atomically replace the target cache file with os.replace().
156
            # This ensures that concurrent readers either see the old complete
157
            # file or the new complete file, but never a partially written file.
158
            # (os.replace is atomic when source and destination are on the same
159
            # filesystem, which is true because both paths are in one cache dir.)
160
            tmp_path_to_cached_fragment = (
161
                f"{path_to_cached_fragment}.{uuid.uuid4().hex}.tmp"
162
            )
163
            with open(
164
                tmp_path_to_cached_fragment, "wb"
165
            ) as cached_fragment_file_:
166
                cached_fragment_file_.write(rendered_html_bytes)
167
            # On Windows in particular, we might get interference from Windows Defender
168
            # for obtaining the file system locks. As a work-around, we try multiple times...
169
            for attempt in range(MAX_RETRIES_FOR_CACHE_FILESYSTEM_LOCKING):
170
                try:
171
                    os.replace(
172
                        tmp_path_to_cached_fragment, path_to_cached_fragment
173
                    )
174
                    break  # Success!
175
                except PermissionError as e:
176
                    if attempt < MAX_RETRIES_FOR_CACHE_FILESYSTEM_LOCKING - 1:
177
                        # Wait 100ms, then 200ms, etc., to let Windows Defender release the lock
178
                        time.sleep(0.1 * (attempt + 1))
179
                    else:
180
                        # Surface the original error
181
                        raise e
182
 
183
        return Markup(rendered_html)
184
 
185
    def _write_no_cache(self, rst_fragment: str) -> str:
186
        assert isinstance(rst_fragment, str), rst_fragment
187
 
188
        # How do I convert a docutils document tree into an HTML string?
189
        # https://stackoverflow.com/a/32168938/598057
190
        # Use a io.StringIO as the warning stream to prevent warnings from
191
        # being printed to sys.stderr.
192
        # https://www.programcreek.com/python/example/88126/docutils.core.publish_parts
193
        warning_stream = io.StringIO()
194
        settings = {
195
            **self.BASE_SETTINGS,
196
            "warning_stream": warning_stream,
197
            STRICTDOC_REFERENCE_PATH_SETTING: self.reference_path,
198
            STRICTDOC_FLAT_ASSETS_SETTING: self.flat_assets,
199
        }
200
 
201
        output = publish_parts(
202
            rst_fragment,
203
            writer="html",
204
            settings_overrides=settings,
205
            source_path=self.source_path,
206
        )
207
 
208
        if warning_stream.tell() > 0:
209
            warnings = warning_stream.getvalue().rstrip("\n")
210
            # A typical RST warning:
211
            # """
212
            # path-to-output-folder/file.rst:4: (WARNING/2) Bullet list ends
213
            # without a blank line; unexpected unindent.
214
            # """
215
            match = re.search(
216
                r".*:(?P<line>\d+): \(.*\) (?P<message>.*)", warnings
217
            )
218
            if match is not None:
219
                error_message = (
220
                    f"RST markup syntax error on line {match.group('line')}: "
221
                    f"{match.group('message')}"
222
                )
223
            else:
224
                error_message = f"RST markup syntax error: {warnings}"
225
            final_message = (
226
                f"problems when converting RST to HTML: {error_message}\n"
227
                "RST fragment: >>>\n"
228
                f"{rst_fragment}"
229
                "<<<"
230
            )
231
            raise RuntimeError(final_message)
232
 
233
        html: str = output["html_body"]
234
 
235
        return html
236
 
237
    def write_with_validation(
238
        self, rst_fragment: str
239
    ) -> Tuple[Optional[str], Optional[str]]:
240
        # How do I convert a docutils document tree into an HTML string?
241
        # https://stackoverflow.com/a/32168938/598057
242
        # Use a io.StringIO as the warning stream to prevent warnings from
243
        # being printed to sys.stderr.
244
        # https://www.programcreek.com/python/example/88126/docutils.core.publish_parts
245
        warning_stream = io.StringIO()
246
        settings = {
247
            **self.BASE_SETTINGS,
248
            "warning_stream": warning_stream,
249
            STRICTDOC_REFERENCE_PATH_SETTING: self.reference_path,
250
            STRICTDOC_FLAT_ASSETS_SETTING: self.flat_assets,
251
        }
252
 
253
        try:
254
            output = publish_parts(
255
                rst_fragment, writer="html", settings_overrides=settings
256
            )
257
            warnings = (
258
                warning_stream.getvalue().rstrip("\n")
259
                if warning_stream.tell() > 0
260
                else None
261
            )
262
        except SystemMessage as exception:
263
            output = None
264
            warnings = str(exception)
265
 
266
        if warnings is not None and len(warnings) > 0:
267
            # A typical RST warning:
268
            # """
269
            # <string>:4: (WARNING/2) Bullet list ends without a blank line;
270
            # unexpected unindent.
271
            # """
272
            match = re.search(
273
                r".*<.*>:(?P<line>\d+): \(.*\) (?P<message>.*)", warnings
274
            )
275
            if match is not None:
276
                error_message = (
277
                    f"RST markup syntax error on line {match.group('line')}: "
278
                    f"{match.group('message')}"
279
                )
280
            else:
281
                error_message = f"RST markup syntax error: {warnings}"
282
            return None, error_message
283
 
284
        html = output["html_body"]
285
 
286
        return html, None
287
 
288
    @staticmethod
289
    def write_anchor_link(title: str, href: str) -> str:
290
        return f"""\
291
:rawhtml:`<a href="{href}">🔗&nbsp;{title}</a>`\
292
"""