StrictDoc Documentation
strictdoc/core/file_system/document_finder.py
Source file coverage
Path:
strictdoc/core/file_system/document_finder.py
Lines:
313
Non-empty lines:
269
Non-empty lines covered with requirements:
269 / 269 (100.0%)
Functions:
7
Functions covered by requirements:
7 / 7 (100.0%)
1
"""
2
@relation(SDOC-SRS-104, SDOC-SRS-115, scope=file)
3
"""
4
 
5
import os
6
import sys
7
from functools import partial
8
from typing import Dict, List, Tuple, Union
9
 
10
from strictdoc.backend.sdoc.models.document import SDocDocument
11
from strictdoc.backend.sdoc.models.document_grammar import DocumentGrammar
12
from strictdoc.core.asset_manager import AssetManager
13
from strictdoc.core.document_meta import DocumentMeta
14
from strictdoc.core.document_tree import DocumentTree
15
from strictdoc.core.file_system.file_tree import (
16
    File,
17
    FileFinder,
18
    FileTree,
19
    Folder,
20
    PathFinder,
21
)
22
from strictdoc.core.format import Format
23
from strictdoc.core.project_config import ProjectConfig
24
from strictdoc.helpers.exception import StrictDocException
25
from strictdoc.helpers.parallelizer import Parallelizer
26
from strictdoc.helpers.paths import SDocRelativePath
27
from strictdoc.helpers.textx import drop_textx_meta
28
from strictdoc.helpers.timing import measure_performance, timing_decorator
29
 
30
# Each entry: (extension, Format instance, is_grammar).
31
# Sorted longest-extension-first so that e.g. ".gra.md" is matched before
32
# ".md" (a "foo.gra.md" file ends with both).
33
FormatDispatchEntry = Tuple[str, Format, bool]
34
 
35
 
36
def _build_extension_dispatch_table(
37
    project_config: ProjectConfig,
38
) -> List[FormatDispatchEntry]:
39
    entries: List[FormatDispatchEntry] = []
40
    for format_ in project_config.formats:
41
        if format_.supports_read():
42
            for extension_ in format_.read_extensions():
43
                entries.append((extension_, format_, False))
44
        if format_.supports_grammar():
45
            for extension_ in format_.grammar_extensions():
46
                entries.append((extension_, format_, True))
47
    entries.sort(key=lambda entry: len(entry[0]), reverse=True)
48
    return entries
49
 
50
 
51
def get_document_extensions(project_config: ProjectConfig) -> List[str]:
52
    """
53
    All extensions natively read (as documents or grammars) by the
54
    project's registered Formats. Used both for DocumentFinder's own file
55
    discovery and, in strictdoc/server/document_watcher.py, so a filesystem
56
    watcher can't drift out of sync with what DocumentFinder actually reads.
57
    """
58
    return [
59
        extension_
60
        for extension_, _, _ in _build_extension_dispatch_table(project_config)
61
    ]
62
 
63
 
64
class DocumentFinder:
65
    @staticmethod
66
    @timing_decorator("Find and read SDoc files")
67
    def find_sdoc_content(
68
        project_config: ProjectConfig, parallelizer: Parallelizer
69
    ) -> Tuple[DocumentTree, AssetManager]:
70
        assert project_config.input_paths is not None
71
        for paths_to_files_or_doc in project_config.input_paths:
72
            if not os.path.exists(paths_to_files_or_doc):
73
                sys.stdout.flush()
74
                raise StrictDocException(
75
                    "error: "
76
                    "Provided path is neither a single file or a folder: "
77
                    f"'{paths_to_files_or_doc}'"
78
                )
79
 
80
        with measure_performance("Completed finding SDoc and assets"):
81
            file_tree, asset_manager = DocumentFinder._build_file_tree(
82
                project_config=project_config
83
            )
84
        with measure_performance("Completed building document tree"):
85
            document_tree = DocumentFinder._build_document_tree(
86
                file_tree, project_config, parallelizer
87
            )
88
 
89
        return document_tree, asset_manager
90
 
91
    @staticmethod
92
    def _process_worker_parse_document(
93
        document_triple: Tuple[Union[Folder, File], File, str],
94
        project_config: ProjectConfig,
95
    ) -> Tuple[File, str, Union[SDocDocument, DocumentGrammar]]:
96
        _, doc_file, file_tree_mount_folder = document_triple
97
        doc_full_path = doc_file.full_path
98
 
99
        with measure_performance(
100
            f"Reading SDOC: {os.path.basename(doc_full_path)}"
101
        ):
102
            document_or_grammar: Union[SDocDocument, DocumentGrammar, None] = (
103
                None
104
            )
105
 
106
            # @relation(SDOC-SRS-104, scope=range_start)
107
            for (
108
                extension_,
109
                format_,
110
                is_grammar_,
111
            ) in _build_extension_dispatch_table(project_config):
112
                if not doc_full_path.endswith(extension_):
113
                    continue
114
                if is_grammar_:
115
                    document_or_grammar = format_.read_grammar(
116
                        doc_file, project_config
117
                    )
118
                    assert isinstance(document_or_grammar, DocumentGrammar)
119
                else:
120
                    document_or_grammar = format_.read_from_file(
121
                        doc_file, project_config
122
                    )
123
                    assert isinstance(document_or_grammar, SDocDocument)
124
                break
125
            # @relation(SDOC-SRS-104, scope=range_end)
126
 
127
            if document_or_grammar is None:
128
                raise NotImplementedError
129
        drop_textx_meta(document_or_grammar)
130
 
131
        return doc_file, file_tree_mount_folder, document_or_grammar
132
 
133
    @staticmethod
134
    def _build_document_tree(
135
        file_trees: List[FileTree],
136
        project_config: ProjectConfig,
137
        parallelizer: Parallelizer,
138
    ) -> DocumentTree:
139
        """
140
        @relation(SDOC-SRS-48, scope=function)
141
        """
142
 
143
        assert isinstance(file_trees, list)
144
 
145
        output_root_html = project_config.export_output_html_root
146
        assert output_root_html is not None
147
 
148
        document_list: List[SDocDocument] = []
149
        map_docs_by_paths = {}
150
        map_docs_by_rel_paths: Dict[str, SDocDocument] = {}
151
        map_grammars_by_filenames = {}
152
 
153
        file_tree_list: List[Tuple[Union[Folder, File], File, str]] = []
154
        for file_tree in file_trees:
155
            file_tree_list.extend(list(file_tree.iterate()))
156
 
157
        process_document_binding = partial(
158
            DocumentFinder._process_worker_parse_document,
159
            project_config=project_config,
160
        )
161
 
162
        with measure_performance("Completed parsing all documents"):
163
            found_documents = parallelizer.run_parallel(
164
                file_tree_list, process_document_binding
165
            )
166
 
167
        doc_file: File
168
        for doc_file, file_tree_mount_folder, document in found_documents:
169
            assert isinstance(file_tree_mount_folder, str), (
170
                file_tree_mount_folder
171
            )
172
 
173
            if isinstance(document, DocumentGrammar):
174
                map_grammars_by_filenames[
175
                    doc_file.rel_path.relative_path_posix
176
                ] = document
177
                continue
178
 
179
            input_doc_full_path: str = doc_file.full_path
180
            map_docs_by_paths[input_doc_full_path] = document
181
            document_list.append(document)
182
 
183
            doc_relative_path_folder: SDocRelativePath = SDocRelativePath(
184
                os.path.dirname(doc_file.rel_path.relative_path)
185
            )
186
            output_document_dir_rel_path: SDocRelativePath = SDocRelativePath(
187
                os.path.join(
188
                    file_tree_mount_folder,
189
                    doc_relative_path_folder.relative_path,
190
                )
191
                if len(doc_relative_path_folder.relative_path) > 0
192
                else file_tree_mount_folder
193
            )
194
 
195
            document_filename = doc_file.file_name
196
            document_filename_base = os.path.splitext(document_filename)[0]
197
 
198
            output_document_dir_full_path: str = os.path.join(
199
                output_root_html, output_document_dir_rel_path.relative_path
200
            )
201
 
202
            input_doc_assets_dir_rel_path: SDocRelativePath = SDocRelativePath(
203
                os.path.join(
204
                    file_tree_mount_folder,
205
                    doc_relative_path_folder.relative_path,
206
                    "_assets",
207
                )
208
                if doc_relative_path_folder.length() > 0
209
                else "/".join((file_tree_mount_folder, "_assets"))
210
            )
211
 
212
            document_meta = DocumentMeta(
213
                doc_file.level,
214
                file_tree_mount_folder,
215
                document_filename,
216
                document_filename_base,
217
                input_doc_full_path,
218
                doc_file.rel_path,
219
                doc_relative_path_folder,
220
                input_doc_assets_dir_rel_path,
221
                output_document_dir_full_path,
222
                output_document_dir_rel_path,
223
            )
224
            document.assign_meta(document_meta)
225
 
226
            output_document_rel_path: SDocRelativePath = SDocRelativePath(
227
                os.path.join(
228
                    output_document_dir_rel_path.relative_path,
229
                    f"{document_filename_base}.html",
230
                )
231
            )
232
 
233
            map_docs_by_paths[input_doc_full_path] = document
234
            map_docs_by_rel_paths[output_document_rel_path.relative_path] = (
235
                document
236
            )
237
 
238
        return DocumentTree(
239
            file_trees,
240
            document_list,
241
            map_docs_by_paths,
242
            map_docs_by_rel_paths,
243
            map_grammars_by_filenames=map_grammars_by_filenames,
244
        )
245
 
246
    @staticmethod
247
    def _build_file_tree(
248
        project_config: ProjectConfig,
249
    ) -> Tuple[List[FileTree], AssetManager]:
250
        assert isinstance(project_config.input_paths, list)
251
        assert len(project_config.input_paths) > 0
252
 
253
        root_trees: List[FileTree] = []
254
        asset_manager = AssetManager()
255
 
256
        for path_to_doc_root_raw in project_config.input_paths:
257
            if os.path.isfile(path_to_doc_root_raw):
258
                path_to_doc_root = path_to_doc_root_raw
259
                if not os.path.isabs(path_to_doc_root):
260
                    path_to_doc_root = os.path.abspath(path_to_doc_root)
261
 
262
                parent_dir = os.path.dirname(path_to_doc_root)
263
                path_to_doc_root_base = os.path.dirname(parent_dir)
264
 
265
                assets_dir: str = os.path.join(parent_dir, "_assets")
266
                if os.path.isdir(assets_dir):
267
                    asset_manager.add_asset_dir(
268
                        full_path=assets_dir,
269
                        relative_path=SDocRelativePath(
270
                            os.path.relpath(assets_dir, path_to_doc_root_base)
271
                        ),
272
                    )
273
                root_trees.append(
274
                    FileTree.create_single_file_tree(path_to_doc_root)
275
                )
276
                continue
277
 
278
            # Strip away the trailing slash to let the later os.path.relpath
279
            # calculations work correctly.
280
            path_to_doc_root = path_to_doc_root_raw.rstrip("/")
281
            path_to_doc_root = os.path.abspath(path_to_doc_root)
282
            path_to_doc_root_base = os.path.dirname(path_to_doc_root)
283
 
284
            # Finding assets.
285
            with measure_performance("Find asset directories"):
286
                tree_asset_dirs: List[str] = PathFinder.find_directories(
287
                    path_to_doc_root,
288
                    "_assets",
289
                    include_paths=project_config.include_doc_paths,
290
                    exclude_paths=project_config.exclude_doc_paths,
291
                )
292
 
293
            for asset_dir_ in tree_asset_dirs:
294
                asset_manager.add_asset_dir(
295
                    full_path=asset_dir_,
296
                    relative_path=SDocRelativePath(
297
                        os.path.relpath(asset_dir_, path_to_doc_root_base)
298
                    ),
299
                )
300
 
301
            # Finding SDoc files.
302
            assert isinstance(project_config.output_dir, str)
303
            with measure_performance("Find SDoc files"):
304
                file_tree_structure = FileFinder.find_files_with_extensions(
305
                    root_path=path_to_doc_root,
306
                    ignored_dirs=[project_config.output_dir],
307
                    extensions=get_document_extensions(project_config),
308
                    include_paths=project_config.include_doc_paths,
309
                    exclude_paths=project_config.exclude_doc_paths,
310
                )
311
            root_trees.append(file_tree_structure)
312
 
313
        return root_trees, asset_manager