Path:
strictdoc/core/file_system/document_finder.py
Lines:
313
Non-empty lines:
269
Non-empty lines covered with requirements:
269 / 269 (100.0%)
Functions:
7
Functions covered by requirements:
7 / 7 (100.0%)
- "2.4. SDoc file extension" (REQUIREMENT)
- "5.1. Finding documents recursively" (REQUIREMENT)
1
"""2
@relation(SDOC-SRS-104, SDOC-SRS-115, scope=file)3
"""4
5
import os
6
import sys
7
from functools import partial
8
from typing import Dict, List, Tuple, Union
9
10
from strictdoc.backend.sdoc.models.document import SDocDocument
11
from strictdoc.backend.sdoc.models.document_grammar import DocumentGrammar
12
from strictdoc.core.asset_manager import AssetManager
13
from strictdoc.core.document_meta import DocumentMeta
14
from strictdoc.core.document_tree import DocumentTree
15
from strictdoc.core.file_system.file_tree import (
16
File,
17
FileFinder,
18
FileTree,
19
Folder,
20
PathFinder,
21
)22
from strictdoc.core.format import Format
23
from strictdoc.core.project_config import ProjectConfig
24
from strictdoc.helpers.exception import StrictDocException
25
from strictdoc.helpers.parallelizer import Parallelizer
26
from strictdoc.helpers.paths import SDocRelativePath
27
from strictdoc.helpers.textx import drop_textx_meta
28
from strictdoc.helpers.timing import measure_performance, timing_decorator
29
30
# Each entry: (extension, Format instance, is_grammar).31
# Sorted longest-extension-first so that e.g. ".gra.md" is matched before32
# ".md" (a "foo.gra.md" file ends with both).33
FormatDispatchEntry = Tuple[str, Format, bool]
34
35
36
def _build_extension_dispatch_table(
37
project_config: ProjectConfig,
38
) -> List[FormatDispatchEntry]:
39
entries: List[FormatDispatchEntry] = []
40
for format_ in project_config.formats:
41
if format_.supports_read():
42
for extension_ in format_.read_extensions():
43
entries.append((extension_, format_, False))
44
if format_.supports_grammar():
45
for extension_ in format_.grammar_extensions():
46
entries.append((extension_, format_, True))
47
entries.sort(key=lambda entry: len(entry[0]), reverse=True)
48
return entries
49
50
51
def get_document_extensions(project_config: ProjectConfig) -> List[str]:
52
"""
53
All extensions natively read (as documents or grammars) by the54
project's registered Formats. Used both for DocumentFinder's own file55
discovery and, in strictdoc/server/document_watcher.py, so a filesystem56
watcher can't drift out of sync with what DocumentFinder actually reads.57
"""58
return [
59
extension_60
for extension_, _, _ in _build_extension_dispatch_table(project_config)
61
]62
63
64
class DocumentFinder:
65
@staticmethod66
@timing_decorator("Find and read SDoc files")
67
def find_sdoc_content(
68
project_config: ProjectConfig, parallelizer: Parallelizer
69
) -> Tuple[DocumentTree, AssetManager]:
70
assert project_config.input_paths is not None
71
for paths_to_files_or_doc in project_config.input_paths:
72
if not os.path.exists(paths_to_files_or_doc):
73
sys.stdout.flush()
74
raise StrictDocException(
75
"error: "76
"Provided path is neither a single file or a folder: "77
f"'{paths_to_files_or_doc}'"
78
)79
80
with measure_performance("Completed finding SDoc and assets"):
81
file_tree, asset_manager = DocumentFinder._build_file_tree(
82
project_config=project_config
83
)84
with measure_performance("Completed building document tree"):
85
document_tree = DocumentFinder._build_document_tree(
86
file_tree, project_config, parallelizer
87
)88
89
return document_tree, asset_manager
90
91
@staticmethod92
def _process_worker_parse_document(
93
document_triple: Tuple[Union[Folder, File], File, str],
94
project_config: ProjectConfig,
95
) -> Tuple[File, str, Union[SDocDocument, DocumentGrammar]]:
96
_, doc_file, file_tree_mount_folder = document_triple
97
doc_full_path = doc_file.full_path
98
99
with measure_performance(
100
f"Reading SDOC: {os.path.basename(doc_full_path)}"
101
):102
document_or_grammar: Union[SDocDocument, DocumentGrammar, None] = (
103
None104
)105
106
# @relation(SDOC-SRS-104, scope=range_start)107
for (
108
extension_,
109
format_,
110
is_grammar_,
111
) in _build_extension_dispatch_table(project_config):
112
if not doc_full_path.endswith(extension_):
113
continue114
if is_grammar_:
115
document_or_grammar = format_.read_grammar(
116
doc_file, project_config
117
)118
assert isinstance(document_or_grammar, DocumentGrammar)
119
else:
120
document_or_grammar = format_.read_from_file(
121
doc_file, project_config
122
)123
assert isinstance(document_or_grammar, SDocDocument)
124
break125
# @relation(SDOC-SRS-104, scope=range_end)126
127
if document_or_grammar is None:
128
raise NotImplementedError
129
drop_textx_meta(document_or_grammar)
130
131
return doc_file, file_tree_mount_folder, document_or_grammar
132
133
@staticmethod- "6.1.4. Preserve generated file names" (REQUIREMENT)
134
def _build_document_tree(
135
file_trees: List[FileTree],
136
project_config: ProjectConfig,
137
parallelizer: Parallelizer,
138
) -> DocumentTree:
139
"""
140
@relation(SDOC-SRS-48, scope=function)141
"""142
143
assert isinstance(file_trees, list)
144
145
output_root_html = project_config.export_output_html_root
146
assert output_root_html is not None
147
148
document_list: List[SDocDocument] = []
149
map_docs_by_paths = {}
150
map_docs_by_rel_paths: Dict[str, SDocDocument] = {}
151
map_grammars_by_filenames = {}
152
153
file_tree_list: List[Tuple[Union[Folder, File], File, str]] = []
154
for file_tree in file_trees:
155
file_tree_list.extend(list(file_tree.iterate()))
156
157
process_document_binding = partial(
158
DocumentFinder._process_worker_parse_document,
159
project_config=project_config,
160
)161
162
with measure_performance("Completed parsing all documents"):
163
found_documents = parallelizer.run_parallel(
164
file_tree_list, process_document_binding
165
)166
167
doc_file: File
168
for doc_file, file_tree_mount_folder, document in found_documents:
169
assert isinstance(file_tree_mount_folder, str), (
170
file_tree_mount_folder171
)172
173
if isinstance(document, DocumentGrammar):
174
map_grammars_by_filenames[
175
doc_file.rel_path.relative_path_posix
176
] = document
177
continue178
179
input_doc_full_path: str = doc_file.full_path
180
map_docs_by_paths[input_doc_full_path] = document
181
document_list.append(document)
182
183
doc_relative_path_folder: SDocRelativePath = SDocRelativePath(
184
os.path.dirname(doc_file.rel_path.relative_path)
185
)186
output_document_dir_rel_path: SDocRelativePath = SDocRelativePath(
187
os.path.join(
188
file_tree_mount_folder,
189
doc_relative_path_folder.relative_path,
190
)191
if len(doc_relative_path_folder.relative_path) > 0
192
else file_tree_mount_folder
193
)194
195
document_filename = doc_file.file_name
196
document_filename_base = os.path.splitext(document_filename)[0]
197
198
output_document_dir_full_path: str = os.path.join(
199
output_root_html, output_document_dir_rel_path.relative_path
200
)201
202
input_doc_assets_dir_rel_path: SDocRelativePath = SDocRelativePath(
203
os.path.join(
204
file_tree_mount_folder,
205
doc_relative_path_folder.relative_path,
206
"_assets",
207
)208
if doc_relative_path_folder.length() > 0
209
else "/".join((file_tree_mount_folder, "_assets"))
210
)211
212
document_meta = DocumentMeta(
213
doc_file.level,
214
file_tree_mount_folder,
215
document_filename,
216
document_filename_base,
217
input_doc_full_path,
218
doc_file.rel_path,
219
doc_relative_path_folder,
220
input_doc_assets_dir_rel_path,
221
output_document_dir_full_path,
222
output_document_dir_rel_path,
223
)224
document.assign_meta(document_meta)
225
226
output_document_rel_path: SDocRelativePath = SDocRelativePath(
227
os.path.join(
228
output_document_dir_rel_path.relative_path,
229
f"{document_filename_base}.html",
230
)231
)232
233
map_docs_by_paths[input_doc_full_path] = document
234
map_docs_by_rel_paths[output_document_rel_path.relative_path] = (
235
document236
)237
238
return DocumentTree(
239
file_trees,
240
document_list,
241
map_docs_by_paths,
242
map_docs_by_rel_paths,
243
map_grammars_by_filenames=map_grammars_by_filenames,
244
)245
246
@staticmethod247
def _build_file_tree(
248
project_config: ProjectConfig,
249
) -> Tuple[List[FileTree], AssetManager]:
250
assert isinstance(project_config.input_paths, list)
251
assert len(project_config.input_paths) > 0
252
253
root_trees: List[FileTree] = []
254
asset_manager = AssetManager()
255
256
for path_to_doc_root_raw in project_config.input_paths:
257
if os.path.isfile(path_to_doc_root_raw):
258
path_to_doc_root = path_to_doc_root_raw
259
if not os.path.isabs(path_to_doc_root):
260
path_to_doc_root = os.path.abspath(path_to_doc_root)
261
262
parent_dir = os.path.dirname(path_to_doc_root)
263
path_to_doc_root_base = os.path.dirname(parent_dir)
264
265
assets_dir: str = os.path.join(parent_dir, "_assets")
266
if os.path.isdir(assets_dir):
267
asset_manager.add_asset_dir(
268
full_path=assets_dir,
269
relative_path=SDocRelativePath(
270
os.path.relpath(assets_dir, path_to_doc_root_base)
271
),272
)273
root_trees.append(
274
FileTree.create_single_file_tree(path_to_doc_root)
275
)276
continue277
278
# Strip away the trailing slash to let the later os.path.relpath279
# calculations work correctly.280
path_to_doc_root = path_to_doc_root_raw.rstrip("/")
281
path_to_doc_root = os.path.abspath(path_to_doc_root)
282
path_to_doc_root_base = os.path.dirname(path_to_doc_root)
283
284
# Finding assets.285
with measure_performance("Find asset directories"):
286
tree_asset_dirs: List[str] = PathFinder.find_directories(
287
path_to_doc_root,
288
"_assets",
289
include_paths=project_config.include_doc_paths,
290
exclude_paths=project_config.exclude_doc_paths,
291
)292
293
for asset_dir_ in tree_asset_dirs:
294
asset_manager.add_asset_dir(
295
full_path=asset_dir_,
296
relative_path=SDocRelativePath(
297
os.path.relpath(asset_dir_, path_to_doc_root_base)
298
),299
)300
301
# Finding SDoc files.302
assert isinstance(project_config.output_dir, str)
303
with measure_performance("Find SDoc files"):
304
file_tree_structure = FileFinder.find_files_with_extensions(
305
root_path=path_to_doc_root,
306
ignored_dirs=[project_config.output_dir],
307
extensions=get_document_extensions(project_config),
308
include_paths=project_config.include_doc_paths,
309
exclude_paths=project_config.exclude_doc_paths,
310
)311
root_trees.append(file_tree_structure)
312
313
return root_trees, asset_manager