Path:
tasks.py
Lines:
1387
Non-empty lines:
1208
Non-empty lines covered with requirements:
1208 / 1208 (100.0%)
Functions:
48
Functions covered by requirements:
48 / 48 (100.0%)
1
import os
2
import re
3
import shutil
4
import sys
5
import tempfile
6
from enum import Enum
7
from pathlib import Path
8
from typing import Dict, Optional
9
10
import invoke
11
from invoke import task
12
13
from developer.git.commit_validator import (
14
validate_commits_locally_or_ci,
15
)16
from strictdoc.core.environment import (
17
BINARY_HTML_STATIC_DIR,
18
BINARY_HTML_TEMPLATES_DIR,
19
HTML_STATIC_DIRS,
20
HTML_TEMPLATE_DIRS,
21
)22
from strictdoc.core.project_config import ProjectConfigDefault
23
from tools.server_check import free_strictdoc_port
24
25
# Specifying encoding because Windows crashes otherwise when running Invoke26
# tasks below:27
# UnicodeEncodeError: 'charmap' codec can't encode character '\ufffd'28
# in position 16: character maps to <undefined>29
# People say, it might also be possible to export PYTHONIOENCODING=utf8 but this30
# seems to work.31
# FIXME: If you are a Windows user and expert, please advise on how to do this32
# properly.33
sys.stdout = open(1, "w", encoding="utf-8", closefd=False, buffering=1)
34
35
STRICTDOC_TMP_DIR = os.path.join(tempfile.gettempdir(), "strictdoc_tmp_dir")
36
TEST_REPORTS_DIR = "build/test_reports"
37
38
# Redirect all __pycache__ output (from Invoke itself and every subprocess it39
# spawns: pytest, mypy, plain python invocations, etc.) into a single folder,40
# instead of littering __pycache__ directories across the source tree.41
os.environ["PYTHONPYCACHEPREFIX"] = os.path.abspath("build/pycache")
42
43
44
def get_pyinstaller_html_template_data_options() -> str:
45
return "\n".join(
46
f'--add-data "{template_dir}{os.pathsep}{BINARY_HTML_TEMPLATES_DIR}"'
47
for template_dir in HTML_TEMPLATE_DIRS
48
)49
50
51
def get_pyinstaller_html_static_data_options() -> str:
52
return "\n".join(
53
f'--add-data "{static_dir}{os.pathsep}{BINARY_HTML_STATIC_DIR}"'
54
for static_dir in HTML_STATIC_DIRS
55
)56
57
58
def get_nuitka_html_template_data_options() -> str:
59
return "\n".join(
60
f'--include-data-dir="{template_dir}={BINARY_HTML_TEMPLATES_DIR}"'
61
for template_dir in HTML_TEMPLATE_DIRS
62
)63
64
65
def get_nuitka_html_static_data_options() -> str:
66
return "\n".join(
67
f'--include-data-dir="{static_dir}={BINARY_HTML_STATIC_DIR}"'
68
for static_dir in HTML_STATIC_DIRS
69
)70
71
72
# To prevent all tasks from building to the same virtual environment.73
# All values correspond to the configuration in the tox.ini config file.74
class ToxEnvironment(str, Enum):
75
DEVELOPMENT = "development"
76
CHECK = "check"
77
DOCUMENTATION = "documentation"
78
RELEASE = "release"
79
RELEASE_LOCAL = "release-local"
80
PYINSTALLER = "pyinstaller"
81
82
83
def run_invoke(
84
context,
85
cmd,
86
environment: Optional[dict] = None,
87
pty: bool = False,
88
warn: bool = False,
89
) -> invoke.runners.Result:
90
def one_line_command(string):
91
return re.sub("\\s+", " ", string).strip()
92
93
return context.run(
94
one_line_command(cmd),
95
env=environment,
96
hide=False,
97
warn=warn,
98
pty=pty,
99
echo=True,
100
)101
102
103
def run_invoke_with_tox(
104
context,
105
environment_type: ToxEnvironment,
106
command: str,
107
environment: Optional[Dict] = None,
108
pty: bool = False,
109
) -> invoke.runners.Result:
110
assert isinstance(environment_type, ToxEnvironment)
111
assert isinstance(command, str)
112
113
tox_py_version = f"py{sys.version_info.major}{sys.version_info.minor}"
114
115
return run_invoke(
116
context,
117
f"""
118
tox119
-e {tox_py_version}-{environment_type.value} --
120
{command}
121
""",
122
environment=environment,
123
pty=pty,
124
)125
126
127
@task(default=True)
128
def list_tasks(context):
129
clean_command = """
130
invoke --list131
"""132
run_invoke(context, clean_command)
133
134
135
@task136
def clean(context):
137
# https://unix.stackexchange.com/a/689930/77389138
clean_command = r"""
139
rm -rf output/ docs/sphinx/build/ &&140
find tests/ -type d \( -name output -o -name Output \) -exec rm -rf {} +
141
"""142
run_invoke(context, clean_command)
143
144
145
@task(aliases=["s"])
146
def server(context, input_path=".", config=None, port=None):
147
assert os.path.isdir(input_path), input_path
148
if config is not None:
149
assert os.path.isfile(config), config
150
config_argument = f"--config {config}" if config is not None else ""
151
port_argument = f"--port {port}" if port is not None else ""
152
153
try:
154
should_continue = free_strictdoc_port(
155
port156
if port is not None
157
else ProjectConfigDefault.DEFAULT_SERVER_PORT
158
)159
except OSError as error:
160
print(error, file=sys.stderr) # noqa: T201
161
return162
163
if not should_continue:
164
return165
166
run_invoke_with_tox(
167
context,
168
ToxEnvironment.DEVELOPMENT,
169
f"""
170
python -m strictdoc.cli.main171
--debug172
--development173
server {input_path} {config_argument}
174
--host 127.0.0.1175
{port_argument}
176
--reload177
--watch178
""",
179
)180
181
182
@task(aliases=["scs"])
183
def screencast_server(context, focus=None, edit=False):
184
"""
185
Manual dev server for tests/screencast scenarios: starts StrictDoc on a186
scenario's project (the shared demo fixture by default, or another187
scenario's project via --focus), for inspecting it in the browser.188
189
By default this serves a disposable copy, rebuilt fresh every time, so190
nothing done through the UI persists. Pass --edit to serve the real,191
persistent files instead (the shared fixture itself, or a generated192
project reused across restarts) when intentionally editing them.193
"""194
195
focus_argument = f"--focus {focus}" if focus is not None else ""
196
edit_argument = "--edit" if edit else ""
197
198
run_invoke_with_tox(
199
context,
200
ToxEnvironment.CHECK,
201
f"python tests/screencast/run_server.py {focus_argument} {edit_argument}",
202
)203
204
205
@task(aliases=["d"])
206
def docs(context):
207
run_invoke_with_tox(
208
context,
209
ToxEnvironment.DOCUMENTATION,
210
"""
211
python3 -m strictdoc.cli.main212
export .213
--formats=html214
--output-dir output/strictdoc_website215
--project-title "StrictDoc"216
""",
217
)218
219
run_invoke_with_tox(
220
context,
221
ToxEnvironment.DOCUMENTATION,
222
"""
223
python3 -m strictdoc.cli.main224
export ./225
--formats=rst226
--output-dir output/sphinx227
--project-title "StrictDoc"228
""",
229
)230
231
run_invoke_with_tox(
232
context,
233
ToxEnvironment.DOCUMENTATION,
234
"""
235
cp -r output/sphinx/rst/docs/* docs/sphinx/source/ &&236
mkdir -p docs/sphinx/source/_assets/ &&237
cp -v docs/_assets/* docs/sphinx/source/_assets/238
""",
239
)240
241
run_invoke_with_tox(
242
context,
243
ToxEnvironment.DOCUMENTATION,
244
"""
245
make --directory docs/sphinx html latexpdf SPHINXOPTS="-W --keep-going"246
""",
247
)248
249
run_invoke(
250
context,
251
(252
"""
253
open docs/sphinx/build/latex/strictdoc.pdf254
"""255
),256
)257
258
259
@task(clean, aliases=["tus"])
260
def test_unit_server(context, focus=None):
261
focus_argument = f"-k {focus}" if focus is not None else ""
262
263
Path(TEST_REPORTS_DIR).mkdir(parents=True, exist_ok=True)
264
265
cwd = os.getcwd()
266
267
path_to_coverage_file = f"{cwd}/build/coverage/unit_server/.coverage"
268
269
run_invoke_with_tox(
270
context,
271
ToxEnvironment.CHECK,
272
f"""
273
coverage run274
--rcfile=.coveragerc.unit_server275
--data-file={path_to_coverage_file}
276
-m pytest277
tests/unit_server/278
{focus_argument}
279
--junit-xml={TEST_REPORTS_DIR}/tests_unit_server.pytest.junit.xml
280
-o junit_suite_name="StrictDoc Web Server Unit Tests"281
-o cache_dir=build/pytest_cache/unit_server282
""",
283
)284
285
286
@task(test_unit_server, aliases=["tusc"])
287
def test_unit_server_report(context):
288
cwd = os.getcwd()
289
290
path_to_coverage_file = f"{cwd}/build/coverage/unit_server/.coverage"
291
292
run_invoke_with_tox(
293
context,
294
ToxEnvironment.CHECK,
295
f"""
296
coverage html297
--rcfile=.coveragerc.unit_server298
--data-file={path_to_coverage_file}
299
""",
300
)301
302
303
@task(aliases=["te"])
304
def test_end2end(
305
context,
306
*,
307
focus=None,
308
exit_first=False,
309
parallelize=False,
310
long_timeouts=False,
311
headless=False,
312
headed=False,
313
shard=None,
314
test_path=None,
315
coverage: bool = False,
316
):317
"""
318
@relation(SDOC-SRS-46, scope=function)319
"""320
321
environment = {}
322
323
coverage_command_or_none = ""
324
coverage_argument_or_none = ""
325
326
if coverage:
327
cwd = os.getcwd()
328
coverage_file_dir = f"{cwd}/build/coverage/end2end/"
329
coverage_file_dir2 = f"{cwd}/build/coverage/end2end_strictdoc/"
330
coverage_file = os.path.join(coverage_file_dir, ".coverage")
331
coverage_rc = os.path.join(cwd, ".coveragerc.end2end")
332
shutil.rmtree(coverage_file_dir, ignore_errors=True)
333
shutil.rmtree(coverage_file_dir2, ignore_errors=True)
334
coverage_command_or_none = f"""
335
coverage run336
--rcfile={coverage_rc}
337
--data-file={coverage_file}
338
-m339
"""340
coverage_argument_or_none = "--strictdoc-coverage"
341
342
long_timeouts_argument = (
343
"--strictdoc-long-timeouts" if long_timeouts else ""
344
)345
346
parallelize_argument = ""
347
if parallelize:
348
print( # noqa: T201
349
"warning: "350
"Running parallelized end-2-end tests is supported "351
"but is not stable."352
)353
parallelize_argument = "--numprocesses=2 --strictdoc-parallelize"
354
355
assert shard is None or re.match(r"[1-9][0-9]*/[1-9][0-9]*", shard), (
356
f"--shard argument has an incorrect format: {shard}."
357
)358
shard_argument = f"--strictdoc-shard={shard}" if shard else ""
359
360
focus_argument = f"-k {focus}" if focus is not None else ""
361
exit_first_argument = "--exitfirst" if exit_first else ""
362
headless_argument = "--headless2" if headless and not headed else "--gui"
363
test_command = f"""
364
{coverage_command_or_none}
365
pytest366
--failed-first367
--capture=no368
--reuse-session369
{parallelize_argument}
370
{shard_argument}
371
{coverage_argument_or_none}
372
{focus_argument}
373
{exit_first_argument}
374
{long_timeouts_argument}
375
{headless_argument}
376
--junit-xml={TEST_REPORTS_DIR}/tests_end2end.pytest.junit.xml
377
-o junit_suite_name="StrictDoc End-to-End Tests"378
-o cache_dir=build/pytest_cache/end2end379
tests/end2end380
"""381
if test_path:
382
test_command = test_command.rstrip() + f"/{test_path}"
383
384
Path(TEST_REPORTS_DIR).mkdir(parents=True, exist_ok=True)
385
386
# On Windows, GitHub Actions fails with:387
# response = {'status': 500, 'value':388
# '{"value":{"error":"unknown error",389
# "message":"unknown error: cannot find Chrome binary", # noqa: ERA001390
# This very likely has to do with PATH isolation that Tox does.391
# FIXME: If you are a Windows expert, please fix this to run on Tox.392
if os.name == "nt":
393
run_invoke(context, test_command)
394
return395
396
run_invoke_with_tox(
397
context,
398
ToxEnvironment.CHECK,
399
test_command,
400
environment=environment,
401
)402
403
404
@task(aliases=["tsc"])
405
def test_screencast(context, *, focus=None, record_video=False):
406
"""
407
Runs the tests/screencast scenarios: fast pass/fail checks by default,408
or (re)recording of the corresponding .webm videos with --record-video.409
"""410
411
focus_argument = f"-k {focus}" if focus is not None else ""
412
record_video_argument = "--strictdoc-record-video" if record_video else ""
413
414
Path(TEST_REPORTS_DIR).mkdir(parents=True, exist_ok=True)
415
416
test_command = f"""
417
pytest418
--capture=no419
{focus_argument}
420
{record_video_argument}
421
--junit-xml={TEST_REPORTS_DIR}/tests_screencast.pytest.junit.xml
422
-o junit_suite_name="StrictDoc Screencast Tests"423
-o cache_dir=build/pytest_screencast424
tests/screencast/scenarios425
"""426
427
run_invoke_with_tox(context, ToxEnvironment.CHECK, test_command)
428
429
430
@task(aliases=["scov"])
431
def screencast_optimize_video(context, focus=None):
432
"""
433
Converts recorded tests/screencast/output/*.webm into muted,434
web-optimized .webm (VP9) and .mp4 (H.264) pairs under435
tests/screencast/output/web/, ready to hand off to the product website.436
437
Run `invoke test-screencast --record-video` first to produce the raw438
recordings this reads from.439
"""440
441
focus_argument = f"--focus {focus}" if focus is not None else ""
442
443
run_invoke(
444
context, f"python tests/screencast/optimize_video.py {focus_argument}"
445
)446
447
448
@task(aliases=["tu"])
449
def test_unit(context, coverage=False, focus=None, path=None, output=False):
450
"""
451
@relation(SDOC-SRS-44, scope=function)452
"""453
454
Path(TEST_REPORTS_DIR).mkdir(parents=True, exist_ok=True)
455
456
focus_argument = f"-k {focus}" if focus is not None else ""
457
output_argument = "--capture=no" if output else ""
458
459
cwd = os.getcwd()
460
461
if path is None:
462
path = "tests/unit"
463
else:
464
assert "tests/unit" in path, path
465
466
path_to_coverage_file = f"{cwd}/build/coverage/unit/.coverage"
467
468
pytest_command = (
469
"""
470
coverage run471
--rcfile=.coveragerc.unit472
--data-file={path_to_coverage_file}473
-m pytest474
"""475
if coverage
476
else "pytest"
477
)478
479
run_invoke_with_tox(
480
context,
481
ToxEnvironment.CHECK,
482
f"""
483
{pytest_command}
484
{focus_argument}
485
{output_argument}
486
--junit-xml={TEST_REPORTS_DIR}/tests_unit.pytest.junit.xml
487
-o cache_dir=build/pytest_cache/unit488
-o junit_suite_name="StrictDoc Unit Tests"489
-p no:seleniumbase490
{path}
491
""",
492
)493
if coverage and not focus and path == "tests/unit":
494
run_invoke_with_tox(
495
context,
496
ToxEnvironment.CHECK,
497
f"""
498
coverage report499
--sort=cover500
--rcfile=.coveragerc.unit501
--data-file={path_to_coverage_file}
502
""",
503
)504
505
506
@task(test_unit, aliases=["tuc"])
507
def test_unit_report(context):
508
cwd = os.getcwd()
509
510
path_to_coverage_file = f"{cwd}/build/coverage/unit/.coverage"
511
512
run_invoke_with_tox(
513
context,
514
ToxEnvironment.CHECK,
515
f"""
516
coverage html517
--rcfile=.coveragerc.unit518
--data-file={path_to_coverage_file}
519
""",
520
)521
522
523
@task(clean, aliases=["ti"])
- "15.8.2. CLI interface black-box integration testing" (REQUIREMENT)
524
def test_integration(
525
context,
526
*,
527
focus=None,
528
debug=False,
529
no_parallelization=False,
530
fail_first=False,
531
coverage=False,
532
strictdoc=None,
533
html2pdf=False,
534
shard=None,
535
environment=ToxEnvironment.CHECK,
536
):537
"""
538
@relation(SDOC-SRS-45, scope=function)539
"""540
541
cwd = os.getcwd()
542
543
if strictdoc is None:
544
strictdoc_exec = "python3 -m strictdoc.cli.main"
545
else:
546
strictdoc_exec = strictdoc
547
548
coverage_path_argument = ""
549
if coverage:
550
path_to_coverage_rc = f"{cwd}/.coveragerc.integration"
551
strictdoc_exec = (
552
f"coverage run --rcfile={path_to_coverage_rc} -m strictdoc.cli.main"
553
)554
if html2pdf:
555
path_to_coverage_dir = f"{cwd}/build/coverage/integration_html2pdf/"
556
else:
557
path_to_coverage_dir = f"{cwd}/build/coverage/integration/"
558
path_to_coverage = os.path.join(path_to_coverage_dir, ".coverage")
559
shutil.rmtree(path_to_coverage_dir, ignore_errors=True)
560
coverage_path_argument = (
561
f'--param COVERAGE_FILE="{path_to_coverage}" '
562
f'--param COVERAGE_PROCESS_START="{path_to_coverage_rc}"'
563
)564
565
debug_opts = "-vv --show-all" if debug else ""
566
focus_or_none = f"--filter {focus}" if focus else ""
567
fail_first_argument = "--max-failures 1" if fail_first else ""
568
junit_xml_report_argument = (
569
"--xunit-xml-output build/test_reports/tests_integration_html2pdf.lit.junit.xml"570
if html2pdf
571
else "--xunit-xml-output build/test_reports/tests_integration.lit.junit.xml"
572
)573
574
# Allow partitioning of integration and html2pdf tests575
partition_opts = ""
576
if shard is not None:
577
match = re.match(r"([1-9][0-9]*)/([1-9][0-9]*)", shard)
578
assert match, f"--shard argument has an incorrect format: {shard}."
579
run_shard = int(match.group(1))
580
num_shards = int(match.group(2))
581
partition_opts = f"--num-shards={num_shards} --run-shard={run_shard}"
582
583
# HTML2PDF tests are running Chrome Driver which does not seem to be584
# parallelizable, or at least not in the way StrictDoc uses it.585
# If HTML2PDF option is provided, do not parallelize and only run the586
# HTML2PDF-specific tests.587
# HTML2PDF tests can be safely partitioned.588
chromedriver_param = ""
589
if not html2pdf:
590
parallelize_opts = "" if not no_parallelization else "--threads 1"
591
html2pdf_param = ""
592
test_folder = f"{cwd}/tests/integration"
593
test_output_dir = "build/tests_integration"
594
else:
595
parallelize_opts = "--threads 1"
596
html2pdf_param = "--param TEST_HTML2PDF=1"
597
chromedriver_path = os.environ.get("CHROMEWEBDRIVER")
598
if chromedriver_path is not None:
599
# NOTE: isfile() check does not work on GitHub Actions / Linux,600
# the exists() check works.601
assert os.path.exists(chromedriver_path), chromedriver_path
602
chromedriver_param = f"--param CHROMEDRIVER={os.path.join(chromedriver_path, 'chromedriver')}"
603
if os.name == "nt":
604
# On Windows, its chromdriver.exe605
chromedriver_param = chromedriver_param + ".exe"
606
test_folder = f"{cwd}/tests/integration/features/html2pdf"
607
test_output_dir = "build/tests_integration_html2pdf"
608
609
# The command sometimes exits with 1 even if the files are deleted.610
# warn=True ensures that the execution continues.611
run_invoke(
612
context,
613
f"""
614
rm -rf {test_output_dir}
615
""",
616
warn=True,
617
)618
619
run_invoke(
620
context,
621
f"""
622
rm -rf {STRICTDOC_TMP_DIR}
623
""",
624
)625
626
Path(STRICTDOC_TMP_DIR).mkdir(exist_ok=True)
627
Path(TEST_REPORTS_DIR).mkdir(parents=True, exist_ok=True)
628
629
itest_command = f"""
630
lit631
--param STRICTDOC_EXEC="{strictdoc_exec}"
632
--param STRICTDOC_TMP_DIR="{STRICTDOC_TMP_DIR}"
633
--param TEST_OUTPUT_DIR="{test_output_dir}"
634
--timeout 180635
--order smart636
{junit_xml_report_argument}
637
{coverage_path_argument}
638
{html2pdf_param}
639
{chromedriver_param}
640
-v641
{debug_opts}
642
{focus_or_none}
643
{fail_first_argument}
644
{parallelize_opts}
645
{partition_opts}
646
{test_folder}
647
"""648
649
# It looks like LIT does not open the RUN: subprocesses in the same650
# environment from which it itself is run from. This issue has been known by651
# us for a couple of years by now. Not using Tox on Windows for the time652
# being.653
if os.name == "nt":
654
run_invoke(context, itest_command)
655
return656
657
run_invoke_with_tox(
658
context,
659
environment,
660
itest_command,
661
environment={"STRICTDOC_CACHE_DIR": "Output/_cache"},
662
)663
664
665
@task666
def coverage_combine(context):
667
run_invoke_with_tox(
668
context,
669
ToxEnvironment.CHECK,
670
"""
671
coverage combine672
--data-file build/coverage/.coverage.combined673
--keep674
build/coverage/end2end_strictdoc/.coverage.*675
build/coverage/integration/.coverage.*676
build/coverage/integration_html2pdf/.coverage.*677
build/coverage/unit/.coverage678
build/coverage/unit_server/.coverage679
""",
680
)681
run_invoke_with_tox(
682
context,
683
ToxEnvironment.CHECK,
684
"""
685
coverage html686
--rcfile .coveragerc.combined687
--data-file build/coverage/.coverage.combined688
""",
689
)690
run_invoke_with_tox(
691
context,
692
ToxEnvironment.CHECK,
693
"""
694
coverage json695
--rcfile .coveragerc.combined696
--data-file build/coverage/.coverage.combined697
--pretty-print698
-o build/coverage/coverage.combined.json699
""",
700
)701
702
703
@task- "15.6.1. Compliance with Python community practices (PEP8 etc)" (REQUIREMENT)
704
def lint_ruff_format(context):
705
"""
706
@relation(SDOC-SRS-42, scope=function)707
"""708
709
result: invoke.runners.Result = run_invoke_with_tox(
710
context,
711
ToxEnvironment.CHECK,
712
"""
713
ruff714
format715
--cache-dir build/ruff_cache716
*.py717
developer/718
docs/719
strictdoc/720
tools/ecss721
tests/unit/722
tests/unit_server/723
tests/integration/*.py724
tests/end2end/725
""",
726
)727
# Ruff always exits with 0, so we handle the output.728
if "reformatted" in result.stdout:
729
print("invoke: ruff format found issues") # noqa: T201
730
result.exited = 1
731
raise invoke.exceptions.UnexpectedExit(result)
732
733
734
@task(aliases=["lr"])
- "15.6.1. Compliance with Python community practices (PEP8 etc)" (REQUIREMENT)
735
def lint_ruff(context):
736
"""
737
@relation(SDOC-SRS-42, scope=function)738
"""739
740
run_invoke_with_tox(
741
context,
742
ToxEnvironment.CHECK,
743
"""
744
ruff check . --fix --exit-non-zero-on-fix --cache-dir build/ruff_cache745
""",
746
)747
748
749
@task(aliases=["lm"])
- "15.5.2. Use of type annotations in Python code" (REQUIREMENT)
- "15.7.1. Static type checking" (REQUIREMENT)
750
def lint_mypy(context):
751
"""
752
@relation(SDOC-SRS-41, SDOC-SRS-43, scope=function)753
"""754
755
# These checks do not seem to be useful:756
# - import757
# --disallow-any-expr758
# --disallow-any-explicit759
# --disallow-any-unimported # noqa: ERA001760
# --disallow-any-decorated761
# - type-abstract. It is ignored on purpose because of assert_cast()762
# implementation. See https://stackoverflow.com/a/74073453/598057.763
run_invoke_with_tox(
764
context,
765
ToxEnvironment.CHECK,
766
"""
767
mypy docs/768
strictdoc/769
tests/unit/strictdoc/backend/sdoc_source_code/test_marker_lexer.py770
771
--show-error-codes772
--disable-error-code=import773
--disable-error-code=type-abstract774
--cache-dir=build/mypy_cache775
--extra-checks776
777
--strict778
--strict-optional779
--strict-equality780
781
--check-untyped-defs782
--disallow-any-generics783
--disallow-incomplete-defs784
--disallow-subclassing-any785
--disallow-untyped-calls786
--disallow-untyped-decorators787
--disallow-untyped-defs788
--no-implicit-optional789
--warn-no-return790
--warn-redundant-casts791
--warn-return-any792
--warn-unreachable793
--warn-unused-ignores794
795
--python-version=3.10796
""",
797
)798
799
800
@task801
def lint_format_js(context):
802
# NOTE: Could not find the '--' equivalent for -w80.803
result: invoke.runners.Result = run_invoke_with_tox(
804
context,
805
ToxEnvironment.CHECK,
806
"""
807
js-beautify808
--indent-size=2809
--end-with-newline810
--replace811
-w100812
strictdoc/export/html/_static/autocompletable_field.js813
strictdoc/export/html/_static/copy_to_clipboard.js814
strictdoc/export/html/_static/deletable_field.js815
strictdoc/export/html/_static/draggable_list.js816
strictdoc/export/html/_static/editable_field.js817
strictdoc/export/html/_static/modal.js818
strictdoc/export/html/_static/movable_field.js819
strictdoc/export/html/_static/scroll_into_view.js820
strictdoc/export/html/_static/static_html_search.js821
strictdoc/export/html/_static/tabs.js822
strictdoc/features/project_index/assets/stable_uri_forwarder.js823
""",
824
)825
# Ruff always exits with 0, so we handle the output.826
if "reformatted" in result.stdout:
827
print("invoke: ruff format found issues") # noqa: T201
828
result.exited = 1
829
raise invoke.exceptions.UnexpectedExit(result)
830
831
832
@task(aliases=["lc"])
833
def lint_commit(context): # noqa: ARG001
834
try:
835
validate_commits_locally_or_ci()
836
except ValueError as e:
837
raise invoke.exceptions.Exit(message=str(e), code=1) from None
838
839
840
@task(aliases=["lf"])
841
def lint_fixit(context, fix=False, auto=False, path="strictdoc/"):
842
if fix:
843
auto_argument = "--automatic" if auto else ""
844
run_invoke_with_tox(
845
context,
846
ToxEnvironment.CHECK,
847
f"""
848
fixit fix {path} {auto_argument}
849
""",
850
pty=True,
851
)852
else:
853
run_invoke_with_tox(
854
context,
855
ToxEnvironment.CHECK,
856
f"""
857
fixit lint --diff {path}
858
""",
859
)860
861
862
@task(aliases=["l"])
863
def lint(context):
864
lint_commit(context)
865
lint_ruff_format(context)
866
lint_ruff(context)
867
lint_mypy(context)
868
869
870
@task(aliases=["t"])
871
def test(context, shard=None):
872
test_unit(context)
873
test_unit_server(context)
874
test_integration(context, shard=shard)
875
876
877
@task(aliases=["ta"])
878
def test_all(context, coverage=False, headless=False):
879
test_unit(context, coverage=coverage)
880
test_unit_server(context)
881
test_integration(context, coverage=coverage)
882
test_integration(context, coverage=coverage, html2pdf=True)
883
test_end2end(context, coverage=coverage, headless=headless)
884
885
886
@task(aliases=["c"])
887
def check(context):
888
lint(context)
889
test(context)
890
891
892
# https://github.com/github-changelog-generator/github-changelog-generator893
# gem install github_changelog_generator894
@task895
def changelog(context, github_token):
896
# The alpha release tags are excluded from the changelog.897
command = f"""
898
github_changelog_generator899
--token {github_token}
900
--user strictdoc-project901
--exclude-tags-regex ".*a\\d+"
902
--project strictdoc903
"""904
run_invoke(context, command)
905
906
907
@task908
def check_dead_links(context):
909
run_invoke_with_tox(
910
context,
911
ToxEnvironment.CHECK,
912
"""
913
python3 tools/link_health.py docs/strictdoc_01_user_guide.sdoc914
""",
915
)916
run_invoke_with_tox(
917
context,
918
ToxEnvironment.CHECK,
919
"""
920
python3 tools/link_health.py docs/strictdoc_02_feature_map.sdoc921
""",
922
)923
run_invoke_with_tox(
924
context,
925
ToxEnvironment.CHECK,
926
"""
927
python3 tools/link_health.py docs/strictdoc_03_faq.sdoc928
""",
929
)930
run_invoke_with_tox(
931
context,
932
ToxEnvironment.CHECK,
933
"""
934
python3 tools/link_health.py docs/strictdoc_04_release_notes.sdoc935
""",
936
)937
run_invoke_with_tox(
938
context,
939
ToxEnvironment.CHECK,
940
"""
941
python3 tools/link_health.py docs/strictdoc_05_troubleshooting.sdoc942
""",
943
)944
run_invoke_with_tox(
945
context,
946
ToxEnvironment.CHECK,
947
"""
948
python3 tools/link_health.py docs/strictdoc_10_contributing.sdoc949
""",
950
)951
run_invoke_with_tox(
952
context,
953
ToxEnvironment.CHECK,
954
"""
955
python3 tools/link_health.py docs/strictdoc_11_developer_guide.sdoc956
""",
957
)958
run_invoke_with_tox(
959
context,
960
ToxEnvironment.CHECK,
961
"""
962
python3 tools/link_health.py docs/strictdoc_24_development_plan.sdoc963
""",
964
)965
run_invoke_with_tox(
966
context,
967
ToxEnvironment.CHECK,
968
"""
969
python3 tools/link_health.py docs/strictdoc_20_l1_system_requirements.sdoc970
""",
971
)972
run_invoke_with_tox(
973
context,
974
ToxEnvironment.CHECK,
975
"""
976
python3 tools/link_health.py docs/strictdoc_21_l2_high_level_requirements.sdoc977
""",
978
)979
run_invoke_with_tox(
980
context,
981
ToxEnvironment.CHECK,
982
"""
983
python3 tools/link_health.py docs/strictdoc_25_design.sdoc984
""",
985
)986
run_invoke_with_tox(
987
context,
988
ToxEnvironment.CHECK,
989
"""
990
python3 tools/link_health.py CONTRIBUTING.md991
""",
992
)993
run_invoke_with_tox(
994
context,
995
ToxEnvironment.CHECK,
996
"""
997
python3 tools/link_health.py NOTICE998
""",
999
)1000
run_invoke_with_tox(
1001
context,
1002
ToxEnvironment.CHECK,
1003
"""
1004
python3 tools/link_health.py README.md1005
""",
1006
)1007
1008
1009
@task1010
def release_local(context):
1011
run_invoke(
1012
context,
1013
"""
1014
rm -rfv build/1015
""",
1016
)1017
run_invoke(
1018
context,
1019
"""
1020
pip uninstall strictdoc -y1021
""",
1022
)1023
run_invoke_with_tox(
1024
context,
1025
ToxEnvironment.RELEASE_LOCAL,
1026
"""
1027
python -m build --outdir build/dist1028
""",
1029
)1030
run_invoke_with_tox(
1031
context,
1032
ToxEnvironment.RELEASE_LOCAL,
1033
"""
1034
twine check build/dist/*1035
""",
1036
)1037
run_invoke_with_tox(
1038
context,
1039
ToxEnvironment.RELEASE_LOCAL,
1040
"""
1041
pip install build/dist/*.tar.gz1042
""",
1043
)1044
test_integration(
1045
context, strictdoc="strictdoc", environment=ToxEnvironment.RELEASE_LOCAL
1046
)1047
1048
1049
@task1050
def release(context, test_pypi=False, username=None, password=None):
1051
"""
1052
A release can be made to PyPI or test package index (TestPyPI):1053
https://pypi.org/project/strictdoc/1054
https://test.pypi.org/project/strictdoc/1055
"""1056
1057
env_user = os.environ.get("TWINE_USERNAME")
1058
env_pass = os.environ.get("TWINE_PASSWORD")
1059
1060
assert not ((username or password) and (env_user or env_pass)), (
1061
username,
1062
password,
1063
env_user,
1064
env_pass,
1065
)1066
assert (username and password) or (env_user and env_pass), (
1067
username,
1068
password,
1069
env_user,
1070
env_pass,
1071
)1072
if env_user:
1073
assert env_user == "__token__"
1074
1075
repository_argument_or_none = ""
1076
if username is not None and password is not None:
1077
repository_argument_or_none = (
1078
""1079
if username
1080
else (
1081
"--repository strictdoc_test"1082
if test_pypi
1083
else "--repository strictdoc_release"
1084
)1085
)1086
user_password = f"-u{username} -p{password}" if username is not None else ""
1087
1088
run_invoke(
1089
context,
1090
"""
1091
rm -rfv build/dist/1092
""",
1093
)1094
run_invoke_with_tox(
1095
context,
1096
ToxEnvironment.RELEASE,
1097
"""
1098
python3 -m build --outdir build/dist1099
""",
1100
)1101
run_invoke_with_tox(
1102
context,
1103
ToxEnvironment.RELEASE,
1104
"""
1105
twine check build/dist/*1106
""",
1107
)1108
# The token is in a core developer's .pypirc file.1109
# https://test.pypi.org/manage/account/token/1110
# https://packaging.python.org/en/latest/specifications/pypirc/#pypirc1111
run_invoke_with_tox(
1112
context,
1113
ToxEnvironment.RELEASE,
1114
f"""
1115
twine upload build/dist/strictdoc-*.tar.gz build/dist/strictdoc-*.whl1116
{repository_argument_or_none}
1117
{user_password}
1118
""",
1119
)1120
1121
1122
@task1123
def release_pyinstaller(context):
1124
path_to_pyi_dist = "/tmp/strictdoc"
1125
html_template_data_options = get_pyinstaller_html_template_data_options()
1126
html_static_data_options = get_pyinstaller_html_static_data_options()
1127
1128
# The --hidden-import strictdoc.server.app flag is needed because without1129
# it, the following is produced:1130
# ERROR: Error loading ASGI app. Could not import1131
# module "strictdoc.server.app".1132
# Solution found here: https://stackoverflow.com/a/71340437/5980571133
# This behavior is not surprising because that's how the uvicorn loads the1134
# application separately from the parent process.1135
#1136
# Compatibility modules can be imported by user-provided statistics1137
# generators at runtime. PyInstaller cannot discover these imports1138
# statically because the generators live outside of StrictDoc's package.1139
#1140
# --hidden-import strictdoc.api is needed for the same reason:1141
# strictdoc.api is never imported anywhere inside StrictDoc's own1142
# package (only by dynamically-loaded, external files such as a1143
# project's own strictdoc_config.py, custom statistics generators, or1144
# custom plugins), so PyInstaller's static analysis -- which walks1145
# imports starting from strictdoc/cli/main.py -- never discovers it on1146
# its own.1147
command = f"""
1148
pyinstaller1149
--clean1150
--name strictdoc1151
--noconfirm1152
--additional-hooks-dir developer/pyinstaller_hooks1153
--distpath {path_to_pyi_dist}
1154
--hidden-import strictdoc.api1155
--hidden-import strictdoc.backend.rst.strictdoc_lexer1156
--hidden-import strictdoc.core.statistics.metric1157
--hidden-import strictdoc.export.html.generators.project_statistics1158
--hidden-import strictdoc.export.html.generators.view_objects.project_statistics_view_object1159
--hidden-import strictdoc.export.html.generators.view_objects.project_tree_stats1160
--hidden-import strictdoc.server.app1161
{html_template_data_options}
1162
{html_static_data_options}
1163
--add-data strictdoc/backend/rst/templates:templates/rst1164
strictdoc/cli/main.py1165
"""1166
1167
run_invoke_with_tox(
1168
context,
1169
ToxEnvironment.PYINSTALLER,
1170
"""
1171
pyinstaller --version1172
""",
1173
)1174
1175
run_invoke_with_tox(context, ToxEnvironment.PYINSTALLER, command)
1176
1177
1178
@task1179
def watch(context, sdocs_path="."):
1180
strictdoc_command = f"""
1181
python -m strictdoc.cli.main1182
export1183
{sdocs_path}
1184
--output-dir output/1185
"""1186
1187
run_invoke_with_tox(
1188
context,
1189
ToxEnvironment.DEVELOPMENT,
1190
f"""
1191
{strictdoc_command}
1192
""",
1193
)1194
1195
paths_to_watch = "."
1196
run_invoke_with_tox(
1197
context,
1198
ToxEnvironment.DEVELOPMENT,
1199
f"""
1200
watchmedo shell-command1201
--patterns="*.py;*.sdoc;*.jinja;*.html;*.css;*.js"1202
--recursive1203
--ignore-pattern='output/;tests/integration'1204
--command='{strictdoc_command}'
1205
--drop1206
{paths_to_watch}
1207
""",
1208
)1209
1210
1211
@task1212
def run(context, command):
1213
run_invoke_with_tox(
1214
context,
1215
ToxEnvironment.DEVELOPMENT,
1216
f"""
1217
{command}
1218
""",
1219
)1220
1221
1222
@task1223
def nuitka(context):
1224
html_template_data_options = get_nuitka_html_template_data_options()
1225
html_static_data_options = get_nuitka_html_static_data_options()
1226
1227
run_invoke(
1228
context,
1229
f"""
1230
PYTHONPATH="{os.getcwd()}"
1231
python -m nuitka1232
--static-libpython=no1233
--standalone1234
--include-module=textx1235
--include-module=strictdoc.server.app1236
--include-module=docutils1237
--include-module=docutils.readers.standalone1238
--include-module=docutils.parsers.rst1239
{html_template_data_options}
1240
{html_static_data_options}
1241
--include-data-dir=strictdoc/backend/rst/templates=templates/rst1242
--include-package-data=docutils1243
strictdoc/cli/main.py1244
""",
1245
)1246
1247
1248
# https://github.com/jrfonseca/gprof2dot1249
# pip install gprof2dot1250
@task()
1251
def performance(context):
1252
command = """
1253
python -m cProfile -o output/profile.prof1254
-m strictdoc.cli.main export . --no-parallelization &&1255
gprof2dot -f pstats output/profile.prof | dot -Tpng -o output/output.png1256
"""1257
run_invoke(context, command)
1258
1259
1260
@task(performance)
1261
def performance_snakeviz(context):
1262
command = """
1263
snakeviz output/profile.prof1264
"""1265
run_invoke(context, command)
1266
1267
1268
@task(aliases=["bd"])
1269
def build_docker(
1270
context,
1271
image: str = "strictdoc:latest",
1272
no_cache: bool = False,
1273
source="pypi",
1274
):1275
no_cache_argument = "--no-cache" if no_cache else ""
1276
run_invoke(
1277
context,
1278
f"""
1279
docker build .1280
--build-arg STRICTDOC_SOURCE={source}
1281
-t {image}
1282
{no_cache_argument}
1283
""",
1284
)1285
1286
1287
@task(aliases=["rd"])
1288
def run_docker(
1289
context, image: str = "strictdoc:latest", command: Optional[str] = None
1290
):1291
command_argument = (
1292
f'/bin/bash -c "{command}"' if command is not None else ""
1293
)1294
1295
run_invoke(
1296
context,
1297
f"""
1298
docker run1299
--name strictdoc1300
--rm1301
-it1302
-e HOST_UID=$(id -u) -e HOST_GID=$(id -g)1303
-v "$(pwd):/data"1304
{image}
1305
{command_argument}
1306
""",
1307
pty=True,
1308
)1309
1310
1311
@task(aliases=["td"])
1312
def test_docker(context, image: str = "strictdoc:latest"):
1313
run_invoke(
1314
context,
1315
"""
1316
rm -rf output/ && mkdir -p output/ && chmod 777 output/1317
""",
1318
)1319
run_docker(
1320
context,
1321
image=image,
1322
command="strictdoc export --formats=html,html2pdf .",
1323
)1324
1325
def check_file_owner(filepath):
1326
import pwd # noqa: PLC0415
1327
1328
file_owner = pwd.getpwuid(os.stat(filepath).st_uid).pw_name
1329
current_user = os.environ.get("USER", "")
1330
return file_owner == current_user
1331
1332
assert check_file_owner(
1333
"output/html2pdf/pdf/docs/strictdoc_01_user_guide.pdf"1334
)1335
1336
1337
@task(aliases=["q"])
1338
def qualification(context):
1339
test_all(context, coverage=True, headless=True)
1340
coverage_combine(context)
1341
1342
1343
@task()
1344
def drawio(context):
1345
if sys.platform == "darwin":
1346
path_to_drawio = "/Applications/draw.io.app/Contents/MacOS/draw.io"
1347
elif sys.platform.startswith("linux"):
1348
path_to_drawio = "drawio"
1349
else:
1350
raise NotImplementedError(
1351
"drawio task is supported only on macOS and Linux."1352
)1353
1354
artifacts = [
1355
(1356
"developer/drawio/Architecture.drawio",
1357
"docs/_assets/StrictDoc_Workspace-Architecture.drawio.png",
1358
),1359
(1360
"developer/drawio/Backlog.drawio",
1361
"docs/_assets/StrictDoc_Workspace-Backlog.drawio.png",
1362
),1363
(1364
"developer/drawio/Roadmap.drawio",
1365
"docs/_assets/StrictDoc_Workspace-Roadmap.drawio.png",
1366
),1367
]1368
1369
for path_to_drawio_, path_to_png_ in artifacts:
1370
print(f"Copying: {path_to_drawio_} -> {path_to_png_}") # noqa: T201
1371
1372
# Basic safety for now to avoid writing wrong files.1373
assert os.path.isfile(path_to_drawio_), path_to_drawio_
1374
assert os.path.isfile(path_to_png_), path_to_png_
1375
1376
run_invoke(
1377
context,
1378
f"""
1379
{path_to_drawio}
1380
--export1381
--format png1382
-o {path_to_png_}
1383
--page-index 01384
{path_to_drawio_}
1385
""",
1386
pty=True,
1387
)