Path:
tasks.py
Lines:
1310
Non-empty lines:
1147
Non-empty lines covered with requirements:
1147 / 1147 (100.0%)
Functions:
45
Functions covered by requirements:
45 / 45 (100.0%)
1
import os
2
import re
3
import shutil
4
import sys
5
import tempfile
6
from enum import Enum
7
from pathlib import Path
8
from typing import Dict, Optional
9
10
import invoke
11
from invoke import task
12
13
from developer.git.commit_validator import (
14
validate_commits_locally_or_ci,
15
)16
from strictdoc.core.environment import (
17
BINARY_HTML_STATIC_DIR,
18
BINARY_HTML_TEMPLATES_DIR,
19
HTML_STATIC_DIRS,
20
HTML_TEMPLATE_DIRS,
21
)22
from strictdoc.core.project_config import ProjectConfigDefault
23
from tools.server_check import free_strictdoc_port
24
25
# Specifying encoding because Windows crashes otherwise when running Invoke26
# tasks below:27
# UnicodeEncodeError: 'charmap' codec can't encode character '\ufffd'28
# in position 16: character maps to <undefined>29
# People say, it might also be possible to export PYTHONIOENCODING=utf8 but this30
# seems to work.31
# FIXME: If you are a Windows user and expert, please advise on how to do this32
# properly.33
sys.stdout = open(1, "w", encoding="utf-8", closefd=False, buffering=1)
34
35
STRICTDOC_TMP_DIR = os.path.join(tempfile.gettempdir(), "strictdoc_tmp_dir")
36
TEST_REPORTS_DIR = "build/test_reports"
37
38
# Redirect all __pycache__ output (from Invoke itself and every subprocess it39
# spawns: pytest, mypy, plain python invocations, etc.) into a single folder,40
# instead of littering __pycache__ directories across the source tree.41
os.environ["PYTHONPYCACHEPREFIX"] = os.path.abspath("build/pycache")
42
43
44
def get_pyinstaller_html_template_data_options() -> str:
45
return "\n".join(
46
f'--add-data "{template_dir}{os.pathsep}{BINARY_HTML_TEMPLATES_DIR}"'
47
for template_dir in HTML_TEMPLATE_DIRS
48
)49
50
51
def get_pyinstaller_html_static_data_options() -> str:
52
return "\n".join(
53
f'--add-data "{static_dir}{os.pathsep}{BINARY_HTML_STATIC_DIR}"'
54
for static_dir in HTML_STATIC_DIRS
55
)56
57
58
def get_nuitka_html_template_data_options() -> str:
59
return "\n".join(
60
f'--include-data-dir="{template_dir}={BINARY_HTML_TEMPLATES_DIR}"'
61
for template_dir in HTML_TEMPLATE_DIRS
62
)63
64
65
def get_nuitka_html_static_data_options() -> str:
66
return "\n".join(
67
f'--include-data-dir="{static_dir}={BINARY_HTML_STATIC_DIR}"'
68
for static_dir in HTML_STATIC_DIRS
69
)70
71
72
# To prevent all tasks from building to the same virtual environment.73
# All values correspond to the configuration in the tox.ini config file.74
class ToxEnvironment(str, Enum):
75
DEVELOPMENT = "development"
76
CHECK = "check"
77
DOCUMENTATION = "documentation"
78
RELEASE = "release"
79
RELEASE_LOCAL = "release-local"
80
PYINSTALLER = "pyinstaller"
81
82
83
def run_invoke(
84
context,
85
cmd,
86
environment: Optional[dict] = None,
87
pty: bool = False,
88
warn: bool = False,
89
) -> invoke.runners.Result:
90
def one_line_command(string):
91
return re.sub("\\s+", " ", string).strip()
92
93
return context.run(
94
one_line_command(cmd),
95
env=environment,
96
hide=False,
97
warn=warn,
98
pty=pty,
99
echo=True,
100
)101
102
103
def run_invoke_with_tox(
104
context,
105
environment_type: ToxEnvironment,
106
command: str,
107
environment: Optional[Dict] = None,
108
pty: bool = False,
109
) -> invoke.runners.Result:
110
assert isinstance(environment_type, ToxEnvironment)
111
assert isinstance(command, str)
112
113
tox_py_version = f"py{sys.version_info.major}{sys.version_info.minor}"
114
115
return run_invoke(
116
context,
117
f"""
118
tox119
-e {tox_py_version}-{environment_type.value} --
120
{command}
121
""",
122
environment=environment,
123
pty=pty,
124
)125
126
127
@task(default=True)
128
def list_tasks(context):
129
clean_command = """
130
invoke --list131
"""132
run_invoke(context, clean_command)
133
134
135
@task136
def clean(context):
137
# https://unix.stackexchange.com/a/689930/77389138
clean_command = r"""
139
rm -rf output/ docs/sphinx/build/ &&140
find tests/ -type d \( -name output -o -name Output \) -exec rm -rf {} +
141
"""142
run_invoke(context, clean_command)
143
144
145
@task(aliases=["s"])
146
def server(context, input_path=".", config=None, port=None):
147
assert os.path.isdir(input_path), input_path
148
if config is not None:
149
assert os.path.isfile(config), config
150
config_argument = f"--config {config}" if config is not None else ""
151
port_argument = f"--port {port}" if port is not None else ""
152
153
try:
154
should_continue = free_strictdoc_port(
155
port156
if port is not None
157
else ProjectConfigDefault.DEFAULT_SERVER_PORT
158
)159
except OSError as error:
160
print(error, file=sys.stderr) # noqa: T201
161
return162
163
if not should_continue:
164
return165
166
run_invoke_with_tox(
167
context,
168
ToxEnvironment.DEVELOPMENT,
169
f"""
170
python -m strictdoc.cli.main171
--debug172
--development173
server {input_path} {config_argument}
174
--host 127.0.0.1175
{port_argument}
176
--reload177
--watch178
""",
179
)180
181
182
@task(aliases=["d"])
183
def docs(context):
184
run_invoke_with_tox(
185
context,
186
ToxEnvironment.DOCUMENTATION,
187
"""
188
python3 -m strictdoc.cli.main189
export .190
--formats=html191
--output-dir output/strictdoc_website192
--project-title "StrictDoc"193
""",
194
)195
196
run_invoke_with_tox(
197
context,
198
ToxEnvironment.DOCUMENTATION,
199
"""
200
python3 -m strictdoc.cli.main201
export ./202
--formats=rst203
--output-dir output/sphinx204
--project-title "StrictDoc"205
""",
206
)207
208
run_invoke_with_tox(
209
context,
210
ToxEnvironment.DOCUMENTATION,
211
"""
212
cp -r output/sphinx/rst/docs/* docs/sphinx/source/ &&213
mkdir -p docs/sphinx/source/_assets/ &&214
cp -v docs/_assets/* docs/sphinx/source/_assets/215
""",
216
)217
218
run_invoke_with_tox(
219
context,
220
ToxEnvironment.DOCUMENTATION,
221
"""
222
make --directory docs/sphinx html latexpdf SPHINXOPTS="-W --keep-going"223
""",
224
)225
226
run_invoke(
227
context,
228
(229
"""
230
open docs/sphinx/build/latex/strictdoc.pdf231
"""232
),233
)234
235
236
@task(clean, aliases=["tus"])
237
def test_unit_server(context, focus=None):
238
focus_argument = f"-k {focus}" if focus is not None else ""
239
240
Path(TEST_REPORTS_DIR).mkdir(parents=True, exist_ok=True)
241
242
cwd = os.getcwd()
243
244
path_to_coverage_file = f"{cwd}/build/coverage/unit_server/.coverage"
245
246
run_invoke_with_tox(
247
context,
248
ToxEnvironment.CHECK,
249
f"""
250
coverage run251
--rcfile=.coveragerc.unit_server252
--data-file={path_to_coverage_file}
253
-m pytest254
tests/unit_server/255
{focus_argument}
256
--junit-xml={TEST_REPORTS_DIR}/tests_unit_server.pytest.junit.xml
257
-o junit_suite_name="StrictDoc Web Server Unit Tests"258
-o cache_dir=build/pytest_cache/unit_server259
""",
260
)261
262
263
@task(test_unit_server, aliases=["tusc"])
264
def test_unit_server_report(context):
265
cwd = os.getcwd()
266
267
path_to_coverage_file = f"{cwd}/build/coverage/unit_server/.coverage"
268
269
run_invoke_with_tox(
270
context,
271
ToxEnvironment.CHECK,
272
f"""
273
coverage html274
--rcfile=.coveragerc.unit_server275
--data-file={path_to_coverage_file}
276
""",
277
)278
279
280
@task(aliases=["te"])
281
def test_end2end(
282
context,
283
*,
284
focus=None,
285
exit_first=False,
286
parallelize=False,
287
long_timeouts=False,
288
headless=False,
289
headed=False,
290
shard=None,
291
test_path=None,
292
coverage: bool = False,
293
):294
"""
295
@relation(SDOC-SRS-46, scope=function)296
"""297
298
environment = {}
299
300
coverage_command_or_none = ""
301
coverage_argument_or_none = ""
302
303
if coverage:
304
cwd = os.getcwd()
305
coverage_file_dir = f"{cwd}/build/coverage/end2end/"
306
coverage_file_dir2 = f"{cwd}/build/coverage/end2end_strictdoc/"
307
coverage_file = os.path.join(coverage_file_dir, ".coverage")
308
coverage_rc = os.path.join(cwd, ".coveragerc.end2end")
309
shutil.rmtree(coverage_file_dir, ignore_errors=True)
310
shutil.rmtree(coverage_file_dir2, ignore_errors=True)
311
coverage_command_or_none = f"""
312
coverage run313
--rcfile={coverage_rc}
314
--data-file={coverage_file}
315
-m316
"""317
coverage_argument_or_none = "--strictdoc-coverage"
318
319
long_timeouts_argument = (
320
"--strictdoc-long-timeouts" if long_timeouts else ""
321
)322
323
parallelize_argument = ""
324
if parallelize:
325
print( # noqa: T201
326
"warning: "327
"Running parallelized end-2-end tests is supported "328
"but is not stable."329
)330
parallelize_argument = "--numprocesses=2 --strictdoc-parallelize"
331
332
assert shard is None or re.match(r"[1-9][0-9]*/[1-9][0-9]*", shard), (
333
f"--shard argument has an incorrect format: {shard}."
334
)335
shard_argument = f"--strictdoc-shard={shard}" if shard else ""
336
337
focus_argument = f"-k {focus}" if focus is not None else ""
338
exit_first_argument = "--exitfirst" if exit_first else ""
339
headless_argument = "--headless2" if headless and not headed else "--gui"
340
test_command = f"""
341
{coverage_command_or_none}
342
pytest343
--failed-first344
--capture=no345
--reuse-session346
{parallelize_argument}
347
{shard_argument}
348
{coverage_argument_or_none}
349
{focus_argument}
350
{exit_first_argument}
351
{long_timeouts_argument}
352
{headless_argument}
353
--junit-xml={TEST_REPORTS_DIR}/tests_end2end.pytest.junit.xml
354
-o junit_suite_name="StrictDoc End-to-End Tests"355
-o cache_dir=build/pytest_cache/end2end356
tests/end2end357
"""358
if test_path:
359
test_command = test_command.rstrip() + f"/{test_path}"
360
361
Path(TEST_REPORTS_DIR).mkdir(parents=True, exist_ok=True)
362
363
# On Windows, GitHub Actions fails with:364
# response = {'status': 500, 'value':365
# '{"value":{"error":"unknown error",366
# "message":"unknown error: cannot find Chrome binary", # noqa: ERA001367
# This very likely has to do with PATH isolation that Tox does.368
# FIXME: If you are a Windows expert, please fix this to run on Tox.369
if os.name == "nt":
370
run_invoke(context, test_command)
371
return372
373
run_invoke_with_tox(
374
context,
375
ToxEnvironment.CHECK,
376
test_command,
377
environment=environment,
378
)379
380
381
@task(aliases=["tu"])
382
def test_unit(context, coverage=False, focus=None, path=None, output=False):
383
"""
384
@relation(SDOC-SRS-44, scope=function)385
"""386
387
Path(TEST_REPORTS_DIR).mkdir(parents=True, exist_ok=True)
388
389
focus_argument = f"-k {focus}" if focus is not None else ""
390
output_argument = "--capture=no" if output else ""
391
392
cwd = os.getcwd()
393
394
if path is None:
395
path = "tests/unit"
396
else:
397
assert "tests/unit" in path, path
398
399
path_to_coverage_file = f"{cwd}/build/coverage/unit/.coverage"
400
401
pytest_command = (
402
"""
403
coverage run404
--rcfile=.coveragerc.unit405
--data-file={path_to_coverage_file}406
-m pytest407
"""408
if coverage
409
else "pytest"
410
)411
412
run_invoke_with_tox(
413
context,
414
ToxEnvironment.CHECK,
415
f"""
416
{pytest_command}
417
{focus_argument}
418
{output_argument}
419
--junit-xml={TEST_REPORTS_DIR}/tests_unit.pytest.junit.xml
420
-o cache_dir=build/pytest_cache/unit421
-o junit_suite_name="StrictDoc Unit Tests"422
-p no:seleniumbase423
{path}
424
""",
425
)426
if coverage and not focus and path == "tests/unit":
427
run_invoke_with_tox(
428
context,
429
ToxEnvironment.CHECK,
430
f"""
431
coverage report432
--sort=cover433
--rcfile=.coveragerc.unit434
--data-file={path_to_coverage_file}
435
""",
436
)437
438
439
@task(test_unit, aliases=["tuc"])
440
def test_unit_report(context):
441
cwd = os.getcwd()
442
443
path_to_coverage_file = f"{cwd}/build/coverage/unit/.coverage"
444
445
run_invoke_with_tox(
446
context,
447
ToxEnvironment.CHECK,
448
f"""
449
coverage html450
--rcfile=.coveragerc.unit451
--data-file={path_to_coverage_file}
452
""",
453
)454
455
456
@task(clean, aliases=["ti"])
- "15.8.2. CLI interface black-box integration testing" (REQUIREMENT)
457
def test_integration(
458
context,
459
focus=None,
460
debug=False,
461
no_parallelization=False,
462
fail_first=False,
463
coverage=False,
464
strictdoc=None,
465
html2pdf=False,
466
shard=None,
467
environment=ToxEnvironment.CHECK,
468
):469
"""
470
@relation(SDOC-SRS-45, scope=function)471
"""472
473
cwd = os.getcwd()
474
475
if strictdoc is None:
476
strictdoc_exec = "python3 -m strictdoc.cli.main"
477
else:
478
strictdoc_exec = strictdoc
479
480
coverage_path_argument = ""
481
if coverage:
482
path_to_coverage_rc = f"{cwd}/.coveragerc.integration"
483
strictdoc_exec = (
484
f"coverage run --rcfile={path_to_coverage_rc} -m strictdoc.cli.main"
485
)486
if html2pdf:
487
path_to_coverage_dir = f"{cwd}/build/coverage/integration_html2pdf/"
488
else:
489
path_to_coverage_dir = f"{cwd}/build/coverage/integration/"
490
path_to_coverage = os.path.join(path_to_coverage_dir, ".coverage")
491
shutil.rmtree(path_to_coverage_dir, ignore_errors=True)
492
coverage_path_argument = (
493
f'--param COVERAGE_FILE="{path_to_coverage}" '
494
f'--param COVERAGE_PROCESS_START="{path_to_coverage_rc}"'
495
)496
497
debug_opts = "-vv --show-all" if debug else ""
498
focus_or_none = f"--filter {focus}" if focus else ""
499
fail_first_argument = "--max-failures 1" if fail_first else ""
500
junit_xml_report_argument = (
501
"--xunit-xml-output build/test_reports/tests_integration_html2pdf.lit.junit.xml"502
if html2pdf
503
else "--xunit-xml-output build/test_reports/tests_integration.lit.junit.xml"
504
)505
506
# Allow partitioning of integration and html2pdf tests507
partition_opts = ""
508
if shard is not None:
509
match = re.match(r"([1-9][0-9]*)/([1-9][0-9]*)", shard)
510
assert match, f"--shard argument has an incorrect format: {shard}."
511
run_shard = int(match.group(1))
512
num_shards = int(match.group(2))
513
partition_opts = f"--num-shards={num_shards} --run-shard={run_shard}"
514
515
# HTML2PDF tests are running Chrome Driver which does not seem to be516
# parallelizable, or at least not in the way StrictDoc uses it.517
# If HTML2PDF option is provided, do not parallelize and only run the518
# HTML2PDF-specific tests.519
# HTML2PDF tests can be safely partitioned.520
chromedriver_param = ""
521
if not html2pdf:
522
parallelize_opts = "" if not no_parallelization else "--threads 1"
523
html2pdf_param = ""
524
test_folder = f"{cwd}/tests/integration"
525
test_output_dir = "build/tests_integration"
526
else:
527
parallelize_opts = "--threads 1"
528
html2pdf_param = "--param TEST_HTML2PDF=1"
529
chromedriver_path = os.environ.get("CHROMEWEBDRIVER")
530
if chromedriver_path is not None:
531
# NOTE: isfile() check does not work on GitHub Actions / Linux,532
# the exists() check works.533
assert os.path.exists(chromedriver_path), chromedriver_path
534
chromedriver_param = f"--param CHROMEDRIVER={os.path.join(chromedriver_path, 'chromedriver')}"
535
if os.name == "nt":
536
# On Windows, its chromdriver.exe537
chromedriver_param = chromedriver_param + ".exe"
538
test_folder = f"{cwd}/tests/integration/features/html2pdf"
539
test_output_dir = "build/tests_integration_html2pdf"
540
541
# The command sometimes exits with 1 even if the files are deleted.542
# warn=True ensures that the execution continues.543
run_invoke(
544
context,
545
f"""
546
rm -rf {test_output_dir}
547
""",
548
warn=True,
549
)550
551
run_invoke(
552
context,
553
f"""
554
rm -rf {STRICTDOC_TMP_DIR}
555
""",
556
)557
558
Path(STRICTDOC_TMP_DIR).mkdir(exist_ok=True)
559
Path(TEST_REPORTS_DIR).mkdir(parents=True, exist_ok=True)
560
561
itest_command = f"""
562
lit563
--param STRICTDOC_EXEC="{strictdoc_exec}"
564
--param STRICTDOC_TMP_DIR="{STRICTDOC_TMP_DIR}"
565
--param TEST_OUTPUT_DIR="{test_output_dir}"
566
--timeout 180567
--order smart568
{junit_xml_report_argument}
569
{coverage_path_argument}
570
{html2pdf_param}
571
{chromedriver_param}
572
-v573
{debug_opts}
574
{focus_or_none}
575
{fail_first_argument}
576
{parallelize_opts}
577
{partition_opts}
578
{test_folder}
579
"""580
581
# It looks like LIT does not open the RUN: subprocesses in the same582
# environment from which it itself is run from. This issue has been known by583
# us for a couple of years by now. Not using Tox on Windows for the time584
# being.585
if os.name == "nt":
586
run_invoke(context, itest_command)
587
return588
589
run_invoke_with_tox(
590
context,
591
environment,
592
itest_command,
593
environment={"STRICTDOC_CACHE_DIR": "Output/_cache"},
594
)595
596
597
@task598
def coverage_combine(context):
599
run_invoke_with_tox(
600
context,
601
ToxEnvironment.CHECK,
602
"""
603
coverage combine604
--data-file build/coverage/.coverage.combined605
--keep606
build/coverage/end2end_strictdoc/.coverage.*607
build/coverage/integration/.coverage.*608
build/coverage/integration_html2pdf/.coverage.*609
build/coverage/unit/.coverage610
build/coverage/unit_server/.coverage611
""",
612
)613
run_invoke_with_tox(
614
context,
615
ToxEnvironment.CHECK,
616
"""
617
coverage html618
--rcfile .coveragerc.combined619
--data-file build/coverage/.coverage.combined620
""",
621
)622
run_invoke_with_tox(
623
context,
624
ToxEnvironment.CHECK,
625
"""
626
coverage json627
--rcfile .coveragerc.combined628
--data-file build/coverage/.coverage.combined629
--pretty-print630
-o build/coverage/coverage.combined.json631
""",
632
)633
634
635
@task- "15.6.1. Compliance with Python community practices (PEP8 etc)" (REQUIREMENT)
636
def lint_ruff_format(context):
637
"""
638
@relation(SDOC-SRS-42, scope=function)639
"""640
641
result: invoke.runners.Result = run_invoke_with_tox(
642
context,
643
ToxEnvironment.CHECK,
644
"""
645
ruff646
format647
--cache-dir build/ruff_cache648
*.py649
developer/650
docs/651
strictdoc/652
tools/ecss653
tests/unit/654
tests/unit_server/655
tests/integration/*.py656
tests/end2end/657
""",
658
)659
# Ruff always exits with 0, so we handle the output.660
if "reformatted" in result.stdout:
661
print("invoke: ruff format found issues") # noqa: T201
662
result.exited = 1
663
raise invoke.exceptions.UnexpectedExit(result)
664
665
666
@task(aliases=["lr"])
- "15.6.1. Compliance with Python community practices (PEP8 etc)" (REQUIREMENT)
667
def lint_ruff(context):
668
"""
669
@relation(SDOC-SRS-42, scope=function)670
"""671
672
run_invoke_with_tox(
673
context,
674
ToxEnvironment.CHECK,
675
"""
676
ruff check . --fix --exit-non-zero-on-fix --cache-dir build/ruff_cache677
""",
678
)679
680
681
@task(aliases=["lm"])
- "15.5.2. Use of type annotations in Python code" (REQUIREMENT)
- "15.7.1. Static type checking" (REQUIREMENT)
682
def lint_mypy(context):
683
"""
684
@relation(SDOC-SRS-41, SDOC-SRS-43, scope=function)685
"""686
687
# These checks do not seem to be useful:688
# - import689
# --disallow-any-expr690
# --disallow-any-explicit691
# --disallow-any-unimported # noqa: ERA001692
# --disallow-any-decorated693
# - type-abstract. It is ignored on purpose because of assert_cast()694
# implementation. See https://stackoverflow.com/a/74073453/598057.695
run_invoke_with_tox(
696
context,
697
ToxEnvironment.CHECK,
698
"""
699
mypy docs/700
strictdoc/701
tests/unit/strictdoc/backend/sdoc_source_code/test_marker_lexer.py702
703
--show-error-codes704
--disable-error-code=import705
--disable-error-code=type-abstract706
--cache-dir=build/mypy_cache707
--extra-checks708
709
--strict710
--strict-optional711
--strict-equality712
713
--check-untyped-defs714
--disallow-any-generics715
--disallow-incomplete-defs716
--disallow-subclassing-any717
--disallow-untyped-calls718
--disallow-untyped-decorators719
--disallow-untyped-defs720
--no-implicit-optional721
--warn-no-return722
--warn-redundant-casts723
--warn-return-any724
--warn-unreachable725
--warn-unused-ignores726
727
--python-version=3.10728
""",
729
)730
731
732
@task733
def lint_format_js(context):
734
# NOTE: Could not find the '--' equivalent for -w80.735
result: invoke.runners.Result = run_invoke_with_tox(
736
context,
737
ToxEnvironment.CHECK,
738
"""
739
js-beautify740
--indent-size=2741
--end-with-newline742
--replace743
-w100744
strictdoc/export/html/_static/static_html_search.js745
strictdoc/export/html/_static/stable_uri_forwarder.js746
""",
747
)748
# Ruff always exits with 0, so we handle the output.749
if "reformatted" in result.stdout:
750
print("invoke: ruff format found issues") # noqa: T201
751
result.exited = 1
752
raise invoke.exceptions.UnexpectedExit(result)
753
754
755
@task(aliases=["lc"])
756
def lint_commit(context): # noqa: ARG001
757
try:
758
validate_commits_locally_or_ci()
759
except ValueError as e:
760
raise invoke.exceptions.Exit(message=str(e), code=1) from None
761
762
763
@task(aliases=["lf"])
764
def lint_fixit(context, fix=False, auto=False, path="strictdoc/"):
765
if fix:
766
auto_argument = "--automatic" if auto else ""
767
run_invoke_with_tox(
768
context,
769
ToxEnvironment.CHECK,
770
f"""
771
fixit fix {path} {auto_argument}
772
""",
773
pty=True,
774
)775
else:
776
run_invoke_with_tox(
777
context,
778
ToxEnvironment.CHECK,
779
f"""
780
fixit lint --diff {path}
781
""",
782
)783
784
785
@task(aliases=["l"])
786
def lint(context):
787
lint_commit(context)
788
lint_ruff_format(context)
789
lint_ruff(context)
790
lint_mypy(context)
791
792
793
@task(aliases=["t"])
794
def test(context, shard=None):
795
test_unit(context)
796
test_unit_server(context)
797
test_integration(context, shard=shard)
798
799
800
@task(aliases=["ta"])
801
def test_all(context, coverage=False, headless=False):
802
test_unit(context, coverage=coverage)
803
test_unit_server(context)
804
test_integration(context, coverage=coverage)
805
test_integration(context, coverage=coverage, html2pdf=True)
806
test_end2end(context, coverage=coverage, headless=headless)
807
808
809
@task(aliases=["c"])
810
def check(context):
811
lint(context)
812
test(context)
813
814
815
# https://github.com/github-changelog-generator/github-changelog-generator816
# gem install github_changelog_generator817
@task818
def changelog(context, github_token):
819
# The alpha release tags are excluded from the changelog.820
command = f"""
821
github_changelog_generator822
--token {github_token}
823
--user strictdoc-project824
--exclude-tags-regex ".*a\\d+"
825
--project strictdoc826
"""827
run_invoke(context, command)
828
829
830
@task831
def check_dead_links(context):
832
run_invoke_with_tox(
833
context,
834
ToxEnvironment.CHECK,
835
"""
836
python3 tools/link_health.py docs/strictdoc_01_user_guide.sdoc837
""",
838
)839
run_invoke_with_tox(
840
context,
841
ToxEnvironment.CHECK,
842
"""
843
python3 tools/link_health.py docs/strictdoc_02_feature_map.sdoc844
""",
845
)846
run_invoke_with_tox(
847
context,
848
ToxEnvironment.CHECK,
849
"""
850
python3 tools/link_health.py docs/strictdoc_03_faq.sdoc851
""",
852
)853
run_invoke_with_tox(
854
context,
855
ToxEnvironment.CHECK,
856
"""
857
python3 tools/link_health.py docs/strictdoc_04_release_notes.sdoc858
""",
859
)860
run_invoke_with_tox(
861
context,
862
ToxEnvironment.CHECK,
863
"""
864
python3 tools/link_health.py docs/strictdoc_05_troubleshooting.sdoc865
""",
866
)867
run_invoke_with_tox(
868
context,
869
ToxEnvironment.CHECK,
870
"""
871
python3 tools/link_health.py docs/strictdoc_10_contributing.sdoc872
""",
873
)874
run_invoke_with_tox(
875
context,
876
ToxEnvironment.CHECK,
877
"""
878
python3 tools/link_health.py docs/strictdoc_11_developer_guide.sdoc879
""",
880
)881
run_invoke_with_tox(
882
context,
883
ToxEnvironment.CHECK,
884
"""
885
python3 tools/link_health.py docs/strictdoc_24_development_plan.sdoc886
""",
887
)888
run_invoke_with_tox(
889
context,
890
ToxEnvironment.CHECK,
891
"""
892
python3 tools/link_health.py docs/strictdoc_20_l1_system_requirements.sdoc893
""",
894
)895
run_invoke_with_tox(
896
context,
897
ToxEnvironment.CHECK,
898
"""
899
python3 tools/link_health.py docs/strictdoc_21_l2_high_level_requirements.sdoc900
""",
901
)902
run_invoke_with_tox(
903
context,
904
ToxEnvironment.CHECK,
905
"""
906
python3 tools/link_health.py docs/strictdoc_25_design.sdoc907
""",
908
)909
run_invoke_with_tox(
910
context,
911
ToxEnvironment.CHECK,
912
"""
913
python3 tools/link_health.py CONTRIBUTING.md914
""",
915
)916
run_invoke_with_tox(
917
context,
918
ToxEnvironment.CHECK,
919
"""
920
python3 tools/link_health.py NOTICE921
""",
922
)923
run_invoke_with_tox(
924
context,
925
ToxEnvironment.CHECK,
926
"""
927
python3 tools/link_health.py README.md928
""",
929
)930
931
932
@task933
def release_local(context):
934
run_invoke(
935
context,
936
"""
937
rm -rfv build/938
""",
939
)940
run_invoke(
941
context,
942
"""
943
pip uninstall strictdoc -y944
""",
945
)946
run_invoke_with_tox(
947
context,
948
ToxEnvironment.RELEASE_LOCAL,
949
"""
950
python -m build --outdir build/dist951
""",
952
)953
run_invoke_with_tox(
954
context,
955
ToxEnvironment.RELEASE_LOCAL,
956
"""
957
twine check build/dist/*958
""",
959
)960
run_invoke_with_tox(
961
context,
962
ToxEnvironment.RELEASE_LOCAL,
963
"""
964
pip install build/dist/*.tar.gz965
""",
966
)967
test_integration(
968
context, strictdoc="strictdoc", environment=ToxEnvironment.RELEASE_LOCAL
969
)970
971
972
@task973
def release(context, test_pypi=False, username=None, password=None):
974
"""
975
A release can be made to PyPI or test package index (TestPyPI):976
https://pypi.org/project/strictdoc/977
https://test.pypi.org/project/strictdoc/978
"""979
980
env_user = os.environ.get("TWINE_USERNAME")
981
env_pass = os.environ.get("TWINE_PASSWORD")
982
983
assert not ((username or password) and (env_user or env_pass)), (
984
username,
985
password,
986
env_user,
987
env_pass,
988
)989
assert (username and password) or (env_user and env_pass), (
990
username,
991
password,
992
env_user,
993
env_pass,
994
)995
if env_user:
996
assert env_user == "__token__"
997
998
repository_argument_or_none = ""
999
if username is not None and password is not None:
1000
repository_argument_or_none = (
1001
""1002
if username
1003
else (
1004
"--repository strictdoc_test"1005
if test_pypi
1006
else "--repository strictdoc_release"
1007
)1008
)1009
user_password = f"-u{username} -p{password}" if username is not None else ""
1010
1011
run_invoke(
1012
context,
1013
"""
1014
rm -rfv build/dist/1015
""",
1016
)1017
run_invoke_with_tox(
1018
context,
1019
ToxEnvironment.RELEASE,
1020
"""
1021
python3 -m build --outdir build/dist1022
""",
1023
)1024
run_invoke_with_tox(
1025
context,
1026
ToxEnvironment.RELEASE,
1027
"""
1028
twine check build/dist/*1029
""",
1030
)1031
# The token is in a core developer's .pypirc file.1032
# https://test.pypi.org/manage/account/token/1033
# https://packaging.python.org/en/latest/specifications/pypirc/#pypirc1034
run_invoke_with_tox(
1035
context,
1036
ToxEnvironment.RELEASE,
1037
f"""
1038
twine upload build/dist/strictdoc-*.tar.gz build/dist/strictdoc-*.whl1039
{repository_argument_or_none}
1040
{user_password}
1041
""",
1042
)1043
1044
1045
@task1046
def release_pyinstaller(context):
1047
path_to_pyi_dist = "/tmp/strictdoc"
1048
html_template_data_options = get_pyinstaller_html_template_data_options()
1049
html_static_data_options = get_pyinstaller_html_static_data_options()
1050
1051
# The --hidden-import strictdoc.server.app flag is needed because without1052
# it, the following is produced:1053
# ERROR: Error loading ASGI app. Could not import1054
# module "strictdoc.server.app".1055
# Solution found here: https://stackoverflow.com/a/71340437/5980571056
# This behavior is not surprising because that's how the uvicorn loads the1057
# application separately from the parent process.1058
#1059
# Compatibility modules can be imported by user-provided statistics1060
# generators at runtime. PyInstaller cannot discover these imports1061
# statically because the generators live outside of StrictDoc's package.1062
#1063
# --hidden-import strictdoc.api is needed for the same reason:1064
# strictdoc.api is never imported anywhere inside StrictDoc's own1065
# package (only by dynamically-loaded, external files such as a1066
# project's own strictdoc_config.py, custom statistics generators, or1067
# custom plugins), so PyInstaller's static analysis -- which walks1068
# imports starting from strictdoc/cli/main.py -- never discovers it on1069
# its own.1070
command = f"""
1071
pyinstaller1072
--clean1073
--name strictdoc1074
--noconfirm1075
--additional-hooks-dir developer/pyinstaller_hooks1076
--distpath {path_to_pyi_dist}
1077
--hidden-import strictdoc.api1078
--hidden-import strictdoc.backend.rst.strictdoc_lexer1079
--hidden-import strictdoc.core.statistics.metric1080
--hidden-import strictdoc.export.html.generators.project_statistics1081
--hidden-import strictdoc.export.html.generators.view_objects.project_statistics_view_object1082
--hidden-import strictdoc.export.html.generators.view_objects.project_tree_stats1083
--hidden-import strictdoc.server.app1084
{html_template_data_options}
1085
{html_static_data_options}
1086
--add-data strictdoc/backend/rst/templates:templates/rst1087
strictdoc/cli/main.py1088
"""1089
1090
run_invoke_with_tox(
1091
context,
1092
ToxEnvironment.PYINSTALLER,
1093
"""
1094
pyinstaller --version1095
""",
1096
)1097
1098
run_invoke_with_tox(context, ToxEnvironment.PYINSTALLER, command)
1099
1100
1101
@task1102
def watch(context, sdocs_path="."):
1103
strictdoc_command = f"""
1104
python -m strictdoc.cli.main1105
export1106
{sdocs_path}
1107
--output-dir output/1108
"""1109
1110
run_invoke_with_tox(
1111
context,
1112
ToxEnvironment.DEVELOPMENT,
1113
f"""
1114
{strictdoc_command}
1115
""",
1116
)1117
1118
paths_to_watch = "."
1119
run_invoke_with_tox(
1120
context,
1121
ToxEnvironment.DEVELOPMENT,
1122
f"""
1123
watchmedo shell-command1124
--patterns="*.py;*.sdoc;*.jinja;*.html;*.css;*.js"1125
--recursive1126
--ignore-pattern='output/;tests/integration'1127
--command='{strictdoc_command}'
1128
--drop1129
{paths_to_watch}
1130
""",
1131
)1132
1133
1134
@task1135
def run(context, command):
1136
run_invoke_with_tox(
1137
context,
1138
ToxEnvironment.DEVELOPMENT,
1139
f"""
1140
{command}
1141
""",
1142
)1143
1144
1145
@task1146
def nuitka(context):
1147
html_template_data_options = get_nuitka_html_template_data_options()
1148
html_static_data_options = get_nuitka_html_static_data_options()
1149
1150
run_invoke(
1151
context,
1152
f"""
1153
PYTHONPATH="{os.getcwd()}"
1154
python -m nuitka1155
--static-libpython=no1156
--standalone1157
--include-module=textx1158
--include-module=strictdoc.server.app1159
--include-module=docutils1160
--include-module=docutils.readers.standalone1161
--include-module=docutils.parsers.rst1162
{html_template_data_options}
1163
{html_static_data_options}
1164
--include-data-dir=strictdoc/backend/rst/templates=templates/rst1165
--include-package-data=docutils1166
strictdoc/cli/main.py1167
""",
1168
)1169
1170
1171
# https://github.com/jrfonseca/gprof2dot1172
# pip install gprof2dot1173
@task()
1174
def performance(context):
1175
command = """
1176
python -m cProfile -o output/profile.prof1177
-m strictdoc.cli.main export . --no-parallelization &&1178
gprof2dot -f pstats output/profile.prof | dot -Tpng -o output/output.png1179
"""1180
run_invoke(context, command)
1181
1182
1183
@task(performance)
1184
def performance_snakeviz(context):
1185
command = """
1186
snakeviz output/profile.prof1187
"""1188
run_invoke(context, command)
1189
1190
1191
@task(aliases=["bd"])
1192
def build_docker(
1193
context,
1194
image: str = "strictdoc:latest",
1195
no_cache: bool = False,
1196
source="pypi",
1197
):1198
no_cache_argument = "--no-cache" if no_cache else ""
1199
run_invoke(
1200
context,
1201
f"""
1202
docker build .1203
--build-arg STRICTDOC_SOURCE={source}
1204
-t {image}
1205
{no_cache_argument}
1206
""",
1207
)1208
1209
1210
@task(aliases=["rd"])
1211
def run_docker(
1212
context, image: str = "strictdoc:latest", command: Optional[str] = None
1213
):1214
command_argument = (
1215
f'/bin/bash -c "{command}"' if command is not None else ""
1216
)1217
1218
run_invoke(
1219
context,
1220
f"""
1221
docker run1222
--name strictdoc1223
--rm1224
-it1225
-e HOST_UID=$(id -u) -e HOST_GID=$(id -g)1226
-v "$(pwd):/data"1227
{image}
1228
{command_argument}
1229
""",
1230
pty=True,
1231
)1232
1233
1234
@task(aliases=["td"])
1235
def test_docker(context, image: str = "strictdoc:latest"):
1236
run_invoke(
1237
context,
1238
"""
1239
rm -rf output/ && mkdir -p output/ && chmod 777 output/1240
""",
1241
)1242
run_docker(
1243
context,
1244
image=image,
1245
command="strictdoc export --formats=html,html2pdf .",
1246
)1247
1248
def check_file_owner(filepath):
1249
import pwd # noqa: PLC0415
1250
1251
file_owner = pwd.getpwuid(os.stat(filepath).st_uid).pw_name
1252
current_user = os.environ.get("USER", "")
1253
return file_owner == current_user
1254
1255
assert check_file_owner(
1256
"output/html2pdf/pdf/docs/strictdoc_01_user_guide.pdf"1257
)1258
1259
1260
@task(aliases=["q"])
1261
def qualification(context):
1262
test_all(context, coverage=True, headless=True)
1263
coverage_combine(context)
1264
1265
1266
@task()
1267
def drawio(context):
1268
if sys.platform == "darwin":
1269
path_to_drawio = "/Applications/draw.io.app/Contents/MacOS/draw.io"
1270
elif sys.platform.startswith("linux"):
1271
path_to_drawio = "drawio"
1272
else:
1273
raise NotImplementedError(
1274
"drawio task is supported only on macOS and Linux."1275
)1276
1277
artifacts = [
1278
(1279
"developer/drawio/Architecture.drawio",
1280
"docs/_assets/StrictDoc_Workspace-Architecture.drawio.png",
1281
),1282
(1283
"developer/drawio/Backlog.drawio",
1284
"docs/_assets/StrictDoc_Workspace-Backlog.drawio.png",
1285
),1286
(1287
"developer/drawio/Roadmap.drawio",
1288
"docs/_assets/StrictDoc_Workspace-Roadmap.drawio.png",
1289
),1290
]1291
1292
for path_to_drawio_, path_to_png_ in artifacts:
1293
print(f"Copying: {path_to_drawio_} -> {path_to_png_}") # noqa: T201
1294
1295
# Basic safety for now to avoid writing wrong files.1296
assert os.path.isfile(path_to_drawio_), path_to_drawio_
1297
assert os.path.isfile(path_to_png_), path_to_png_
1298
1299
run_invoke(
1300
context,
1301
f"""
1302
{path_to_drawio}
1303
--export1304
--format png1305
-o {path_to_png_}
1306
--page-index 01307
{path_to_drawio_}
1308
""",
1309
pty=True,
1310
)