StrictDoc Documentation
tasks.py
Source file coverage
Path:
tasks.py
Lines:
1387
Non-empty lines:
1208
Non-empty lines covered with requirements:
1208 / 1208 (100.0%)
Functions:
48
Functions covered by requirements:
48 / 48 (100.0%)
1
import os
2
import re
3
import shutil
4
import sys
5
import tempfile
6
from enum import Enum
7
from pathlib import Path
8
from typing import Dict, Optional
9
 
10
import invoke
11
from invoke import task
12
 
13
from developer.git.commit_validator import (
14
    validate_commits_locally_or_ci,
15
)
16
from strictdoc.core.environment import (
17
    BINARY_HTML_STATIC_DIR,
18
    BINARY_HTML_TEMPLATES_DIR,
19
    HTML_STATIC_DIRS,
20
    HTML_TEMPLATE_DIRS,
21
)
22
from strictdoc.core.project_config import ProjectConfigDefault
23
from tools.server_check import free_strictdoc_port
24
 
25
# Specifying encoding because Windows crashes otherwise when running Invoke
26
# tasks below:
27
# UnicodeEncodeError: 'charmap' codec can't encode character '\ufffd'
28
# in position 16: character maps to <undefined>
29
# People say, it might also be possible to export PYTHONIOENCODING=utf8 but this
30
# seems to work.
31
# FIXME: If you are a Windows user and expert, please advise on how to do this
32
# properly.
33
sys.stdout = open(1, "w", encoding="utf-8", closefd=False, buffering=1)
34
 
35
STRICTDOC_TMP_DIR = os.path.join(tempfile.gettempdir(), "strictdoc_tmp_dir")
36
TEST_REPORTS_DIR = "build/test_reports"
37
 
38
# Redirect all __pycache__ output (from Invoke itself and every subprocess it
39
# spawns: pytest, mypy, plain python invocations, etc.) into a single folder,
40
# instead of littering __pycache__ directories across the source tree.
41
os.environ["PYTHONPYCACHEPREFIX"] = os.path.abspath("build/pycache")
42
 
43
 
44
def get_pyinstaller_html_template_data_options() -> str:
45
    return "\n".join(
46
        f'--add-data "{template_dir}{os.pathsep}{BINARY_HTML_TEMPLATES_DIR}"'
47
        for template_dir in HTML_TEMPLATE_DIRS
48
    )
49
 
50
 
51
def get_pyinstaller_html_static_data_options() -> str:
52
    return "\n".join(
53
        f'--add-data "{static_dir}{os.pathsep}{BINARY_HTML_STATIC_DIR}"'
54
        for static_dir in HTML_STATIC_DIRS
55
    )
56
 
57
 
58
def get_nuitka_html_template_data_options() -> str:
59
    return "\n".join(
60
        f'--include-data-dir="{template_dir}={BINARY_HTML_TEMPLATES_DIR}"'
61
        for template_dir in HTML_TEMPLATE_DIRS
62
    )
63
 
64
 
65
def get_nuitka_html_static_data_options() -> str:
66
    return "\n".join(
67
        f'--include-data-dir="{static_dir}={BINARY_HTML_STATIC_DIR}"'
68
        for static_dir in HTML_STATIC_DIRS
69
    )
70
 
71
 
72
# To prevent all tasks from building to the same virtual environment.
73
# All values correspond to the configuration in the tox.ini config file.
74
class ToxEnvironment(str, Enum):
75
    DEVELOPMENT = "development"
76
    CHECK = "check"
77
    DOCUMENTATION = "documentation"
78
    RELEASE = "release"
79
    RELEASE_LOCAL = "release-local"
80
    PYINSTALLER = "pyinstaller"
81
 
82
 
83
def run_invoke(
84
    context,
85
    cmd,
86
    environment: Optional[dict] = None,
87
    pty: bool = False,
88
    warn: bool = False,
89
) -> invoke.runners.Result:
90
    def one_line_command(string):
91
        return re.sub("\\s+", " ", string).strip()
92
 
93
    return context.run(
94
        one_line_command(cmd),
95
        env=environment,
96
        hide=False,
97
        warn=warn,
98
        pty=pty,
99
        echo=True,
100
    )
101
 
102
 
103
def run_invoke_with_tox(
104
    context,
105
    environment_type: ToxEnvironment,
106
    command: str,
107
    environment: Optional[Dict] = None,
108
    pty: bool = False,
109
) -> invoke.runners.Result:
110
    assert isinstance(environment_type, ToxEnvironment)
111
    assert isinstance(command, str)
112
 
113
    tox_py_version = f"py{sys.version_info.major}{sys.version_info.minor}"
114
 
115
    return run_invoke(
116
        context,
117
        f"""
118
            tox
119
                -e {tox_py_version}-{environment_type.value} --
120
                {command}
121
        """,
122
        environment=environment,
123
        pty=pty,
124
    )
125
 
126
 
127
@task(default=True)
128
def list_tasks(context):
129
    clean_command = """
130
        invoke --list
131
    """
132
    run_invoke(context, clean_command)
133
 
134
 
135
@task
136
def clean(context):
137
    # https://unix.stackexchange.com/a/689930/77389
138
    clean_command = r"""
139
        rm -rf output/ docs/sphinx/build/ &&
140
        find tests/ -type d \( -name output -o -name Output \) -exec rm -rf {} +
141
    """
142
    run_invoke(context, clean_command)
143
 
144
 
145
@task(aliases=["s"])
146
def server(context, input_path=".", config=None, port=None):
147
    assert os.path.isdir(input_path), input_path
148
    if config is not None:
149
        assert os.path.isfile(config), config
150
    config_argument = f"--config {config}" if config is not None else ""
151
    port_argument = f"--port {port}" if port is not None else ""
152
 
153
    try:
154
        should_continue = free_strictdoc_port(
155
            port
156
            if port is not None
157
            else ProjectConfigDefault.DEFAULT_SERVER_PORT
158
        )
159
    except OSError as error:
160
        print(error, file=sys.stderr)  # noqa: T201
161
        return
162
 
163
    if not should_continue:
164
        return
165
 
166
    run_invoke_with_tox(
167
        context,
168
        ToxEnvironment.DEVELOPMENT,
169
        f"""
170
            python -m strictdoc.cli.main
171
                --debug
172
                --development
173
                server {input_path} {config_argument}
174
                    --host 127.0.0.1
175
                    {port_argument}
176
                    --reload
177
                    --watch
178
        """,
179
    )
180
 
181
 
182
@task(aliases=["scs"])
183
def screencast_server(context, focus=None, edit=False):
184
    """
185
    Manual dev server for tests/screencast scenarios: starts StrictDoc on a
186
    scenario's project (the shared demo fixture by default, or another
187
    scenario's project via --focus), for inspecting it in the browser.
188
 
189
    By default this serves a disposable copy, rebuilt fresh every time, so
190
    nothing done through the UI persists. Pass --edit to serve the real,
191
    persistent files instead (the shared fixture itself, or a generated
192
    project reused across restarts) when intentionally editing them.
193
    """
194
 
195
    focus_argument = f"--focus {focus}" if focus is not None else ""
196
    edit_argument = "--edit" if edit else ""
197
 
198
    run_invoke_with_tox(
199
        context,
200
        ToxEnvironment.CHECK,
201
        f"python tests/screencast/run_server.py {focus_argument} {edit_argument}",
202
    )
203
 
204
 
205
@task(aliases=["d"])
206
def docs(context):
207
    run_invoke_with_tox(
208
        context,
209
        ToxEnvironment.DOCUMENTATION,
210
        """
211
            python3 -m strictdoc.cli.main
212
                export .
213
                    --formats=html
214
                    --output-dir output/strictdoc_website
215
                    --project-title "StrictDoc"
216
        """,
217
    )
218
 
219
    run_invoke_with_tox(
220
        context,
221
        ToxEnvironment.DOCUMENTATION,
222
        """
223
            python3 -m strictdoc.cli.main
224
                export ./
225
                    --formats=rst
226
                    --output-dir output/sphinx
227
                    --project-title "StrictDoc"
228
        """,
229
    )
230
 
231
    run_invoke_with_tox(
232
        context,
233
        ToxEnvironment.DOCUMENTATION,
234
        """
235
            cp -r output/sphinx/rst/docs/* docs/sphinx/source/ &&
236
            mkdir -p docs/sphinx/source/_assets/ &&
237
            cp -v docs/_assets/* docs/sphinx/source/_assets/
238
        """,
239
    )
240
 
241
    run_invoke_with_tox(
242
        context,
243
        ToxEnvironment.DOCUMENTATION,
244
        """
245
            make --directory docs/sphinx html latexpdf SPHINXOPTS="-W --keep-going"
246
        """,
247
    )
248
 
249
    run_invoke(
250
        context,
251
        (
252
            """
253
                open docs/sphinx/build/latex/strictdoc.pdf
254
            """
255
        ),
256
    )
257
 
258
 
259
@task(clean, aliases=["tus"])
260
def test_unit_server(context, focus=None):
261
    focus_argument = f"-k {focus}" if focus is not None else ""
262
 
263
    Path(TEST_REPORTS_DIR).mkdir(parents=True, exist_ok=True)
264
 
265
    cwd = os.getcwd()
266
 
267
    path_to_coverage_file = f"{cwd}/build/coverage/unit_server/.coverage"
268
 
269
    run_invoke_with_tox(
270
        context,
271
        ToxEnvironment.CHECK,
272
        f"""
273
            coverage run
274
            --rcfile=.coveragerc.unit_server
275
            --data-file={path_to_coverage_file}
276
            -m pytest
277
            tests/unit_server/
278
                {focus_argument}
279
                --junit-xml={TEST_REPORTS_DIR}/tests_unit_server.pytest.junit.xml
280
                -o junit_suite_name="StrictDoc Web Server Unit Tests"
281
                -o cache_dir=build/pytest_cache/unit_server
282
        """,
283
    )
284
 
285
 
286
@task(test_unit_server, aliases=["tusc"])
287
def test_unit_server_report(context):
288
    cwd = os.getcwd()
289
 
290
    path_to_coverage_file = f"{cwd}/build/coverage/unit_server/.coverage"
291
 
292
    run_invoke_with_tox(
293
        context,
294
        ToxEnvironment.CHECK,
295
        f"""
296
            coverage html
297
                --rcfile=.coveragerc.unit_server
298
                --data-file={path_to_coverage_file}
299
        """,
300
    )
301
 
302
 
303
@task(aliases=["te"])
304
def test_end2end(
305
    context,
306
    *,
307
    focus=None,
308
    exit_first=False,
309
    parallelize=False,
310
    long_timeouts=False,
311
    headless=False,
312
    headed=False,
313
    shard=None,
314
    test_path=None,
315
    coverage: bool = False,
316
):
317
    """
318
    @relation(SDOC-SRS-46, scope=function)
319
    """
320
 
321
    environment = {}
322
 
323
    coverage_command_or_none = ""
324
    coverage_argument_or_none = ""
325
 
326
    if coverage:
327
        cwd = os.getcwd()
328
        coverage_file_dir = f"{cwd}/build/coverage/end2end/"
329
        coverage_file_dir2 = f"{cwd}/build/coverage/end2end_strictdoc/"
330
        coverage_file = os.path.join(coverage_file_dir, ".coverage")
331
        coverage_rc = os.path.join(cwd, ".coveragerc.end2end")
332
        shutil.rmtree(coverage_file_dir, ignore_errors=True)
333
        shutil.rmtree(coverage_file_dir2, ignore_errors=True)
334
        coverage_command_or_none = f"""
335
            coverage run
336
                --rcfile={coverage_rc}
337
                --data-file={coverage_file}
338
                -m
339
        """
340
        coverage_argument_or_none = "--strictdoc-coverage"
341
 
342
    long_timeouts_argument = (
343
        "--strictdoc-long-timeouts" if long_timeouts else ""
344
    )
345
 
346
    parallelize_argument = ""
347
    if parallelize:
348
        print(  # noqa: T201
349
            "warning: "
350
            "Running parallelized end-2-end tests is supported "
351
            "but is not stable."
352
        )
353
        parallelize_argument = "--numprocesses=2 --strictdoc-parallelize"
354
 
355
    assert shard is None or re.match(r"[1-9][0-9]*/[1-9][0-9]*", shard), (
356
        f"--shard argument has an incorrect format: {shard}."
357
    )
358
    shard_argument = f"--strictdoc-shard={shard}" if shard else ""
359
 
360
    focus_argument = f"-k {focus}" if focus is not None else ""
361
    exit_first_argument = "--exitfirst" if exit_first else ""
362
    headless_argument = "--headless2" if headless and not headed else "--gui"
363
    test_command = f"""
364
            {coverage_command_or_none}
365
            pytest
366
            --failed-first
367
            --capture=no
368
            --reuse-session
369
            {parallelize_argument}
370
            {shard_argument}
371
            {coverage_argument_or_none}
372
            {focus_argument}
373
            {exit_first_argument}
374
            {long_timeouts_argument}
375
            {headless_argument}
376
            --junit-xml={TEST_REPORTS_DIR}/tests_end2end.pytest.junit.xml
377
            -o junit_suite_name="StrictDoc End-to-End Tests"
378
            -o cache_dir=build/pytest_cache/end2end
379
            tests/end2end
380
    """
381
    if test_path:
382
        test_command = test_command.rstrip() + f"/{test_path}"
383
 
384
    Path(TEST_REPORTS_DIR).mkdir(parents=True, exist_ok=True)
385
 
386
    # On Windows, GitHub Actions fails with:
387
    # response = {'status': 500, 'value':
388
    # '{"value":{"error":"unknown error",
389
    # "message":"unknown error: cannot find Chrome binary",  # noqa: ERA001
390
    # This very likely has to do with PATH isolation that Tox does.
391
    # FIXME: If you are a Windows expert, please fix this to run on Tox.
392
    if os.name == "nt":
393
        run_invoke(context, test_command)
394
        return
395
 
396
    run_invoke_with_tox(
397
        context,
398
        ToxEnvironment.CHECK,
399
        test_command,
400
        environment=environment,
401
    )
402
 
403
 
404
@task(aliases=["tsc"])
405
def test_screencast(context, *, focus=None, record_video=False):
406
    """
407
    Runs the tests/screencast scenarios: fast pass/fail checks by default,
408
    or (re)recording of the corresponding .webm videos with --record-video.
409
    """
410
 
411
    focus_argument = f"-k {focus}" if focus is not None else ""
412
    record_video_argument = "--strictdoc-record-video" if record_video else ""
413
 
414
    Path(TEST_REPORTS_DIR).mkdir(parents=True, exist_ok=True)
415
 
416
    test_command = f"""
417
        pytest
418
        --capture=no
419
        {focus_argument}
420
        {record_video_argument}
421
        --junit-xml={TEST_REPORTS_DIR}/tests_screencast.pytest.junit.xml
422
        -o junit_suite_name="StrictDoc Screencast Tests"
423
        -o cache_dir=build/pytest_screencast
424
        tests/screencast/scenarios
425
    """
426
 
427
    run_invoke_with_tox(context, ToxEnvironment.CHECK, test_command)
428
 
429
 
430
@task(aliases=["scov"])
431
def screencast_optimize_video(context, focus=None):
432
    """
433
    Converts recorded tests/screencast/output/*.webm into muted,
434
    web-optimized .webm (VP9) and .mp4 (H.264) pairs under
435
    tests/screencast/output/web/, ready to hand off to the product website.
436
 
437
    Run `invoke test-screencast --record-video` first to produce the raw
438
    recordings this reads from.
439
    """
440
 
441
    focus_argument = f"--focus {focus}" if focus is not None else ""
442
 
443
    run_invoke(
444
        context, f"python tests/screencast/optimize_video.py {focus_argument}"
445
    )
446
 
447
 
448
@task(aliases=["tu"])
449
def test_unit(context, coverage=False, focus=None, path=None, output=False):
450
    """
451
    @relation(SDOC-SRS-44, scope=function)
452
    """
453
 
454
    Path(TEST_REPORTS_DIR).mkdir(parents=True, exist_ok=True)
455
 
456
    focus_argument = f"-k {focus}" if focus is not None else ""
457
    output_argument = "--capture=no" if output else ""
458
 
459
    cwd = os.getcwd()
460
 
461
    if path is None:
462
        path = "tests/unit"
463
    else:
464
        assert "tests/unit" in path, path
465
 
466
    path_to_coverage_file = f"{cwd}/build/coverage/unit/.coverage"
467
 
468
    pytest_command = (
469
        """
470
        coverage run
471
            --rcfile=.coveragerc.unit
472
            --data-file={path_to_coverage_file}
473
            -m pytest
474
    """
475
        if coverage
476
        else "pytest"
477
    )
478
 
479
    run_invoke_with_tox(
480
        context,
481
        ToxEnvironment.CHECK,
482
        f"""
483
            {pytest_command}
484
            {focus_argument}
485
            {output_argument}
486
            --junit-xml={TEST_REPORTS_DIR}/tests_unit.pytest.junit.xml
487
            -o cache_dir=build/pytest_cache/unit
488
            -o junit_suite_name="StrictDoc Unit Tests"
489
            -p no:seleniumbase
490
            {path}
491
        """,
492
    )
493
    if coverage and not focus and path == "tests/unit":
494
        run_invoke_with_tox(
495
            context,
496
            ToxEnvironment.CHECK,
497
            f"""
498
                coverage report
499
                    --sort=cover
500
                    --rcfile=.coveragerc.unit
501
                    --data-file={path_to_coverage_file}
502
            """,
503
        )
504
 
505
 
506
@task(test_unit, aliases=["tuc"])
507
def test_unit_report(context):
508
    cwd = os.getcwd()
509
 
510
    path_to_coverage_file = f"{cwd}/build/coverage/unit/.coverage"
511
 
512
    run_invoke_with_tox(
513
        context,
514
        ToxEnvironment.CHECK,
515
        f"""
516
            coverage html
517
                --rcfile=.coveragerc.unit
518
                --data-file={path_to_coverage_file}
519
        """,
520
    )
521
 
522
 
523
@task(clean, aliases=["ti"])
524
def test_integration(
525
    context,
526
    *,
527
    focus=None,
528
    debug=False,
529
    no_parallelization=False,
530
    fail_first=False,
531
    coverage=False,
532
    strictdoc=None,
533
    html2pdf=False,
534
    shard=None,
535
    environment=ToxEnvironment.CHECK,
536
):
537
    """
538
    @relation(SDOC-SRS-45, scope=function)
539
    """
540
 
541
    cwd = os.getcwd()
542
 
543
    if strictdoc is None:
544
        strictdoc_exec = "python3 -m strictdoc.cli.main"
545
    else:
546
        strictdoc_exec = strictdoc
547
 
548
    coverage_path_argument = ""
549
    if coverage:
550
        path_to_coverage_rc = f"{cwd}/.coveragerc.integration"
551
        strictdoc_exec = (
552
            f"coverage run --rcfile={path_to_coverage_rc} -m strictdoc.cli.main"
553
        )
554
        if html2pdf:
555
            path_to_coverage_dir = f"{cwd}/build/coverage/integration_html2pdf/"
556
        else:
557
            path_to_coverage_dir = f"{cwd}/build/coverage/integration/"
558
        path_to_coverage = os.path.join(path_to_coverage_dir, ".coverage")
559
        shutil.rmtree(path_to_coverage_dir, ignore_errors=True)
560
        coverage_path_argument = (
561
            f'--param COVERAGE_FILE="{path_to_coverage}" '
562
            f'--param COVERAGE_PROCESS_START="{path_to_coverage_rc}"'
563
        )
564
 
565
    debug_opts = "-vv --show-all" if debug else ""
566
    focus_or_none = f"--filter {focus}" if focus else ""
567
    fail_first_argument = "--max-failures 1" if fail_first else ""
568
    junit_xml_report_argument = (
569
        "--xunit-xml-output build/test_reports/tests_integration_html2pdf.lit.junit.xml"
570
        if html2pdf
571
        else "--xunit-xml-output build/test_reports/tests_integration.lit.junit.xml"
572
    )
573
 
574
    # Allow partitioning of integration and html2pdf tests
575
    partition_opts = ""
576
    if shard is not None:
577
        match = re.match(r"([1-9][0-9]*)/([1-9][0-9]*)", shard)
578
        assert match, f"--shard argument has an incorrect format: {shard}."
579
        run_shard = int(match.group(1))
580
        num_shards = int(match.group(2))
581
        partition_opts = f"--num-shards={num_shards} --run-shard={run_shard}"
582
 
583
    # HTML2PDF tests are running Chrome Driver which does not seem to be
584
    # parallelizable, or at least not in the way StrictDoc uses it.
585
    # If HTML2PDF option is provided, do not parallelize and only run the
586
    # HTML2PDF-specific tests.
587
    # HTML2PDF tests can be safely partitioned.
588
    chromedriver_param = ""
589
    if not html2pdf:
590
        parallelize_opts = "" if not no_parallelization else "--threads 1"
591
        html2pdf_param = ""
592
        test_folder = f"{cwd}/tests/integration"
593
        test_output_dir = "build/tests_integration"
594
    else:
595
        parallelize_opts = "--threads 1"
596
        html2pdf_param = "--param TEST_HTML2PDF=1"
597
        chromedriver_path = os.environ.get("CHROMEWEBDRIVER")
598
        if chromedriver_path is not None:
599
            # NOTE: isfile() check does not work on GitHub Actions / Linux,
600
            #       the exists() check works.
601
            assert os.path.exists(chromedriver_path), chromedriver_path
602
            chromedriver_param = f"--param CHROMEDRIVER={os.path.join(chromedriver_path, 'chromedriver')}"
603
            if os.name == "nt":
604
                # On Windows, its chromdriver.exe
605
                chromedriver_param = chromedriver_param + ".exe"
606
        test_folder = f"{cwd}/tests/integration/features/html2pdf"
607
        test_output_dir = "build/tests_integration_html2pdf"
608
 
609
    # The command sometimes exits with 1 even if the files are deleted.
610
    # warn=True ensures that the execution continues.
611
    run_invoke(
612
        context,
613
        f"""
614
        rm -rf {test_output_dir}
615
        """,
616
        warn=True,
617
    )
618
 
619
    run_invoke(
620
        context,
621
        f"""
622
        rm -rf {STRICTDOC_TMP_DIR}
623
        """,
624
    )
625
 
626
    Path(STRICTDOC_TMP_DIR).mkdir(exist_ok=True)
627
    Path(TEST_REPORTS_DIR).mkdir(parents=True, exist_ok=True)
628
 
629
    itest_command = f"""
630
        lit
631
        --param STRICTDOC_EXEC="{strictdoc_exec}"
632
        --param STRICTDOC_TMP_DIR="{STRICTDOC_TMP_DIR}"
633
        --param TEST_OUTPUT_DIR="{test_output_dir}"
634
        --timeout 180
635
        --order smart
636
        {junit_xml_report_argument}
637
        {coverage_path_argument}
638
        {html2pdf_param}
639
        {chromedriver_param}
640
        -v
641
        {debug_opts}
642
        {focus_or_none}
643
        {fail_first_argument}
644
        {parallelize_opts}
645
        {partition_opts}
646
        {test_folder}
647
    """
648
 
649
    # It looks like LIT does not open the RUN: subprocesses in the same
650
    # environment from which it itself is run from. This issue has been known by
651
    # us for a couple of years by now. Not using Tox on Windows for the time
652
    # being.
653
    if os.name == "nt":
654
        run_invoke(context, itest_command)
655
        return
656
 
657
    run_invoke_with_tox(
658
        context,
659
        environment,
660
        itest_command,
661
        environment={"STRICTDOC_CACHE_DIR": "Output/_cache"},
662
    )
663
 
664
 
665
@task
666
def coverage_combine(context):
667
    run_invoke_with_tox(
668
        context,
669
        ToxEnvironment.CHECK,
670
        """
671
            coverage combine
672
                --data-file build/coverage/.coverage.combined
673
                --keep
674
                build/coverage/end2end_strictdoc/.coverage.*
675
                build/coverage/integration/.coverage.*
676
                build/coverage/integration_html2pdf/.coverage.*
677
                build/coverage/unit/.coverage
678
                build/coverage/unit_server/.coverage
679
        """,
680
    )
681
    run_invoke_with_tox(
682
        context,
683
        ToxEnvironment.CHECK,
684
        """
685
            coverage html
686
                --rcfile .coveragerc.combined
687
                --data-file build/coverage/.coverage.combined
688
        """,
689
    )
690
    run_invoke_with_tox(
691
        context,
692
        ToxEnvironment.CHECK,
693
        """
694
            coverage json
695
                --rcfile .coveragerc.combined
696
                --data-file build/coverage/.coverage.combined
697
                --pretty-print
698
                -o build/coverage/coverage.combined.json
699
        """,
700
    )
701
 
702
 
703
@task
704
def lint_ruff_format(context):
705
    """
706
    @relation(SDOC-SRS-42, scope=function)
707
    """
708
 
709
    result: invoke.runners.Result = run_invoke_with_tox(
710
        context,
711
        ToxEnvironment.CHECK,
712
        """
713
            ruff
714
                format
715
                --cache-dir build/ruff_cache
716
                *.py
717
                developer/
718
                docs/
719
                strictdoc/
720
                tools/ecss
721
                tests/unit/
722
                tests/unit_server/
723
                tests/integration/*.py
724
                tests/end2end/
725
        """,
726
    )
727
    # Ruff always exits with 0, so we handle the output.
728
    if "reformatted" in result.stdout:
729
        print("invoke: ruff format found issues")  # noqa: T201
730
        result.exited = 1
731
        raise invoke.exceptions.UnexpectedExit(result)
732
 
733
 
734
@task(aliases=["lr"])
735
def lint_ruff(context):
736
    """
737
    @relation(SDOC-SRS-42, scope=function)
738
    """
739
 
740
    run_invoke_with_tox(
741
        context,
742
        ToxEnvironment.CHECK,
743
        """
744
            ruff check . --fix --exit-non-zero-on-fix --cache-dir build/ruff_cache
745
        """,
746
    )
747
 
748
 
749
@task(aliases=["lm"])
750
def lint_mypy(context):
751
    """
752
    @relation(SDOC-SRS-41, SDOC-SRS-43, scope=function)
753
    """
754
 
755
    # These checks do not seem to be useful:
756
    # - import
757
    # --disallow-any-expr
758
    # --disallow-any-explicit
759
    # --disallow-any-unimported  # noqa: ERA001
760
    # --disallow-any-decorated
761
    # - type-abstract. It is ignored on purpose because of assert_cast()
762
    #   implementation. See https://stackoverflow.com/a/74073453/598057.
763
    run_invoke_with_tox(
764
        context,
765
        ToxEnvironment.CHECK,
766
        """
767
            mypy docs/
768
                 strictdoc/
769
                 tests/unit/strictdoc/backend/sdoc_source_code/test_marker_lexer.py
770
 
771
                --show-error-codes
772
                --disable-error-code=import
773
                --disable-error-code=type-abstract
774
                --cache-dir=build/mypy_cache
775
                --extra-checks
776
 
777
                --strict
778
                --strict-optional
779
                --strict-equality
780
 
781
                --check-untyped-defs
782
                --disallow-any-generics
783
                --disallow-incomplete-defs
784
                --disallow-subclassing-any
785
                --disallow-untyped-calls
786
                --disallow-untyped-decorators
787
                --disallow-untyped-defs
788
                --no-implicit-optional
789
                --warn-no-return
790
                --warn-redundant-casts
791
                --warn-return-any
792
                --warn-unreachable
793
                --warn-unused-ignores
794
 
795
                --python-version=3.10
796
        """,
797
    )
798
 
799
 
800
@task
801
def lint_format_js(context):
802
    # NOTE: Could not find the '--' equivalent for -w80.
803
    result: invoke.runners.Result = run_invoke_with_tox(
804
        context,
805
        ToxEnvironment.CHECK,
806
        """
807
            js-beautify
808
                --indent-size=2
809
                --end-with-newline
810
                --replace
811
                -w100
812
                strictdoc/export/html/_static/autocompletable_field.js
813
                strictdoc/export/html/_static/copy_to_clipboard.js
814
                strictdoc/export/html/_static/deletable_field.js
815
                strictdoc/export/html/_static/draggable_list.js
816
                strictdoc/export/html/_static/editable_field.js
817
                strictdoc/export/html/_static/modal.js
818
                strictdoc/export/html/_static/movable_field.js
819
                strictdoc/export/html/_static/scroll_into_view.js
820
                strictdoc/export/html/_static/static_html_search.js
821
                strictdoc/export/html/_static/tabs.js
822
                strictdoc/features/project_index/assets/stable_uri_forwarder.js
823
        """,
824
    )
825
    # Ruff always exits with 0, so we handle the output.
826
    if "reformatted" in result.stdout:
827
        print("invoke: ruff format found issues")  # noqa: T201
828
        result.exited = 1
829
        raise invoke.exceptions.UnexpectedExit(result)
830
 
831
 
832
@task(aliases=["lc"])
833
def lint_commit(context):  # noqa: ARG001
834
    try:
835
        validate_commits_locally_or_ci()
836
    except ValueError as e:
837
        raise invoke.exceptions.Exit(message=str(e), code=1) from None
838
 
839
 
840
@task(aliases=["lf"])
841
def lint_fixit(context, fix=False, auto=False, path="strictdoc/"):
842
    if fix:
843
        auto_argument = "--automatic" if auto else ""
844
        run_invoke_with_tox(
845
            context,
846
            ToxEnvironment.CHECK,
847
            f"""
848
                fixit fix {path} {auto_argument}
849
            """,
850
            pty=True,
851
        )
852
    else:
853
        run_invoke_with_tox(
854
            context,
855
            ToxEnvironment.CHECK,
856
            f"""
857
                fixit lint --diff {path}
858
            """,
859
        )
860
 
861
 
862
@task(aliases=["l"])
863
def lint(context):
864
    lint_commit(context)
865
    lint_ruff_format(context)
866
    lint_ruff(context)
867
    lint_mypy(context)
868
 
869
 
870
@task(aliases=["t"])
871
def test(context, shard=None):
872
    test_unit(context)
873
    test_unit_server(context)
874
    test_integration(context, shard=shard)
875
 
876
 
877
@task(aliases=["ta"])
878
def test_all(context, coverage=False, headless=False):
879
    test_unit(context, coverage=coverage)
880
    test_unit_server(context)
881
    test_integration(context, coverage=coverage)
882
    test_integration(context, coverage=coverage, html2pdf=True)
883
    test_end2end(context, coverage=coverage, headless=headless)
884
 
885
 
886
@task(aliases=["c"])
887
def check(context):
888
    lint(context)
889
    test(context)
890
 
891
 
892
# https://github.com/github-changelog-generator/github-changelog-generator
893
# gem install github_changelog_generator
894
@task
895
def changelog(context, github_token):
896
    # The alpha release tags are excluded from the changelog.
897
    command = f"""
898
        github_changelog_generator
899
        --token {github_token}
900
        --user strictdoc-project
901
        --exclude-tags-regex ".*a\\d+"
902
        --project strictdoc
903
        """
904
    run_invoke(context, command)
905
 
906
 
907
@task
908
def check_dead_links(context):
909
    run_invoke_with_tox(
910
        context,
911
        ToxEnvironment.CHECK,
912
        """
913
            python3 tools/link_health.py docs/strictdoc_01_user_guide.sdoc
914
        """,
915
    )
916
    run_invoke_with_tox(
917
        context,
918
        ToxEnvironment.CHECK,
919
        """
920
            python3 tools/link_health.py docs/strictdoc_02_feature_map.sdoc
921
        """,
922
    )
923
    run_invoke_with_tox(
924
        context,
925
        ToxEnvironment.CHECK,
926
        """
927
            python3 tools/link_health.py docs/strictdoc_03_faq.sdoc
928
        """,
929
    )
930
    run_invoke_with_tox(
931
        context,
932
        ToxEnvironment.CHECK,
933
        """
934
            python3 tools/link_health.py docs/strictdoc_04_release_notes.sdoc
935
        """,
936
    )
937
    run_invoke_with_tox(
938
        context,
939
        ToxEnvironment.CHECK,
940
        """
941
            python3 tools/link_health.py docs/strictdoc_05_troubleshooting.sdoc
942
        """,
943
    )
944
    run_invoke_with_tox(
945
        context,
946
        ToxEnvironment.CHECK,
947
        """
948
            python3 tools/link_health.py docs/strictdoc_10_contributing.sdoc
949
        """,
950
    )
951
    run_invoke_with_tox(
952
        context,
953
        ToxEnvironment.CHECK,
954
        """
955
            python3 tools/link_health.py docs/strictdoc_11_developer_guide.sdoc
956
        """,
957
    )
958
    run_invoke_with_tox(
959
        context,
960
        ToxEnvironment.CHECK,
961
        """
962
            python3 tools/link_health.py docs/strictdoc_24_development_plan.sdoc
963
        """,
964
    )
965
    run_invoke_with_tox(
966
        context,
967
        ToxEnvironment.CHECK,
968
        """
969
            python3 tools/link_health.py docs/strictdoc_20_l1_system_requirements.sdoc
970
        """,
971
    )
972
    run_invoke_with_tox(
973
        context,
974
        ToxEnvironment.CHECK,
975
        """
976
            python3 tools/link_health.py docs/strictdoc_21_l2_high_level_requirements.sdoc
977
        """,
978
    )
979
    run_invoke_with_tox(
980
        context,
981
        ToxEnvironment.CHECK,
982
        """
983
            python3 tools/link_health.py docs/strictdoc_25_design.sdoc
984
        """,
985
    )
986
    run_invoke_with_tox(
987
        context,
988
        ToxEnvironment.CHECK,
989
        """
990
            python3 tools/link_health.py CONTRIBUTING.md
991
        """,
992
    )
993
    run_invoke_with_tox(
994
        context,
995
        ToxEnvironment.CHECK,
996
        """
997
            python3 tools/link_health.py NOTICE
998
        """,
999
    )
1000
    run_invoke_with_tox(
1001
        context,
1002
        ToxEnvironment.CHECK,
1003
        """
1004
            python3 tools/link_health.py README.md
1005
        """,
1006
    )
1007
 
1008
 
1009
@task
1010
def release_local(context):
1011
    run_invoke(
1012
        context,
1013
        """
1014
            rm -rfv build/
1015
        """,
1016
    )
1017
    run_invoke(
1018
        context,
1019
        """
1020
            pip uninstall strictdoc -y
1021
        """,
1022
    )
1023
    run_invoke_with_tox(
1024
        context,
1025
        ToxEnvironment.RELEASE_LOCAL,
1026
        """
1027
            python -m build --outdir build/dist
1028
        """,
1029
    )
1030
    run_invoke_with_tox(
1031
        context,
1032
        ToxEnvironment.RELEASE_LOCAL,
1033
        """
1034
            twine check build/dist/*
1035
        """,
1036
    )
1037
    run_invoke_with_tox(
1038
        context,
1039
        ToxEnvironment.RELEASE_LOCAL,
1040
        """
1041
            pip install build/dist/*.tar.gz
1042
        """,
1043
    )
1044
    test_integration(
1045
        context, strictdoc="strictdoc", environment=ToxEnvironment.RELEASE_LOCAL
1046
    )
1047
 
1048
 
1049
@task
1050
def release(context, test_pypi=False, username=None, password=None):
1051
    """
1052
    A release can be made to PyPI or test package index (TestPyPI):
1053
    https://pypi.org/project/strictdoc/
1054
    https://test.pypi.org/project/strictdoc/
1055
    """
1056
 
1057
    env_user = os.environ.get("TWINE_USERNAME")
1058
    env_pass = os.environ.get("TWINE_PASSWORD")
1059
 
1060
    assert not ((username or password) and (env_user or env_pass)), (
1061
        username,
1062
        password,
1063
        env_user,
1064
        env_pass,
1065
    )
1066
    assert (username and password) or (env_user and env_pass), (
1067
        username,
1068
        password,
1069
        env_user,
1070
        env_pass,
1071
    )
1072
    if env_user:
1073
        assert env_user == "__token__"
1074
 
1075
    repository_argument_or_none = ""
1076
    if username is not None and password is not None:
1077
        repository_argument_or_none = (
1078
            ""
1079
            if username
1080
            else (
1081
                "--repository strictdoc_test"
1082
                if test_pypi
1083
                else "--repository strictdoc_release"
1084
            )
1085
        )
1086
    user_password = f"-u{username} -p{password}" if username is not None else ""
1087
 
1088
    run_invoke(
1089
        context,
1090
        """
1091
            rm -rfv build/dist/
1092
        """,
1093
    )
1094
    run_invoke_with_tox(
1095
        context,
1096
        ToxEnvironment.RELEASE,
1097
        """
1098
            python3 -m build --outdir build/dist
1099
        """,
1100
    )
1101
    run_invoke_with_tox(
1102
        context,
1103
        ToxEnvironment.RELEASE,
1104
        """
1105
            twine check build/dist/*
1106
        """,
1107
    )
1108
    # The token is in a core developer's .pypirc file.
1109
    # https://test.pypi.org/manage/account/token/
1110
    # https://packaging.python.org/en/latest/specifications/pypirc/#pypirc
1111
    run_invoke_with_tox(
1112
        context,
1113
        ToxEnvironment.RELEASE,
1114
        f"""
1115
            twine upload build/dist/strictdoc-*.tar.gz build/dist/strictdoc-*.whl
1116
                {repository_argument_or_none}
1117
                {user_password}
1118
        """,
1119
    )
1120
 
1121
 
1122
@task
1123
def release_pyinstaller(context):
1124
    path_to_pyi_dist = "/tmp/strictdoc"
1125
    html_template_data_options = get_pyinstaller_html_template_data_options()
1126
    html_static_data_options = get_pyinstaller_html_static_data_options()
1127
 
1128
    # The --hidden-import strictdoc.server.app flag is needed because without
1129
    # it, the following is produced:
1130
    # ERROR: Error loading ASGI app. Could not import
1131
    # module "strictdoc.server.app".
1132
    # Solution found here: https://stackoverflow.com/a/71340437/598057
1133
    # This behavior is not surprising because that's how the uvicorn loads the
1134
    # application separately from the parent process.
1135
    #
1136
    # Compatibility modules can be imported by user-provided statistics
1137
    # generators at runtime. PyInstaller cannot discover these imports
1138
    # statically because the generators live outside of StrictDoc's package.
1139
    #
1140
    # --hidden-import strictdoc.api is needed for the same reason:
1141
    # strictdoc.api is never imported anywhere inside StrictDoc's own
1142
    # package (only by dynamically-loaded, external files such as a
1143
    # project's own strictdoc_config.py, custom statistics generators, or
1144
    # custom plugins), so PyInstaller's static analysis -- which walks
1145
    # imports starting from strictdoc/cli/main.py -- never discovers it on
1146
    # its own.
1147
    command = f"""
1148
        pyinstaller
1149
            --clean
1150
            --name strictdoc
1151
            --noconfirm
1152
            --additional-hooks-dir developer/pyinstaller_hooks
1153
            --distpath {path_to_pyi_dist}
1154
            --hidden-import strictdoc.api
1155
            --hidden-import strictdoc.backend.rst.strictdoc_lexer
1156
            --hidden-import strictdoc.core.statistics.metric
1157
            --hidden-import strictdoc.export.html.generators.project_statistics
1158
            --hidden-import strictdoc.export.html.generators.view_objects.project_statistics_view_object
1159
            --hidden-import strictdoc.export.html.generators.view_objects.project_tree_stats
1160
            --hidden-import strictdoc.server.app
1161
            {html_template_data_options}
1162
            {html_static_data_options}
1163
            --add-data strictdoc/backend/rst/templates:templates/rst
1164
            strictdoc/cli/main.py
1165
    """
1166
 
1167
    run_invoke_with_tox(
1168
        context,
1169
        ToxEnvironment.PYINSTALLER,
1170
        """
1171
    pyinstaller --version
1172
    """,
1173
    )
1174
 
1175
    run_invoke_with_tox(context, ToxEnvironment.PYINSTALLER, command)
1176
 
1177
 
1178
@task
1179
def watch(context, sdocs_path="."):
1180
    strictdoc_command = f"""
1181
        python -m strictdoc.cli.main
1182
            export
1183
            {sdocs_path}
1184
            --output-dir output/
1185
    """
1186
 
1187
    run_invoke_with_tox(
1188
        context,
1189
        ToxEnvironment.DEVELOPMENT,
1190
        f"""
1191
            {strictdoc_command}
1192
        """,
1193
    )
1194
 
1195
    paths_to_watch = "."
1196
    run_invoke_with_tox(
1197
        context,
1198
        ToxEnvironment.DEVELOPMENT,
1199
        f"""
1200
            watchmedo shell-command
1201
            --patterns="*.py;*.sdoc;*.jinja;*.html;*.css;*.js"
1202
            --recursive
1203
            --ignore-pattern='output/;tests/integration'
1204
            --command='{strictdoc_command}'
1205
            --drop
1206
            {paths_to_watch}
1207
        """,
1208
    )
1209
 
1210
 
1211
@task
1212
def run(context, command):
1213
    run_invoke_with_tox(
1214
        context,
1215
        ToxEnvironment.DEVELOPMENT,
1216
        f"""
1217
            {command}
1218
        """,
1219
    )
1220
 
1221
 
1222
@task
1223
def nuitka(context):
1224
    html_template_data_options = get_nuitka_html_template_data_options()
1225
    html_static_data_options = get_nuitka_html_static_data_options()
1226
 
1227
    run_invoke(
1228
        context,
1229
        f"""
1230
        PYTHONPATH="{os.getcwd()}"
1231
        python -m nuitka
1232
            --static-libpython=no
1233
            --standalone
1234
            --include-module=textx
1235
            --include-module=strictdoc.server.app
1236
            --include-module=docutils
1237
            --include-module=docutils.readers.standalone
1238
            --include-module=docutils.parsers.rst
1239
            {html_template_data_options}
1240
            {html_static_data_options}
1241
            --include-data-dir=strictdoc/backend/rst/templates=templates/rst
1242
            --include-package-data=docutils
1243
            strictdoc/cli/main.py
1244
        """,
1245
    )
1246
 
1247
 
1248
# https://github.com/jrfonseca/gprof2dot
1249
# pip install gprof2dot
1250
@task()
1251
def performance(context):
1252
    command = """
1253
        python -m cProfile -o output/profile.prof
1254
            -m strictdoc.cli.main export . --no-parallelization &&
1255
        gprof2dot -f pstats output/profile.prof | dot -Tpng -o output/output.png
1256
    """
1257
    run_invoke(context, command)
1258
 
1259
 
1260
@task(performance)
1261
def performance_snakeviz(context):
1262
    command = """
1263
        snakeviz output/profile.prof
1264
    """
1265
    run_invoke(context, command)
1266
 
1267
 
1268
@task(aliases=["bd"])
1269
def build_docker(
1270
    context,
1271
    image: str = "strictdoc:latest",
1272
    no_cache: bool = False,
1273
    source="pypi",
1274
):
1275
    no_cache_argument = "--no-cache" if no_cache else ""
1276
    run_invoke(
1277
        context,
1278
        f"""
1279
        docker build .
1280
            --build-arg STRICTDOC_SOURCE={source}
1281
            -t {image}
1282
            {no_cache_argument}
1283
        """,
1284
    )
1285
 
1286
 
1287
@task(aliases=["rd"])
1288
def run_docker(
1289
    context, image: str = "strictdoc:latest", command: Optional[str] = None
1290
):
1291
    command_argument = (
1292
        f'/bin/bash -c "{command}"' if command is not None else ""
1293
    )
1294
 
1295
    run_invoke(
1296
        context,
1297
        f"""
1298
        docker run
1299
            --name strictdoc
1300
            --rm
1301
            -it
1302
            -e HOST_UID=$(id -u) -e HOST_GID=$(id -g)
1303
            -v "$(pwd):/data"
1304
            {image}
1305
            {command_argument}
1306
        """,
1307
        pty=True,
1308
    )
1309
 
1310
 
1311
@task(aliases=["td"])
1312
def test_docker(context, image: str = "strictdoc:latest"):
1313
    run_invoke(
1314
        context,
1315
        """
1316
        rm -rf output/ && mkdir -p output/ && chmod 777 output/
1317
        """,
1318
    )
1319
    run_docker(
1320
        context,
1321
        image=image,
1322
        command="strictdoc export --formats=html,html2pdf .",
1323
    )
1324
 
1325
    def check_file_owner(filepath):
1326
        import pwd  # noqa: PLC0415
1327
 
1328
        file_owner = pwd.getpwuid(os.stat(filepath).st_uid).pw_name
1329
        current_user = os.environ.get("USER", "")
1330
        return file_owner == current_user
1331
 
1332
    assert check_file_owner(
1333
        "output/html2pdf/pdf/docs/strictdoc_01_user_guide.pdf"
1334
    )
1335
 
1336
 
1337
@task(aliases=["q"])
1338
def qualification(context):
1339
    test_all(context, coverage=True, headless=True)
1340
    coverage_combine(context)
1341
 
1342
 
1343
@task()
1344
def drawio(context):
1345
    if sys.platform == "darwin":
1346
        path_to_drawio = "/Applications/draw.io.app/Contents/MacOS/draw.io"
1347
    elif sys.platform.startswith("linux"):
1348
        path_to_drawio = "drawio"
1349
    else:
1350
        raise NotImplementedError(
1351
            "drawio task is supported only on macOS and Linux."
1352
        )
1353
 
1354
    artifacts = [
1355
        (
1356
            "developer/drawio/Architecture.drawio",
1357
            "docs/_assets/StrictDoc_Workspace-Architecture.drawio.png",
1358
        ),
1359
        (
1360
            "developer/drawio/Backlog.drawio",
1361
            "docs/_assets/StrictDoc_Workspace-Backlog.drawio.png",
1362
        ),
1363
        (
1364
            "developer/drawio/Roadmap.drawio",
1365
            "docs/_assets/StrictDoc_Workspace-Roadmap.drawio.png",
1366
        ),
1367
    ]
1368
 
1369
    for path_to_drawio_, path_to_png_ in artifacts:
1370
        print(f"Copying: {path_to_drawio_} -> {path_to_png_}")  # noqa: T201
1371
 
1372
        # Basic safety for now to avoid writing wrong files.
1373
        assert os.path.isfile(path_to_drawio_), path_to_drawio_
1374
        assert os.path.isfile(path_to_png_), path_to_png_
1375
 
1376
        run_invoke(
1377
            context,
1378
            f"""
1379
            {path_to_drawio}
1380
                --export
1381
                --format png
1382
                -o {path_to_png_}
1383
                --page-index 0
1384
                {path_to_drawio_}
1385
            """,
1386
            pty=True,
1387
        )