StrictDoc Documentation
tasks.py
Source file coverage
Path:
tasks.py
Lines:
1310
Non-empty lines:
1147
Non-empty lines covered with requirements:
1147 / 1147 (100.0%)
Functions:
45
Functions covered by requirements:
45 / 45 (100.0%)
1
import os
2
import re
3
import shutil
4
import sys
5
import tempfile
6
from enum import Enum
7
from pathlib import Path
8
from typing import Dict, Optional
9
 
10
import invoke
11
from invoke import task
12
 
13
from developer.git.commit_validator import (
14
    validate_commits_locally_or_ci,
15
)
16
from strictdoc.core.environment import (
17
    BINARY_HTML_STATIC_DIR,
18
    BINARY_HTML_TEMPLATES_DIR,
19
    HTML_STATIC_DIRS,
20
    HTML_TEMPLATE_DIRS,
21
)
22
from strictdoc.core.project_config import ProjectConfigDefault
23
from tools.server_check import free_strictdoc_port
24
 
25
# Specifying encoding because Windows crashes otherwise when running Invoke
26
# tasks below:
27
# UnicodeEncodeError: 'charmap' codec can't encode character '\ufffd'
28
# in position 16: character maps to <undefined>
29
# People say, it might also be possible to export PYTHONIOENCODING=utf8 but this
30
# seems to work.
31
# FIXME: If you are a Windows user and expert, please advise on how to do this
32
# properly.
33
sys.stdout = open(1, "w", encoding="utf-8", closefd=False, buffering=1)
34
 
35
STRICTDOC_TMP_DIR = os.path.join(tempfile.gettempdir(), "strictdoc_tmp_dir")
36
TEST_REPORTS_DIR = "build/test_reports"
37
 
38
# Redirect all __pycache__ output (from Invoke itself and every subprocess it
39
# spawns: pytest, mypy, plain python invocations, etc.) into a single folder,
40
# instead of littering __pycache__ directories across the source tree.
41
os.environ["PYTHONPYCACHEPREFIX"] = os.path.abspath("build/pycache")
42
 
43
 
44
def get_pyinstaller_html_template_data_options() -> str:
45
    return "\n".join(
46
        f'--add-data "{template_dir}{os.pathsep}{BINARY_HTML_TEMPLATES_DIR}"'
47
        for template_dir in HTML_TEMPLATE_DIRS
48
    )
49
 
50
 
51
def get_pyinstaller_html_static_data_options() -> str:
52
    return "\n".join(
53
        f'--add-data "{static_dir}{os.pathsep}{BINARY_HTML_STATIC_DIR}"'
54
        for static_dir in HTML_STATIC_DIRS
55
    )
56
 
57
 
58
def get_nuitka_html_template_data_options() -> str:
59
    return "\n".join(
60
        f'--include-data-dir="{template_dir}={BINARY_HTML_TEMPLATES_DIR}"'
61
        for template_dir in HTML_TEMPLATE_DIRS
62
    )
63
 
64
 
65
def get_nuitka_html_static_data_options() -> str:
66
    return "\n".join(
67
        f'--include-data-dir="{static_dir}={BINARY_HTML_STATIC_DIR}"'
68
        for static_dir in HTML_STATIC_DIRS
69
    )
70
 
71
 
72
# To prevent all tasks from building to the same virtual environment.
73
# All values correspond to the configuration in the tox.ini config file.
74
class ToxEnvironment(str, Enum):
75
    DEVELOPMENT = "development"
76
    CHECK = "check"
77
    DOCUMENTATION = "documentation"
78
    RELEASE = "release"
79
    RELEASE_LOCAL = "release-local"
80
    PYINSTALLER = "pyinstaller"
81
 
82
 
83
def run_invoke(
84
    context,
85
    cmd,
86
    environment: Optional[dict] = None,
87
    pty: bool = False,
88
    warn: bool = False,
89
) -> invoke.runners.Result:
90
    def one_line_command(string):
91
        return re.sub("\\s+", " ", string).strip()
92
 
93
    return context.run(
94
        one_line_command(cmd),
95
        env=environment,
96
        hide=False,
97
        warn=warn,
98
        pty=pty,
99
        echo=True,
100
    )
101
 
102
 
103
def run_invoke_with_tox(
104
    context,
105
    environment_type: ToxEnvironment,
106
    command: str,
107
    environment: Optional[Dict] = None,
108
    pty: bool = False,
109
) -> invoke.runners.Result:
110
    assert isinstance(environment_type, ToxEnvironment)
111
    assert isinstance(command, str)
112
 
113
    tox_py_version = f"py{sys.version_info.major}{sys.version_info.minor}"
114
 
115
    return run_invoke(
116
        context,
117
        f"""
118
            tox
119
                -e {tox_py_version}-{environment_type.value} --
120
                {command}
121
        """,
122
        environment=environment,
123
        pty=pty,
124
    )
125
 
126
 
127
@task(default=True)
128
def list_tasks(context):
129
    clean_command = """
130
        invoke --list
131
    """
132
    run_invoke(context, clean_command)
133
 
134
 
135
@task
136
def clean(context):
137
    # https://unix.stackexchange.com/a/689930/77389
138
    clean_command = r"""
139
        rm -rf output/ docs/sphinx/build/ &&
140
        find tests/ -type d \( -name output -o -name Output \) -exec rm -rf {} +
141
    """
142
    run_invoke(context, clean_command)
143
 
144
 
145
@task(aliases=["s"])
146
def server(context, input_path=".", config=None, port=None):
147
    assert os.path.isdir(input_path), input_path
148
    if config is not None:
149
        assert os.path.isfile(config), config
150
    config_argument = f"--config {config}" if config is not None else ""
151
    port_argument = f"--port {port}" if port is not None else ""
152
 
153
    try:
154
        should_continue = free_strictdoc_port(
155
            port
156
            if port is not None
157
            else ProjectConfigDefault.DEFAULT_SERVER_PORT
158
        )
159
    except OSError as error:
160
        print(error, file=sys.stderr)  # noqa: T201
161
        return
162
 
163
    if not should_continue:
164
        return
165
 
166
    run_invoke_with_tox(
167
        context,
168
        ToxEnvironment.DEVELOPMENT,
169
        f"""
170
            python -m strictdoc.cli.main
171
                --debug
172
                --development
173
                server {input_path} {config_argument}
174
                    --host 127.0.0.1
175
                    {port_argument}
176
                    --reload
177
                    --watch
178
        """,
179
    )
180
 
181
 
182
@task(aliases=["d"])
183
def docs(context):
184
    run_invoke_with_tox(
185
        context,
186
        ToxEnvironment.DOCUMENTATION,
187
        """
188
            python3 -m strictdoc.cli.main
189
                export .
190
                    --formats=html
191
                    --output-dir output/strictdoc_website
192
                    --project-title "StrictDoc"
193
        """,
194
    )
195
 
196
    run_invoke_with_tox(
197
        context,
198
        ToxEnvironment.DOCUMENTATION,
199
        """
200
            python3 -m strictdoc.cli.main
201
                export ./
202
                    --formats=rst
203
                    --output-dir output/sphinx
204
                    --project-title "StrictDoc"
205
        """,
206
    )
207
 
208
    run_invoke_with_tox(
209
        context,
210
        ToxEnvironment.DOCUMENTATION,
211
        """
212
            cp -r output/sphinx/rst/docs/* docs/sphinx/source/ &&
213
            mkdir -p docs/sphinx/source/_assets/ &&
214
            cp -v docs/_assets/* docs/sphinx/source/_assets/
215
        """,
216
    )
217
 
218
    run_invoke_with_tox(
219
        context,
220
        ToxEnvironment.DOCUMENTATION,
221
        """
222
            make --directory docs/sphinx html latexpdf SPHINXOPTS="-W --keep-going"
223
        """,
224
    )
225
 
226
    run_invoke(
227
        context,
228
        (
229
            """
230
                open docs/sphinx/build/latex/strictdoc.pdf
231
            """
232
        ),
233
    )
234
 
235
 
236
@task(clean, aliases=["tus"])
237
def test_unit_server(context, focus=None):
238
    focus_argument = f"-k {focus}" if focus is not None else ""
239
 
240
    Path(TEST_REPORTS_DIR).mkdir(parents=True, exist_ok=True)
241
 
242
    cwd = os.getcwd()
243
 
244
    path_to_coverage_file = f"{cwd}/build/coverage/unit_server/.coverage"
245
 
246
    run_invoke_with_tox(
247
        context,
248
        ToxEnvironment.CHECK,
249
        f"""
250
            coverage run
251
            --rcfile=.coveragerc.unit_server
252
            --data-file={path_to_coverage_file}
253
            -m pytest
254
            tests/unit_server/
255
                {focus_argument}
256
                --junit-xml={TEST_REPORTS_DIR}/tests_unit_server.pytest.junit.xml
257
                -o junit_suite_name="StrictDoc Web Server Unit Tests"
258
                -o cache_dir=build/pytest_cache/unit_server
259
        """,
260
    )
261
 
262
 
263
@task(test_unit_server, aliases=["tusc"])
264
def test_unit_server_report(context):
265
    cwd = os.getcwd()
266
 
267
    path_to_coverage_file = f"{cwd}/build/coverage/unit_server/.coverage"
268
 
269
    run_invoke_with_tox(
270
        context,
271
        ToxEnvironment.CHECK,
272
        f"""
273
            coverage html
274
                --rcfile=.coveragerc.unit_server
275
                --data-file={path_to_coverage_file}
276
        """,
277
    )
278
 
279
 
280
@task(aliases=["te"])
281
def test_end2end(
282
    context,
283
    *,
284
    focus=None,
285
    exit_first=False,
286
    parallelize=False,
287
    long_timeouts=False,
288
    headless=False,
289
    headed=False,
290
    shard=None,
291
    test_path=None,
292
    coverage: bool = False,
293
):
294
    """
295
    @relation(SDOC-SRS-46, scope=function)
296
    """
297
 
298
    environment = {}
299
 
300
    coverage_command_or_none = ""
301
    coverage_argument_or_none = ""
302
 
303
    if coverage:
304
        cwd = os.getcwd()
305
        coverage_file_dir = f"{cwd}/build/coverage/end2end/"
306
        coverage_file_dir2 = f"{cwd}/build/coverage/end2end_strictdoc/"
307
        coverage_file = os.path.join(coverage_file_dir, ".coverage")
308
        coverage_rc = os.path.join(cwd, ".coveragerc.end2end")
309
        shutil.rmtree(coverage_file_dir, ignore_errors=True)
310
        shutil.rmtree(coverage_file_dir2, ignore_errors=True)
311
        coverage_command_or_none = f"""
312
            coverage run
313
                --rcfile={coverage_rc}
314
                --data-file={coverage_file}
315
                -m
316
        """
317
        coverage_argument_or_none = "--strictdoc-coverage"
318
 
319
    long_timeouts_argument = (
320
        "--strictdoc-long-timeouts" if long_timeouts else ""
321
    )
322
 
323
    parallelize_argument = ""
324
    if parallelize:
325
        print(  # noqa: T201
326
            "warning: "
327
            "Running parallelized end-2-end tests is supported "
328
            "but is not stable."
329
        )
330
        parallelize_argument = "--numprocesses=2 --strictdoc-parallelize"
331
 
332
    assert shard is None or re.match(r"[1-9][0-9]*/[1-9][0-9]*", shard), (
333
        f"--shard argument has an incorrect format: {shard}."
334
    )
335
    shard_argument = f"--strictdoc-shard={shard}" if shard else ""
336
 
337
    focus_argument = f"-k {focus}" if focus is not None else ""
338
    exit_first_argument = "--exitfirst" if exit_first else ""
339
    headless_argument = "--headless2" if headless and not headed else "--gui"
340
    test_command = f"""
341
            {coverage_command_or_none}
342
            pytest
343
            --failed-first
344
            --capture=no
345
            --reuse-session
346
            {parallelize_argument}
347
            {shard_argument}
348
            {coverage_argument_or_none}
349
            {focus_argument}
350
            {exit_first_argument}
351
            {long_timeouts_argument}
352
            {headless_argument}
353
            --junit-xml={TEST_REPORTS_DIR}/tests_end2end.pytest.junit.xml
354
            -o junit_suite_name="StrictDoc End-to-End Tests"
355
            -o cache_dir=build/pytest_cache/end2end
356
            tests/end2end
357
    """
358
    if test_path:
359
        test_command = test_command.rstrip() + f"/{test_path}"
360
 
361
    Path(TEST_REPORTS_DIR).mkdir(parents=True, exist_ok=True)
362
 
363
    # On Windows, GitHub Actions fails with:
364
    # response = {'status': 500, 'value':
365
    # '{"value":{"error":"unknown error",
366
    # "message":"unknown error: cannot find Chrome binary",  # noqa: ERA001
367
    # This very likely has to do with PATH isolation that Tox does.
368
    # FIXME: If you are a Windows expert, please fix this to run on Tox.
369
    if os.name == "nt":
370
        run_invoke(context, test_command)
371
        return
372
 
373
    run_invoke_with_tox(
374
        context,
375
        ToxEnvironment.CHECK,
376
        test_command,
377
        environment=environment,
378
    )
379
 
380
 
381
@task(aliases=["tu"])
382
def test_unit(context, coverage=False, focus=None, path=None, output=False):
383
    """
384
    @relation(SDOC-SRS-44, scope=function)
385
    """
386
 
387
    Path(TEST_REPORTS_DIR).mkdir(parents=True, exist_ok=True)
388
 
389
    focus_argument = f"-k {focus}" if focus is not None else ""
390
    output_argument = "--capture=no" if output else ""
391
 
392
    cwd = os.getcwd()
393
 
394
    if path is None:
395
        path = "tests/unit"
396
    else:
397
        assert "tests/unit" in path, path
398
 
399
    path_to_coverage_file = f"{cwd}/build/coverage/unit/.coverage"
400
 
401
    pytest_command = (
402
        """
403
        coverage run
404
            --rcfile=.coveragerc.unit
405
            --data-file={path_to_coverage_file}
406
            -m pytest
407
    """
408
        if coverage
409
        else "pytest"
410
    )
411
 
412
    run_invoke_with_tox(
413
        context,
414
        ToxEnvironment.CHECK,
415
        f"""
416
            {pytest_command}
417
            {focus_argument}
418
            {output_argument}
419
            --junit-xml={TEST_REPORTS_DIR}/tests_unit.pytest.junit.xml
420
            -o cache_dir=build/pytest_cache/unit
421
            -o junit_suite_name="StrictDoc Unit Tests"
422
            -p no:seleniumbase
423
            {path}
424
        """,
425
    )
426
    if coverage and not focus and path == "tests/unit":
427
        run_invoke_with_tox(
428
            context,
429
            ToxEnvironment.CHECK,
430
            f"""
431
                coverage report
432
                    --sort=cover
433
                    --rcfile=.coveragerc.unit
434
                    --data-file={path_to_coverage_file}
435
            """,
436
        )
437
 
438
 
439
@task(test_unit, aliases=["tuc"])
440
def test_unit_report(context):
441
    cwd = os.getcwd()
442
 
443
    path_to_coverage_file = f"{cwd}/build/coverage/unit/.coverage"
444
 
445
    run_invoke_with_tox(
446
        context,
447
        ToxEnvironment.CHECK,
448
        f"""
449
            coverage html
450
                --rcfile=.coveragerc.unit
451
                --data-file={path_to_coverage_file}
452
        """,
453
    )
454
 
455
 
456
@task(clean, aliases=["ti"])
457
def test_integration(
458
    context,
459
    focus=None,
460
    debug=False,
461
    no_parallelization=False,
462
    fail_first=False,
463
    coverage=False,
464
    strictdoc=None,
465
    html2pdf=False,
466
    shard=None,
467
    environment=ToxEnvironment.CHECK,
468
):
469
    """
470
    @relation(SDOC-SRS-45, scope=function)
471
    """
472
 
473
    cwd = os.getcwd()
474
 
475
    if strictdoc is None:
476
        strictdoc_exec = "python3 -m strictdoc.cli.main"
477
    else:
478
        strictdoc_exec = strictdoc
479
 
480
    coverage_path_argument = ""
481
    if coverage:
482
        path_to_coverage_rc = f"{cwd}/.coveragerc.integration"
483
        strictdoc_exec = (
484
            f"coverage run --rcfile={path_to_coverage_rc} -m strictdoc.cli.main"
485
        )
486
        if html2pdf:
487
            path_to_coverage_dir = f"{cwd}/build/coverage/integration_html2pdf/"
488
        else:
489
            path_to_coverage_dir = f"{cwd}/build/coverage/integration/"
490
        path_to_coverage = os.path.join(path_to_coverage_dir, ".coverage")
491
        shutil.rmtree(path_to_coverage_dir, ignore_errors=True)
492
        coverage_path_argument = (
493
            f'--param COVERAGE_FILE="{path_to_coverage}" '
494
            f'--param COVERAGE_PROCESS_START="{path_to_coverage_rc}"'
495
        )
496
 
497
    debug_opts = "-vv --show-all" if debug else ""
498
    focus_or_none = f"--filter {focus}" if focus else ""
499
    fail_first_argument = "--max-failures 1" if fail_first else ""
500
    junit_xml_report_argument = (
501
        "--xunit-xml-output build/test_reports/tests_integration_html2pdf.lit.junit.xml"
502
        if html2pdf
503
        else "--xunit-xml-output build/test_reports/tests_integration.lit.junit.xml"
504
    )
505
 
506
    # Allow partitioning of integration and html2pdf tests
507
    partition_opts = ""
508
    if shard is not None:
509
        match = re.match(r"([1-9][0-9]*)/([1-9][0-9]*)", shard)
510
        assert match, f"--shard argument has an incorrect format: {shard}."
511
        run_shard = int(match.group(1))
512
        num_shards = int(match.group(2))
513
        partition_opts = f"--num-shards={num_shards} --run-shard={run_shard}"
514
 
515
    # HTML2PDF tests are running Chrome Driver which does not seem to be
516
    # parallelizable, or at least not in the way StrictDoc uses it.
517
    # If HTML2PDF option is provided, do not parallelize and only run the
518
    # HTML2PDF-specific tests.
519
    # HTML2PDF tests can be safely partitioned.
520
    chromedriver_param = ""
521
    if not html2pdf:
522
        parallelize_opts = "" if not no_parallelization else "--threads 1"
523
        html2pdf_param = ""
524
        test_folder = f"{cwd}/tests/integration"
525
        test_output_dir = "build/tests_integration"
526
    else:
527
        parallelize_opts = "--threads 1"
528
        html2pdf_param = "--param TEST_HTML2PDF=1"
529
        chromedriver_path = os.environ.get("CHROMEWEBDRIVER")
530
        if chromedriver_path is not None:
531
            # NOTE: isfile() check does not work on GitHub Actions / Linux,
532
            #       the exists() check works.
533
            assert os.path.exists(chromedriver_path), chromedriver_path
534
            chromedriver_param = f"--param CHROMEDRIVER={os.path.join(chromedriver_path, 'chromedriver')}"
535
            if os.name == "nt":
536
                # On Windows, its chromdriver.exe
537
                chromedriver_param = chromedriver_param + ".exe"
538
        test_folder = f"{cwd}/tests/integration/features/html2pdf"
539
        test_output_dir = "build/tests_integration_html2pdf"
540
 
541
    # The command sometimes exits with 1 even if the files are deleted.
542
    # warn=True ensures that the execution continues.
543
    run_invoke(
544
        context,
545
        f"""
546
        rm -rf {test_output_dir}
547
        """,
548
        warn=True,
549
    )
550
 
551
    run_invoke(
552
        context,
553
        f"""
554
        rm -rf {STRICTDOC_TMP_DIR}
555
        """,
556
    )
557
 
558
    Path(STRICTDOC_TMP_DIR).mkdir(exist_ok=True)
559
    Path(TEST_REPORTS_DIR).mkdir(parents=True, exist_ok=True)
560
 
561
    itest_command = f"""
562
        lit
563
        --param STRICTDOC_EXEC="{strictdoc_exec}"
564
        --param STRICTDOC_TMP_DIR="{STRICTDOC_TMP_DIR}"
565
        --param TEST_OUTPUT_DIR="{test_output_dir}"
566
        --timeout 180
567
        --order smart
568
        {junit_xml_report_argument}
569
        {coverage_path_argument}
570
        {html2pdf_param}
571
        {chromedriver_param}
572
        -v
573
        {debug_opts}
574
        {focus_or_none}
575
        {fail_first_argument}
576
        {parallelize_opts}
577
        {partition_opts}
578
        {test_folder}
579
    """
580
 
581
    # It looks like LIT does not open the RUN: subprocesses in the same
582
    # environment from which it itself is run from. This issue has been known by
583
    # us for a couple of years by now. Not using Tox on Windows for the time
584
    # being.
585
    if os.name == "nt":
586
        run_invoke(context, itest_command)
587
        return
588
 
589
    run_invoke_with_tox(
590
        context,
591
        environment,
592
        itest_command,
593
        environment={"STRICTDOC_CACHE_DIR": "Output/_cache"},
594
    )
595
 
596
 
597
@task
598
def coverage_combine(context):
599
    run_invoke_with_tox(
600
        context,
601
        ToxEnvironment.CHECK,
602
        """
603
            coverage combine
604
                --data-file build/coverage/.coverage.combined
605
                --keep
606
                build/coverage/end2end_strictdoc/.coverage.*
607
                build/coverage/integration/.coverage.*
608
                build/coverage/integration_html2pdf/.coverage.*
609
                build/coverage/unit/.coverage
610
                build/coverage/unit_server/.coverage
611
        """,
612
    )
613
    run_invoke_with_tox(
614
        context,
615
        ToxEnvironment.CHECK,
616
        """
617
            coverage html
618
                --rcfile .coveragerc.combined
619
                --data-file build/coverage/.coverage.combined
620
        """,
621
    )
622
    run_invoke_with_tox(
623
        context,
624
        ToxEnvironment.CHECK,
625
        """
626
            coverage json
627
                --rcfile .coveragerc.combined
628
                --data-file build/coverage/.coverage.combined
629
                --pretty-print
630
                -o build/coverage/coverage.combined.json
631
        """,
632
    )
633
 
634
 
635
@task
636
def lint_ruff_format(context):
637
    """
638
    @relation(SDOC-SRS-42, scope=function)
639
    """
640
 
641
    result: invoke.runners.Result = run_invoke_with_tox(
642
        context,
643
        ToxEnvironment.CHECK,
644
        """
645
            ruff
646
                format
647
                --cache-dir build/ruff_cache
648
                *.py
649
                developer/
650
                docs/
651
                strictdoc/
652
                tools/ecss
653
                tests/unit/
654
                tests/unit_server/
655
                tests/integration/*.py
656
                tests/end2end/
657
        """,
658
    )
659
    # Ruff always exits with 0, so we handle the output.
660
    if "reformatted" in result.stdout:
661
        print("invoke: ruff format found issues")  # noqa: T201
662
        result.exited = 1
663
        raise invoke.exceptions.UnexpectedExit(result)
664
 
665
 
666
@task(aliases=["lr"])
667
def lint_ruff(context):
668
    """
669
    @relation(SDOC-SRS-42, scope=function)
670
    """
671
 
672
    run_invoke_with_tox(
673
        context,
674
        ToxEnvironment.CHECK,
675
        """
676
            ruff check . --fix --exit-non-zero-on-fix --cache-dir build/ruff_cache
677
        """,
678
    )
679
 
680
 
681
@task(aliases=["lm"])
682
def lint_mypy(context):
683
    """
684
    @relation(SDOC-SRS-41, SDOC-SRS-43, scope=function)
685
    """
686
 
687
    # These checks do not seem to be useful:
688
    # - import
689
    # --disallow-any-expr
690
    # --disallow-any-explicit
691
    # --disallow-any-unimported  # noqa: ERA001
692
    # --disallow-any-decorated
693
    # - type-abstract. It is ignored on purpose because of assert_cast()
694
    #   implementation. See https://stackoverflow.com/a/74073453/598057.
695
    run_invoke_with_tox(
696
        context,
697
        ToxEnvironment.CHECK,
698
        """
699
            mypy docs/
700
                 strictdoc/
701
                 tests/unit/strictdoc/backend/sdoc_source_code/test_marker_lexer.py
702
 
703
                --show-error-codes
704
                --disable-error-code=import
705
                --disable-error-code=type-abstract
706
                --cache-dir=build/mypy_cache
707
                --extra-checks
708
 
709
                --strict
710
                --strict-optional
711
                --strict-equality
712
 
713
                --check-untyped-defs
714
                --disallow-any-generics
715
                --disallow-incomplete-defs
716
                --disallow-subclassing-any
717
                --disallow-untyped-calls
718
                --disallow-untyped-decorators
719
                --disallow-untyped-defs
720
                --no-implicit-optional
721
                --warn-no-return
722
                --warn-redundant-casts
723
                --warn-return-any
724
                --warn-unreachable
725
                --warn-unused-ignores
726
 
727
                --python-version=3.10
728
        """,
729
    )
730
 
731
 
732
@task
733
def lint_format_js(context):
734
    # NOTE: Could not find the '--' equivalent for -w80.
735
    result: invoke.runners.Result = run_invoke_with_tox(
736
        context,
737
        ToxEnvironment.CHECK,
738
        """
739
            js-beautify
740
                --indent-size=2
741
                --end-with-newline
742
                --replace
743
                -w100
744
                strictdoc/export/html/_static/static_html_search.js
745
                strictdoc/export/html/_static/stable_uri_forwarder.js
746
        """,
747
    )
748
    # Ruff always exits with 0, so we handle the output.
749
    if "reformatted" in result.stdout:
750
        print("invoke: ruff format found issues")  # noqa: T201
751
        result.exited = 1
752
        raise invoke.exceptions.UnexpectedExit(result)
753
 
754
 
755
@task(aliases=["lc"])
756
def lint_commit(context):  # noqa: ARG001
757
    try:
758
        validate_commits_locally_or_ci()
759
    except ValueError as e:
760
        raise invoke.exceptions.Exit(message=str(e), code=1) from None
761
 
762
 
763
@task(aliases=["lf"])
764
def lint_fixit(context, fix=False, auto=False, path="strictdoc/"):
765
    if fix:
766
        auto_argument = "--automatic" if auto else ""
767
        run_invoke_with_tox(
768
            context,
769
            ToxEnvironment.CHECK,
770
            f"""
771
                fixit fix {path} {auto_argument}
772
            """,
773
            pty=True,
774
        )
775
    else:
776
        run_invoke_with_tox(
777
            context,
778
            ToxEnvironment.CHECK,
779
            f"""
780
                fixit lint --diff {path}
781
            """,
782
        )
783
 
784
 
785
@task(aliases=["l"])
786
def lint(context):
787
    lint_commit(context)
788
    lint_ruff_format(context)
789
    lint_ruff(context)
790
    lint_mypy(context)
791
 
792
 
793
@task(aliases=["t"])
794
def test(context, shard=None):
795
    test_unit(context)
796
    test_unit_server(context)
797
    test_integration(context, shard=shard)
798
 
799
 
800
@task(aliases=["ta"])
801
def test_all(context, coverage=False, headless=False):
802
    test_unit(context, coverage=coverage)
803
    test_unit_server(context)
804
    test_integration(context, coverage=coverage)
805
    test_integration(context, coverage=coverage, html2pdf=True)
806
    test_end2end(context, coverage=coverage, headless=headless)
807
 
808
 
809
@task(aliases=["c"])
810
def check(context):
811
    lint(context)
812
    test(context)
813
 
814
 
815
# https://github.com/github-changelog-generator/github-changelog-generator
816
# gem install github_changelog_generator
817
@task
818
def changelog(context, github_token):
819
    # The alpha release tags are excluded from the changelog.
820
    command = f"""
821
        github_changelog_generator
822
        --token {github_token}
823
        --user strictdoc-project
824
        --exclude-tags-regex ".*a\\d+"
825
        --project strictdoc
826
        """
827
    run_invoke(context, command)
828
 
829
 
830
@task
831
def check_dead_links(context):
832
    run_invoke_with_tox(
833
        context,
834
        ToxEnvironment.CHECK,
835
        """
836
            python3 tools/link_health.py docs/strictdoc_01_user_guide.sdoc
837
        """,
838
    )
839
    run_invoke_with_tox(
840
        context,
841
        ToxEnvironment.CHECK,
842
        """
843
            python3 tools/link_health.py docs/strictdoc_02_feature_map.sdoc
844
        """,
845
    )
846
    run_invoke_with_tox(
847
        context,
848
        ToxEnvironment.CHECK,
849
        """
850
            python3 tools/link_health.py docs/strictdoc_03_faq.sdoc
851
        """,
852
    )
853
    run_invoke_with_tox(
854
        context,
855
        ToxEnvironment.CHECK,
856
        """
857
            python3 tools/link_health.py docs/strictdoc_04_release_notes.sdoc
858
        """,
859
    )
860
    run_invoke_with_tox(
861
        context,
862
        ToxEnvironment.CHECK,
863
        """
864
            python3 tools/link_health.py docs/strictdoc_05_troubleshooting.sdoc
865
        """,
866
    )
867
    run_invoke_with_tox(
868
        context,
869
        ToxEnvironment.CHECK,
870
        """
871
            python3 tools/link_health.py docs/strictdoc_10_contributing.sdoc
872
        """,
873
    )
874
    run_invoke_with_tox(
875
        context,
876
        ToxEnvironment.CHECK,
877
        """
878
            python3 tools/link_health.py docs/strictdoc_11_developer_guide.sdoc
879
        """,
880
    )
881
    run_invoke_with_tox(
882
        context,
883
        ToxEnvironment.CHECK,
884
        """
885
            python3 tools/link_health.py docs/strictdoc_24_development_plan.sdoc
886
        """,
887
    )
888
    run_invoke_with_tox(
889
        context,
890
        ToxEnvironment.CHECK,
891
        """
892
            python3 tools/link_health.py docs/strictdoc_20_l1_system_requirements.sdoc
893
        """,
894
    )
895
    run_invoke_with_tox(
896
        context,
897
        ToxEnvironment.CHECK,
898
        """
899
            python3 tools/link_health.py docs/strictdoc_21_l2_high_level_requirements.sdoc
900
        """,
901
    )
902
    run_invoke_with_tox(
903
        context,
904
        ToxEnvironment.CHECK,
905
        """
906
            python3 tools/link_health.py docs/strictdoc_25_design.sdoc
907
        """,
908
    )
909
    run_invoke_with_tox(
910
        context,
911
        ToxEnvironment.CHECK,
912
        """
913
            python3 tools/link_health.py CONTRIBUTING.md
914
        """,
915
    )
916
    run_invoke_with_tox(
917
        context,
918
        ToxEnvironment.CHECK,
919
        """
920
            python3 tools/link_health.py NOTICE
921
        """,
922
    )
923
    run_invoke_with_tox(
924
        context,
925
        ToxEnvironment.CHECK,
926
        """
927
            python3 tools/link_health.py README.md
928
        """,
929
    )
930
 
931
 
932
@task
933
def release_local(context):
934
    run_invoke(
935
        context,
936
        """
937
            rm -rfv build/
938
        """,
939
    )
940
    run_invoke(
941
        context,
942
        """
943
            pip uninstall strictdoc -y
944
        """,
945
    )
946
    run_invoke_with_tox(
947
        context,
948
        ToxEnvironment.RELEASE_LOCAL,
949
        """
950
            python -m build --outdir build/dist
951
        """,
952
    )
953
    run_invoke_with_tox(
954
        context,
955
        ToxEnvironment.RELEASE_LOCAL,
956
        """
957
            twine check build/dist/*
958
        """,
959
    )
960
    run_invoke_with_tox(
961
        context,
962
        ToxEnvironment.RELEASE_LOCAL,
963
        """
964
            pip install build/dist/*.tar.gz
965
        """,
966
    )
967
    test_integration(
968
        context, strictdoc="strictdoc", environment=ToxEnvironment.RELEASE_LOCAL
969
    )
970
 
971
 
972
@task
973
def release(context, test_pypi=False, username=None, password=None):
974
    """
975
    A release can be made to PyPI or test package index (TestPyPI):
976
    https://pypi.org/project/strictdoc/
977
    https://test.pypi.org/project/strictdoc/
978
    """
979
 
980
    env_user = os.environ.get("TWINE_USERNAME")
981
    env_pass = os.environ.get("TWINE_PASSWORD")
982
 
983
    assert not ((username or password) and (env_user or env_pass)), (
984
        username,
985
        password,
986
        env_user,
987
        env_pass,
988
    )
989
    assert (username and password) or (env_user and env_pass), (
990
        username,
991
        password,
992
        env_user,
993
        env_pass,
994
    )
995
    if env_user:
996
        assert env_user == "__token__"
997
 
998
    repository_argument_or_none = ""
999
    if username is not None and password is not None:
1000
        repository_argument_or_none = (
1001
            ""
1002
            if username
1003
            else (
1004
                "--repository strictdoc_test"
1005
                if test_pypi
1006
                else "--repository strictdoc_release"
1007
            )
1008
        )
1009
    user_password = f"-u{username} -p{password}" if username is not None else ""
1010
 
1011
    run_invoke(
1012
        context,
1013
        """
1014
            rm -rfv build/dist/
1015
        """,
1016
    )
1017
    run_invoke_with_tox(
1018
        context,
1019
        ToxEnvironment.RELEASE,
1020
        """
1021
            python3 -m build --outdir build/dist
1022
        """,
1023
    )
1024
    run_invoke_with_tox(
1025
        context,
1026
        ToxEnvironment.RELEASE,
1027
        """
1028
            twine check build/dist/*
1029
        """,
1030
    )
1031
    # The token is in a core developer's .pypirc file.
1032
    # https://test.pypi.org/manage/account/token/
1033
    # https://packaging.python.org/en/latest/specifications/pypirc/#pypirc
1034
    run_invoke_with_tox(
1035
        context,
1036
        ToxEnvironment.RELEASE,
1037
        f"""
1038
            twine upload build/dist/strictdoc-*.tar.gz build/dist/strictdoc-*.whl
1039
                {repository_argument_or_none}
1040
                {user_password}
1041
        """,
1042
    )
1043
 
1044
 
1045
@task
1046
def release_pyinstaller(context):
1047
    path_to_pyi_dist = "/tmp/strictdoc"
1048
    html_template_data_options = get_pyinstaller_html_template_data_options()
1049
    html_static_data_options = get_pyinstaller_html_static_data_options()
1050
 
1051
    # The --hidden-import strictdoc.server.app flag is needed because without
1052
    # it, the following is produced:
1053
    # ERROR: Error loading ASGI app. Could not import
1054
    # module "strictdoc.server.app".
1055
    # Solution found here: https://stackoverflow.com/a/71340437/598057
1056
    # This behavior is not surprising because that's how the uvicorn loads the
1057
    # application separately from the parent process.
1058
    #
1059
    # Compatibility modules can be imported by user-provided statistics
1060
    # generators at runtime. PyInstaller cannot discover these imports
1061
    # statically because the generators live outside of StrictDoc's package.
1062
    #
1063
    # --hidden-import strictdoc.api is needed for the same reason:
1064
    # strictdoc.api is never imported anywhere inside StrictDoc's own
1065
    # package (only by dynamically-loaded, external files such as a
1066
    # project's own strictdoc_config.py, custom statistics generators, or
1067
    # custom plugins), so PyInstaller's static analysis -- which walks
1068
    # imports starting from strictdoc/cli/main.py -- never discovers it on
1069
    # its own.
1070
    command = f"""
1071
        pyinstaller
1072
            --clean
1073
            --name strictdoc
1074
            --noconfirm
1075
            --additional-hooks-dir developer/pyinstaller_hooks
1076
            --distpath {path_to_pyi_dist}
1077
            --hidden-import strictdoc.api
1078
            --hidden-import strictdoc.backend.rst.strictdoc_lexer
1079
            --hidden-import strictdoc.core.statistics.metric
1080
            --hidden-import strictdoc.export.html.generators.project_statistics
1081
            --hidden-import strictdoc.export.html.generators.view_objects.project_statistics_view_object
1082
            --hidden-import strictdoc.export.html.generators.view_objects.project_tree_stats
1083
            --hidden-import strictdoc.server.app
1084
            {html_template_data_options}
1085
            {html_static_data_options}
1086
            --add-data strictdoc/backend/rst/templates:templates/rst
1087
            strictdoc/cli/main.py
1088
    """
1089
 
1090
    run_invoke_with_tox(
1091
        context,
1092
        ToxEnvironment.PYINSTALLER,
1093
        """
1094
    pyinstaller --version
1095
    """,
1096
    )
1097
 
1098
    run_invoke_with_tox(context, ToxEnvironment.PYINSTALLER, command)
1099
 
1100
 
1101
@task
1102
def watch(context, sdocs_path="."):
1103
    strictdoc_command = f"""
1104
        python -m strictdoc.cli.main
1105
            export
1106
            {sdocs_path}
1107
            --output-dir output/
1108
    """
1109
 
1110
    run_invoke_with_tox(
1111
        context,
1112
        ToxEnvironment.DEVELOPMENT,
1113
        f"""
1114
            {strictdoc_command}
1115
        """,
1116
    )
1117
 
1118
    paths_to_watch = "."
1119
    run_invoke_with_tox(
1120
        context,
1121
        ToxEnvironment.DEVELOPMENT,
1122
        f"""
1123
            watchmedo shell-command
1124
            --patterns="*.py;*.sdoc;*.jinja;*.html;*.css;*.js"
1125
            --recursive
1126
            --ignore-pattern='output/;tests/integration'
1127
            --command='{strictdoc_command}'
1128
            --drop
1129
            {paths_to_watch}
1130
        """,
1131
    )
1132
 
1133
 
1134
@task
1135
def run(context, command):
1136
    run_invoke_with_tox(
1137
        context,
1138
        ToxEnvironment.DEVELOPMENT,
1139
        f"""
1140
            {command}
1141
        """,
1142
    )
1143
 
1144
 
1145
@task
1146
def nuitka(context):
1147
    html_template_data_options = get_nuitka_html_template_data_options()
1148
    html_static_data_options = get_nuitka_html_static_data_options()
1149
 
1150
    run_invoke(
1151
        context,
1152
        f"""
1153
        PYTHONPATH="{os.getcwd()}"
1154
        python -m nuitka
1155
            --static-libpython=no
1156
            --standalone
1157
            --include-module=textx
1158
            --include-module=strictdoc.server.app
1159
            --include-module=docutils
1160
            --include-module=docutils.readers.standalone
1161
            --include-module=docutils.parsers.rst
1162
            {html_template_data_options}
1163
            {html_static_data_options}
1164
            --include-data-dir=strictdoc/backend/rst/templates=templates/rst
1165
            --include-package-data=docutils
1166
            strictdoc/cli/main.py
1167
        """,
1168
    )
1169
 
1170
 
1171
# https://github.com/jrfonseca/gprof2dot
1172
# pip install gprof2dot
1173
@task()
1174
def performance(context):
1175
    command = """
1176
        python -m cProfile -o output/profile.prof
1177
            -m strictdoc.cli.main export . --no-parallelization &&
1178
        gprof2dot -f pstats output/profile.prof | dot -Tpng -o output/output.png
1179
    """
1180
    run_invoke(context, command)
1181
 
1182
 
1183
@task(performance)
1184
def performance_snakeviz(context):
1185
    command = """
1186
        snakeviz output/profile.prof
1187
    """
1188
    run_invoke(context, command)
1189
 
1190
 
1191
@task(aliases=["bd"])
1192
def build_docker(
1193
    context,
1194
    image: str = "strictdoc:latest",
1195
    no_cache: bool = False,
1196
    source="pypi",
1197
):
1198
    no_cache_argument = "--no-cache" if no_cache else ""
1199
    run_invoke(
1200
        context,
1201
        f"""
1202
        docker build .
1203
            --build-arg STRICTDOC_SOURCE={source}
1204
            -t {image}
1205
            {no_cache_argument}
1206
        """,
1207
    )
1208
 
1209
 
1210
@task(aliases=["rd"])
1211
def run_docker(
1212
    context, image: str = "strictdoc:latest", command: Optional[str] = None
1213
):
1214
    command_argument = (
1215
        f'/bin/bash -c "{command}"' if command is not None else ""
1216
    )
1217
 
1218
    run_invoke(
1219
        context,
1220
        f"""
1221
        docker run
1222
            --name strictdoc
1223
            --rm
1224
            -it
1225
            -e HOST_UID=$(id -u) -e HOST_GID=$(id -g)
1226
            -v "$(pwd):/data"
1227
            {image}
1228
            {command_argument}
1229
        """,
1230
        pty=True,
1231
    )
1232
 
1233
 
1234
@task(aliases=["td"])
1235
def test_docker(context, image: str = "strictdoc:latest"):
1236
    run_invoke(
1237
        context,
1238
        """
1239
        rm -rf output/ && mkdir -p output/ && chmod 777 output/
1240
        """,
1241
    )
1242
    run_docker(
1243
        context,
1244
        image=image,
1245
        command="strictdoc export --formats=html,html2pdf .",
1246
    )
1247
 
1248
    def check_file_owner(filepath):
1249
        import pwd  # noqa: PLC0415
1250
 
1251
        file_owner = pwd.getpwuid(os.stat(filepath).st_uid).pw_name
1252
        current_user = os.environ.get("USER", "")
1253
        return file_owner == current_user
1254
 
1255
    assert check_file_owner(
1256
        "output/html2pdf/pdf/docs/strictdoc_01_user_guide.pdf"
1257
    )
1258
 
1259
 
1260
@task(aliases=["q"])
1261
def qualification(context):
1262
    test_all(context, coverage=True, headless=True)
1263
    coverage_combine(context)
1264
 
1265
 
1266
@task()
1267
def drawio(context):
1268
    if sys.platform == "darwin":
1269
        path_to_drawio = "/Applications/draw.io.app/Contents/MacOS/draw.io"
1270
    elif sys.platform.startswith("linux"):
1271
        path_to_drawio = "drawio"
1272
    else:
1273
        raise NotImplementedError(
1274
            "drawio task is supported only on macOS and Linux."
1275
        )
1276
 
1277
    artifacts = [
1278
        (
1279
            "developer/drawio/Architecture.drawio",
1280
            "docs/_assets/StrictDoc_Workspace-Architecture.drawio.png",
1281
        ),
1282
        (
1283
            "developer/drawio/Backlog.drawio",
1284
            "docs/_assets/StrictDoc_Workspace-Backlog.drawio.png",
1285
        ),
1286
        (
1287
            "developer/drawio/Roadmap.drawio",
1288
            "docs/_assets/StrictDoc_Workspace-Roadmap.drawio.png",
1289
        ),
1290
    ]
1291
 
1292
    for path_to_drawio_, path_to_png_ in artifacts:
1293
        print(f"Copying: {path_to_drawio_} -> {path_to_png_}")  # noqa: T201
1294
 
1295
        # Basic safety for now to avoid writing wrong files.
1296
        assert os.path.isfile(path_to_drawio_), path_to_drawio_
1297
        assert os.path.isfile(path_to_png_), path_to_png_
1298
 
1299
        run_invoke(
1300
            context,
1301
            f"""
1302
            {path_to_drawio}
1303
                --export
1304
                --format png
1305
                -o {path_to_png_}
1306
                --page-index 0
1307
                {path_to_drawio_}
1308
            """,
1309
            pty=True,
1310
        )