StrictDoc Documentation
tasks.py
Source file coverage
Path:
tasks.py
Lines:
1278
Non-empty lines:
1119
Non-empty lines covered with requirements:
1119 / 1119 (100.0%)
Functions:
45
Functions covered by requirements:
45 / 45 (100.0%)
1
import os
2
import re
3
import shutil
4
import sys
5
import tempfile
6
from enum import Enum
7
from pathlib import Path
8
from typing import Dict, Optional
9
 
10
import invoke
11
from invoke import task
12
 
13
from developer.git.commit_validator import (
14
    validate_commits_locally_or_ci,
15
)
16
from strictdoc.core.environment import (
17
    BINARY_HTML_STATIC_DIR,
18
    BINARY_HTML_TEMPLATES_DIR,
19
    HTML_STATIC_DIRS,
20
    HTML_TEMPLATE_DIRS,
21
)
22
 
23
# Specifying encoding because Windows crashes otherwise when running Invoke
24
# tasks below:
25
# UnicodeEncodeError: 'charmap' codec can't encode character '\ufffd'
26
# in position 16: character maps to <undefined>
27
# People say, it might also be possible to export PYTHONIOENCODING=utf8 but this
28
# seems to work.
29
# FIXME: If you are a Windows user and expert, please advise on how to do this
30
# properly.
31
sys.stdout = open(1, "w", encoding="utf-8", closefd=False, buffering=1)
32
 
33
STRICTDOC_TMP_DIR = os.path.join(tempfile.gettempdir(), "strictdoc_tmp_dir")
34
TEST_REPORTS_DIR = "build/test_reports"
35
 
36
 
37
def get_pyinstaller_html_template_data_options() -> str:
38
    return "\n".join(
39
        f'--add-data "{template_dir}{os.pathsep}{BINARY_HTML_TEMPLATES_DIR}"'
40
        for template_dir in HTML_TEMPLATE_DIRS
41
    )
42
 
43
 
44
def get_pyinstaller_html_static_data_options() -> str:
45
    return "\n".join(
46
        f'--add-data "{static_dir}{os.pathsep}{BINARY_HTML_STATIC_DIR}"'
47
        for static_dir in HTML_STATIC_DIRS
48
    )
49
 
50
 
51
def get_nuitka_html_template_data_options() -> str:
52
    return "\n".join(
53
        f'--include-data-dir="{template_dir}={BINARY_HTML_TEMPLATES_DIR}"'
54
        for template_dir in HTML_TEMPLATE_DIRS
55
    )
56
 
57
 
58
def get_nuitka_html_static_data_options() -> str:
59
    return "\n".join(
60
        f'--include-data-dir="{static_dir}={BINARY_HTML_STATIC_DIR}"'
61
        for static_dir in HTML_STATIC_DIRS
62
    )
63
 
64
 
65
# To prevent all tasks from building to the same virtual environment.
66
# All values correspond to the configuration in the tox.ini config file.
67
class ToxEnvironment(str, Enum):
68
    DEVELOPMENT = "development"
69
    CHECK = "check"
70
    DOCUMENTATION = "documentation"
71
    RELEASE = "release"
72
    RELEASE_LOCAL = "release-local"
73
    PYINSTALLER = "pyinstaller"
74
 
75
 
76
def run_invoke(
77
    context,
78
    cmd,
79
    environment: Optional[dict] = None,
80
    pty: bool = False,
81
    warn: bool = False,
82
) -> invoke.runners.Result:
83
    def one_line_command(string):
84
        return re.sub("\\s+", " ", string).strip()
85
 
86
    return context.run(
87
        one_line_command(cmd),
88
        env=environment,
89
        hide=False,
90
        warn=warn,
91
        pty=pty,
92
        echo=True,
93
    )
94
 
95
 
96
def run_invoke_with_tox(
97
    context,
98
    environment_type: ToxEnvironment,
99
    command: str,
100
    environment: Optional[Dict] = None,
101
    pty: bool = False,
102
) -> invoke.runners.Result:
103
    assert isinstance(environment_type, ToxEnvironment)
104
    assert isinstance(command, str)
105
 
106
    tox_py_version = f"py{sys.version_info.major}{sys.version_info.minor}"
107
 
108
    return run_invoke(
109
        context,
110
        f"""
111
            tox
112
                -e {tox_py_version}-{environment_type.value} --
113
                {command}
114
        """,
115
        environment=environment,
116
        pty=pty,
117
    )
118
 
119
 
120
@task(default=True)
121
def list_tasks(context):
122
    clean_command = """
123
        invoke --list
124
    """
125
    run_invoke(context, clean_command)
126
 
127
 
128
@task
129
def clean(context):
130
    # https://unix.stackexchange.com/a/689930/77389
131
    clean_command = """
132
        rm -rf output/ docs/sphinx/build/
133
    """
134
    run_invoke(context, clean_command)
135
 
136
 
137
@task(aliases=["s"])
138
def server(context, input_path=".", config=None):
139
    assert os.path.isdir(input_path), input_path
140
    if config is not None:
141
        assert os.path.isfile(config), config
142
    config_argument = f"--config {config}" if config is not None else ""
143
    run_invoke_with_tox(
144
        context,
145
        ToxEnvironment.DEVELOPMENT,
146
        f"""
147
            python -m strictdoc.cli.main
148
                --debug
149
                server {input_path} {config_argument}
150
                    --host 127.0.0.1
151
                    --reload
152
        """,
153
    )
154
 
155
 
156
@task(aliases=["d"])
157
def docs(context):
158
    run_invoke_with_tox(
159
        context,
160
        ToxEnvironment.DOCUMENTATION,
161
        """
162
            python3 -m strictdoc.cli.main
163
                export .
164
                    --formats=html
165
                    --output-dir output/strictdoc_website
166
                    --project-title "StrictDoc"
167
        """,
168
    )
169
 
170
    run_invoke_with_tox(
171
        context,
172
        ToxEnvironment.DOCUMENTATION,
173
        """
174
            python3 -m strictdoc.cli.main
175
                export ./
176
                    --formats=rst
177
                    --output-dir output/sphinx
178
                    --project-title "StrictDoc"
179
        """,
180
    )
181
 
182
    run_invoke_with_tox(
183
        context,
184
        ToxEnvironment.DOCUMENTATION,
185
        """
186
            cp -r output/sphinx/rst/docs/* docs/sphinx/source/ &&
187
            cp -r output/sphinx/rst/docs_extra/* docs/sphinx/source/ &&
188
            mkdir -p docs/sphinx/source/_assets/ &&
189
            cp -v docs/_assets/* docs/sphinx/source/_assets/
190
        """,
191
    )
192
 
193
    run_invoke_with_tox(
194
        context,
195
        ToxEnvironment.DOCUMENTATION,
196
        """
197
            make --directory docs/sphinx html latexpdf SPHINXOPTS="-W --keep-going"
198
        """,
199
    )
200
 
201
    run_invoke(
202
        context,
203
        (
204
            """
205
                open docs/sphinx/build/latex/strictdoc.pdf
206
            """
207
        ),
208
    )
209
 
210
 
211
@task(aliases=["tus"])
212
def test_unit_server(context, focus=None):
213
    focus_argument = f"-k {focus}" if focus is not None else ""
214
 
215
    Path(TEST_REPORTS_DIR).mkdir(parents=True, exist_ok=True)
216
 
217
    cwd = os.getcwd()
218
 
219
    path_to_coverage_file = f"{cwd}/build/coverage/unit_server/.coverage"
220
 
221
    run_invoke_with_tox(
222
        context,
223
        ToxEnvironment.CHECK,
224
        f"""
225
            coverage run
226
            --rcfile=.coveragerc.unit_server
227
            --data-file={path_to_coverage_file}
228
            -m pytest
229
            tests/unit_server/
230
                {focus_argument}
231
                --junit-xml={TEST_REPORTS_DIR}/tests_unit_server.pytest.junit.xml
232
                -o junit_suite_name="StrictDoc Web Server Unit Tests"
233
                -o cache_dir=build/pytest_unit_server
234
        """,
235
    )
236
 
237
 
238
@task(test_unit_server, aliases=["tusc"])
239
def test_unit_server_report(context):
240
    cwd = os.getcwd()
241
 
242
    path_to_coverage_file = f"{cwd}/build/coverage/unit_server/.coverage"
243
 
244
    run_invoke_with_tox(
245
        context,
246
        ToxEnvironment.CHECK,
247
        f"""
248
            coverage html
249
                --rcfile=.coveragerc.unit_server
250
                --data-file={path_to_coverage_file}
251
        """,
252
    )
253
 
254
 
255
@task(aliases=["te"])
256
def test_end2end(
257
    context,
258
    *,
259
    focus=None,
260
    exit_first=False,
261
    parallelize=False,
262
    long_timeouts=False,
263
    headless=False,
264
    headed=False,
265
    shard=None,
266
    test_path=None,
267
    coverage: bool = False,
268
):
269
    """
270
    @relation(SDOC-SRS-46, scope=function)
271
    """
272
 
273
    environment = {}
274
 
275
    coverage_command_or_none = ""
276
    coverage_argument_or_none = ""
277
 
278
    if coverage:
279
        cwd = os.getcwd()
280
        coverage_file_dir = f"{cwd}/build/coverage/end2end/"
281
        coverage_file_dir2 = f"{cwd}/build/coverage/end2end_strictdoc/"
282
        coverage_file = os.path.join(coverage_file_dir, ".coverage")
283
        coverage_rc = os.path.join(cwd, ".coveragerc.end2end")
284
        shutil.rmtree(coverage_file_dir, ignore_errors=True)
285
        shutil.rmtree(coverage_file_dir2, ignore_errors=True)
286
        coverage_command_or_none = f"""
287
            coverage run
288
                --rcfile={coverage_rc}
289
                --data-file={coverage_file}
290
                -m
291
        """
292
        coverage_argument_or_none = "--strictdoc-coverage"
293
 
294
    long_timeouts_argument = (
295
        "--strictdoc-long-timeouts" if long_timeouts else ""
296
    )
297
 
298
    parallelize_argument = ""
299
    if parallelize:
300
        print(  # noqa: T201
301
            "warning: "
302
            "Running parallelized end-2-end tests is supported "
303
            "but is not stable."
304
        )
305
        parallelize_argument = "--numprocesses=2 --strictdoc-parallelize"
306
 
307
    assert shard is None or re.match(r"[1-9][0-9]*/[1-9][0-9]*", shard), (
308
        f"--shard argument has an incorrect format: {shard}."
309
    )
310
    shard_argument = f"--strictdoc-shard={shard}" if shard else ""
311
 
312
    focus_argument = f"-k {focus}" if focus is not None else ""
313
    exit_first_argument = "--exitfirst" if exit_first else ""
314
    headless_argument = "--headless2" if headless and not headed else "--gui"
315
    test_command = f"""
316
            {coverage_command_or_none}
317
            pytest
318
            --failed-first
319
            --capture=no
320
            --reuse-session
321
            {parallelize_argument}
322
            {shard_argument}
323
            {coverage_argument_or_none}
324
            {focus_argument}
325
            {exit_first_argument}
326
            {long_timeouts_argument}
327
            {headless_argument}
328
            --junit-xml={TEST_REPORTS_DIR}/tests_end2end.pytest.junit.xml
329
            -o junit_suite_name="StrictDoc End-to-End Tests"
330
            -o cache_dir=build/pytest_end2end
331
            tests/end2end
332
    """
333
    if test_path:
334
        test_command = test_command.rstrip() + f"/{test_path}"
335
 
336
    Path(TEST_REPORTS_DIR).mkdir(parents=True, exist_ok=True)
337
 
338
    # On Windows, GitHub Actions fails with:
339
    # response = {'status': 500, 'value':
340
    # '{"value":{"error":"unknown error",
341
    # "message":"unknown error: cannot find Chrome binary",  # noqa: ERA001
342
    # This very likely has to do with PATH isolation that Tox does.
343
    # FIXME: If you are a Windows expert, please fix this to run on Tox.
344
    if os.name == "nt":
345
        run_invoke(context, test_command)
346
        return
347
 
348
    run_invoke_with_tox(
349
        context,
350
        ToxEnvironment.CHECK,
351
        test_command,
352
        environment=environment,
353
    )
354
 
355
 
356
@task(aliases=["tu"])
357
def test_unit(context, coverage=False, focus=None, path=None, output=False):
358
    """
359
    @relation(SDOC-SRS-44, scope=function)
360
    """
361
 
362
    Path(TEST_REPORTS_DIR).mkdir(parents=True, exist_ok=True)
363
 
364
    focus_argument = f"-k {focus}" if focus is not None else ""
365
    output_argument = "--capture=no" if output else ""
366
 
367
    cwd = os.getcwd()
368
 
369
    if path is None:
370
        path = "tests/unit"
371
    else:
372
        assert "tests/unit" in path, path
373
 
374
    path_to_coverage_file = f"{cwd}/build/coverage/unit/.coverage"
375
 
376
    pytest_command = (
377
        """
378
        coverage run
379
            --rcfile=.coveragerc.unit
380
            --data-file={path_to_coverage_file}
381
            -m pytest
382
    """
383
        if coverage
384
        else "pytest"
385
    )
386
 
387
    run_invoke_with_tox(
388
        context,
389
        ToxEnvironment.CHECK,
390
        f"""
391
            {pytest_command}
392
            {focus_argument}
393
            {output_argument}
394
            --junit-xml={TEST_REPORTS_DIR}/tests_unit.pytest.junit.xml
395
            -o cache_dir=build/pytest_unit_with_coverage
396
            -o junit_suite_name="StrictDoc Unit Tests"
397
            -p no:seleniumbase
398
            {path}
399
        """,
400
    )
401
    if coverage and not focus and path == "tests/unit":
402
        run_invoke_with_tox(
403
            context,
404
            ToxEnvironment.CHECK,
405
            f"""
406
                coverage report
407
                    --sort=cover
408
                    --rcfile=.coveragerc.unit
409
                    --data-file={path_to_coverage_file}
410
            """,
411
        )
412
 
413
 
414
@task(test_unit, aliases=["tuc"])
415
def test_unit_report(context):
416
    cwd = os.getcwd()
417
 
418
    path_to_coverage_file = f"{cwd}/build/coverage/unit/.coverage"
419
 
420
    run_invoke_with_tox(
421
        context,
422
        ToxEnvironment.CHECK,
423
        f"""
424
            coverage html
425
                --rcfile=.coveragerc.unit
426
                --data-file={path_to_coverage_file}
427
        """,
428
    )
429
 
430
 
431
@task(aliases=["ti"])
432
def test_integration(
433
    context,
434
    focus=None,
435
    debug=False,
436
    no_parallelization=False,
437
    fail_first=False,
438
    coverage=False,
439
    strictdoc=None,
440
    html2pdf=False,
441
    shard=None,
442
    environment=ToxEnvironment.CHECK,
443
):
444
    """
445
    @relation(SDOC-SRS-45, scope=function)
446
    """
447
 
448
    cwd = os.getcwd()
449
 
450
    if strictdoc is None:
451
        strictdoc_exec = "python3 -m strictdoc.cli.main"
452
    else:
453
        strictdoc_exec = strictdoc
454
 
455
    coverage_path_argument = ""
456
    if coverage:
457
        path_to_coverage_rc = f"{cwd}/.coveragerc.integration"
458
        strictdoc_exec = (
459
            f"coverage run --rcfile={path_to_coverage_rc} -m strictdoc.cli.main"
460
        )
461
        if html2pdf:
462
            path_to_coverage_dir = f"{cwd}/build/coverage/integration_html2pdf/"
463
        else:
464
            path_to_coverage_dir = f"{cwd}/build/coverage/integration/"
465
        path_to_coverage = os.path.join(path_to_coverage_dir, ".coverage")
466
        shutil.rmtree(path_to_coverage_dir, ignore_errors=True)
467
        coverage_path_argument = (
468
            f'--param COVERAGE_FILE="{path_to_coverage}" '
469
            f'--param COVERAGE_PROCESS_START="{path_to_coverage_rc}"'
470
        )
471
 
472
    debug_opts = "-vv --show-all" if debug else ""
473
    focus_or_none = f"--filter {focus}" if focus else ""
474
    fail_first_argument = "--max-failures 1" if fail_first else ""
475
    junit_xml_report_argument = (
476
        "--xunit-xml-output build/test_reports/tests_integration_html2pdf.lit.junit.xml"
477
        if html2pdf
478
        else "--xunit-xml-output build/test_reports/tests_integration.lit.junit.xml"
479
    )
480
 
481
    # Allow partitioning of integration and html2pdf tests
482
    partition_opts = ""
483
    if shard is not None:
484
        match = re.match(r"([1-9][0-9]*)/([1-9][0-9]*)", shard)
485
        assert match, f"--shard argument has an incorrect format: {shard}."
486
        run_shard = int(match.group(1))
487
        num_shards = int(match.group(2))
488
        partition_opts = f"--num-shards={num_shards} --run-shard={run_shard}"
489
 
490
    # HTML2PDF tests are running Chrome Driver which does not seem to be
491
    # parallelizable, or at least not in the way StrictDoc uses it.
492
    # If HTML2PDF option is provided, do not parallelize and only run the
493
    # HTML2PDF-specific tests.
494
    # HTML2PDF tests can be safely partitioned.
495
    chromedriver_param = ""
496
    if not html2pdf:
497
        parallelize_opts = "" if not no_parallelization else "--threads 1"
498
        html2pdf_param = ""
499
        test_folder = f"{cwd}/tests/integration"
500
        test_output_dir = "build/tests_integration"
501
    else:
502
        parallelize_opts = "--threads 1"
503
        html2pdf_param = "--param TEST_HTML2PDF=1"
504
        chromedriver_path = os.environ.get("CHROMEWEBDRIVER")
505
        if chromedriver_path is not None:
506
            # NOTE: isfile() check does not work on GitHub Actions / Linux,
507
            #       the exists() check works.
508
            assert os.path.exists(chromedriver_path), chromedriver_path
509
            chromedriver_param = f"--param CHROMEDRIVER={os.path.join(chromedriver_path, 'chromedriver')}"
510
            if os.name == "nt":
511
                # On Windows, its chromdriver.exe
512
                chromedriver_param = chromedriver_param + ".exe"
513
        test_folder = f"{cwd}/tests/integration/features/html2pdf"
514
        test_output_dir = "build/tests_integration_html2pdf"
515
 
516
    # The command sometimes exits with 1 even if the files are deleted.
517
    # warn=True ensures that the execution continues.
518
    run_invoke(
519
        context,
520
        f"""
521
        rm -rf {test_output_dir}
522
        """,
523
        warn=True,
524
    )
525
 
526
    run_invoke(
527
        context,
528
        f"""
529
        rm -rf {STRICTDOC_TMP_DIR}
530
        """,
531
    )
532
 
533
    Path(STRICTDOC_TMP_DIR).mkdir(exist_ok=True)
534
    Path(TEST_REPORTS_DIR).mkdir(parents=True, exist_ok=True)
535
 
536
    itest_command = f"""
537
        lit
538
        --param STRICTDOC_EXEC="{strictdoc_exec}"
539
        --param STRICTDOC_TMP_DIR="{STRICTDOC_TMP_DIR}"
540
        --param TEST_OUTPUT_DIR="{test_output_dir}"
541
        --timeout 180
542
        --order smart
543
        {junit_xml_report_argument}
544
        {coverage_path_argument}
545
        {html2pdf_param}
546
        {chromedriver_param}
547
        -v
548
        {debug_opts}
549
        {focus_or_none}
550
        {fail_first_argument}
551
        {parallelize_opts}
552
        {partition_opts}
553
        {test_folder}
554
    """
555
 
556
    # It looks like LIT does not open the RUN: subprocesses in the same
557
    # environment from which it itself is run from. This issue has been known by
558
    # us for a couple of years by now. Not using Tox on Windows for the time
559
    # being.
560
    if os.name == "nt":
561
        run_invoke(context, itest_command)
562
        return
563
 
564
    run_invoke_with_tox(
565
        context,
566
        environment,
567
        itest_command,
568
        environment={"STRICTDOC_CACHE_DIR": "Output/_cache"},
569
    )
570
 
571
 
572
@task
573
def coverage_combine(context):
574
    run_invoke_with_tox(
575
        context,
576
        ToxEnvironment.CHECK,
577
        """
578
            coverage combine
579
                --data-file build/coverage/.coverage.combined
580
                --keep
581
                build/coverage/end2end_strictdoc/.coverage.*
582
                build/coverage/integration/.coverage.*
583
                build/coverage/integration_html2pdf/.coverage.*
584
                build/coverage/unit/.coverage
585
                build/coverage/unit_server/.coverage
586
        """,
587
    )
588
    run_invoke_with_tox(
589
        context,
590
        ToxEnvironment.CHECK,
591
        """
592
            coverage html
593
                --rcfile .coveragerc.combined
594
                --data-file build/coverage/.coverage.combined
595
        """,
596
    )
597
    run_invoke_with_tox(
598
        context,
599
        ToxEnvironment.CHECK,
600
        """
601
            coverage json
602
                --rcfile .coveragerc.combined
603
                --data-file build/coverage/.coverage.combined
604
                --pretty-print
605
                -o build/coverage/coverage.combined.json
606
        """,
607
    )
608
 
609
 
610
@task
611
def lint_ruff_format(context):
612
    """
613
    @relation(SDOC-SRS-42, scope=function)
614
    """
615
 
616
    result: invoke.runners.Result = run_invoke_with_tox(
617
        context,
618
        ToxEnvironment.CHECK,
619
        """
620
            ruff
621
                format
622
                --cache-dir build/ruff
623
                *.py
624
                developer/
625
                docs/
626
                strictdoc/
627
                tools/ecss
628
                tests/unit/
629
                tests/unit_server/
630
                tests/integration/*.py
631
                tests/end2end/
632
        """,
633
    )
634
    # Ruff always exits with 0, so we handle the output.
635
    if "reformatted" in result.stdout:
636
        print("invoke: ruff format found issues")  # noqa: T201
637
        result.exited = 1
638
        raise invoke.exceptions.UnexpectedExit(result)
639
 
640
 
641
@task(aliases=["lr"])
642
def lint_ruff(context):
643
    """
644
    @relation(SDOC-SRS-42, scope=function)
645
    """
646
 
647
    run_invoke_with_tox(
648
        context,
649
        ToxEnvironment.CHECK,
650
        """
651
            ruff check . --fix --exit-non-zero-on-fix --cache-dir build/ruff
652
        """,
653
    )
654
 
655
 
656
@task(aliases=["lm"])
657
def lint_mypy(context):
658
    """
659
    @relation(SDOC-SRS-41, SDOC-SRS-43, scope=function)
660
    """
661
 
662
    # These checks do not seem to be useful:
663
    # - import
664
    # --disallow-any-expr
665
    # --disallow-any-explicit
666
    # --disallow-any-unimported  # noqa: ERA001
667
    # --disallow-any-decorated
668
    # - type-abstract. It is ignored on purpose because of assert_cast()
669
    #   implementation. See https://stackoverflow.com/a/74073453/598057.
670
    run_invoke_with_tox(
671
        context,
672
        ToxEnvironment.CHECK,
673
        """
674
            mypy docs/
675
                 strictdoc/
676
                 tests/unit/strictdoc/backend/sdoc_source_code/test_marker_lexer.py
677
 
678
                --show-error-codes
679
                --disable-error-code=import
680
                --disable-error-code=type-abstract
681
                --cache-dir=build/mypy
682
                --extra-checks
683
 
684
                --strict
685
                --strict-optional
686
                --strict-equality
687
 
688
                --check-untyped-defs
689
                --disallow-any-generics
690
                --disallow-incomplete-defs
691
                --disallow-subclassing-any
692
                --disallow-untyped-calls
693
                --disallow-untyped-decorators
694
                --disallow-untyped-defs
695
                --no-implicit-optional
696
                --warn-no-return
697
                --warn-redundant-casts
698
                --warn-return-any
699
                --warn-unreachable
700
                --warn-unused-ignores
701
 
702
                --python-version=3.10
703
        """,
704
    )
705
 
706
 
707
@task
708
def lint_format_js(context):
709
    # NOTE: Could not find the '--' equivalent for -w80.
710
    result: invoke.runners.Result = run_invoke_with_tox(
711
        context,
712
        ToxEnvironment.CHECK,
713
        """
714
            js-beautify
715
                --indent-size=2
716
                --end-with-newline
717
                --replace
718
                -w100
719
                strictdoc/export/html/_static/static_html_search.js
720
                strictdoc/export/html/_static/stable_uri_forwarder.js
721
        """,
722
    )
723
    # Ruff always exits with 0, so we handle the output.
724
    if "reformatted" in result.stdout:
725
        print("invoke: ruff format found issues")  # noqa: T201
726
        result.exited = 1
727
        raise invoke.exceptions.UnexpectedExit(result)
728
 
729
 
730
@task(aliases=["lc"])
731
def lint_commit(context):  # noqa: ARG001
732
    try:
733
        validate_commits_locally_or_ci()
734
    except ValueError as e:
735
        raise invoke.exceptions.Exit(message=str(e), code=1) from None
736
 
737
 
738
@task(aliases=["lf"])
739
def lint_fixit(context, fix=False, auto=False, path="strictdoc/"):
740
    if fix:
741
        auto_argument = "--automatic" if auto else ""
742
        run_invoke_with_tox(
743
            context,
744
            ToxEnvironment.CHECK,
745
            f"""
746
                fixit fix {path} {auto_argument}
747
            """,
748
            pty=True,
749
        )
750
    else:
751
        run_invoke_with_tox(
752
            context,
753
            ToxEnvironment.CHECK,
754
            f"""
755
                fixit lint --diff {path}
756
            """,
757
        )
758
 
759
 
760
@task(aliases=["l"])
761
def lint(context):
762
    lint_commit(context)
763
    lint_ruff_format(context)
764
    lint_ruff(context)
765
    lint_mypy(context)
766
 
767
 
768
@task(aliases=["t"])
769
def test(context, shard=None):
770
    test_unit(context)
771
    test_unit_server(context)
772
    test_integration(context, shard=shard)
773
 
774
 
775
@task(aliases=["ta"])
776
def test_all(context, coverage=False, headless=False):
777
    test_unit(context, coverage=coverage)
778
    test_unit_server(context)
779
    test_integration(context, coverage=coverage)
780
    test_integration(context, coverage=coverage, html2pdf=True)
781
    test_end2end(context, coverage=coverage, headless=headless)
782
 
783
 
784
@task(aliases=["c"])
785
def check(context):
786
    lint(context)
787
    test(context)
788
 
789
 
790
# https://github.com/github-changelog-generator/github-changelog-generator
791
# gem install github_changelog_generator
792
@task
793
def changelog(context, github_token):
794
    # The alpha release tags are excluded from the changelog.
795
    command = f"""
796
        github_changelog_generator
797
        --token {github_token}
798
        --user strictdoc-project
799
        --exclude-tags-regex ".*a\\d+"
800
        --project strictdoc
801
        """
802
    run_invoke(context, command)
803
 
804
 
805
@task
806
def check_dead_links(context):
807
    run_invoke_with_tox(
808
        context,
809
        ToxEnvironment.CHECK,
810
        """
811
            python3 tools/link_health.py docs/strictdoc_01_user_guide.sdoc
812
        """,
813
    )
814
    run_invoke_with_tox(
815
        context,
816
        ToxEnvironment.CHECK,
817
        """
818
            python3 tools/link_health.py docs/strictdoc_02_feature_map.sdoc
819
        """,
820
    )
821
    run_invoke_with_tox(
822
        context,
823
        ToxEnvironment.CHECK,
824
        """
825
            python3 tools/link_health.py docs/strictdoc_03_faq.sdoc
826
        """,
827
    )
828
    run_invoke_with_tox(
829
        context,
830
        ToxEnvironment.CHECK,
831
        """
832
            python3 tools/link_health.py docs/strictdoc_04_release_notes.sdoc
833
        """,
834
    )
835
    run_invoke_with_tox(
836
        context,
837
        ToxEnvironment.CHECK,
838
        """
839
            python3 tools/link_health.py docs/strictdoc_05_troubleshooting.sdoc
840
        """,
841
    )
842
    run_invoke_with_tox(
843
        context,
844
        ToxEnvironment.CHECK,
845
        """
846
            python3 tools/link_health.py docs/strictdoc_10_contributing.sdoc
847
        """,
848
    )
849
    run_invoke_with_tox(
850
        context,
851
        ToxEnvironment.CHECK,
852
        """
853
            python3 tools/link_health.py docs/strictdoc_11_developer_guide.sdoc
854
        """,
855
    )
856
    run_invoke_with_tox(
857
        context,
858
        ToxEnvironment.CHECK,
859
        """
860
            python3 tools/link_health.py docs/strictdoc_24_development_plan.sdoc
861
        """,
862
    )
863
    run_invoke_with_tox(
864
        context,
865
        ToxEnvironment.CHECK,
866
        """
867
            python3 tools/link_health.py docs/strictdoc_20_l1_system_requirements.sdoc
868
        """,
869
    )
870
    run_invoke_with_tox(
871
        context,
872
        ToxEnvironment.CHECK,
873
        """
874
            python3 tools/link_health.py docs/strictdoc_21_l2_high_level_requirements.sdoc
875
        """,
876
    )
877
    run_invoke_with_tox(
878
        context,
879
        ToxEnvironment.CHECK,
880
        """
881
            python3 tools/link_health.py docs/strictdoc_25_design.sdoc
882
        """,
883
    )
884
    run_invoke_with_tox(
885
        context,
886
        ToxEnvironment.CHECK,
887
        """
888
            python3 tools/link_health.py CONTRIBUTING.md
889
        """,
890
    )
891
    run_invoke_with_tox(
892
        context,
893
        ToxEnvironment.CHECK,
894
        """
895
            python3 tools/link_health.py NOTICE
896
        """,
897
    )
898
    run_invoke_with_tox(
899
        context,
900
        ToxEnvironment.CHECK,
901
        """
902
            python3 tools/link_health.py README.md
903
        """,
904
    )
905
 
906
 
907
@task
908
def release_local(context):
909
    run_invoke(
910
        context,
911
        """
912
            rm -rfv dist/ build/
913
        """,
914
    )
915
    run_invoke(
916
        context,
917
        """
918
            pip uninstall strictdoc -y
919
        """,
920
    )
921
    run_invoke_with_tox(
922
        context,
923
        ToxEnvironment.RELEASE_LOCAL,
924
        """
925
            python -m build
926
        """,
927
    )
928
    run_invoke_with_tox(
929
        context,
930
        ToxEnvironment.RELEASE_LOCAL,
931
        """
932
            twine check dist/*
933
        """,
934
    )
935
    run_invoke_with_tox(
936
        context,
937
        ToxEnvironment.RELEASE_LOCAL,
938
        """
939
            pip install dist/*.tar.gz
940
        """,
941
    )
942
    test_integration(
943
        context, strictdoc="strictdoc", environment=ToxEnvironment.RELEASE_LOCAL
944
    )
945
 
946
 
947
@task
948
def release(context, test_pypi=False, username=None, password=None):
949
    """
950
    A release can be made to PyPI or test package index (TestPyPI):
951
    https://pypi.org/project/strictdoc/
952
    https://test.pypi.org/project/strictdoc/
953
    """
954
 
955
    env_user = os.environ.get("TWINE_USERNAME")
956
    env_pass = os.environ.get("TWINE_PASSWORD")
957
 
958
    assert not ((username or password) and (env_user or env_pass)), (
959
        username,
960
        password,
961
        env_user,
962
        env_pass,
963
    )
964
    assert (username and password) or (env_user and env_pass), (
965
        username,
966
        password,
967
        env_user,
968
        env_pass,
969
    )
970
    if env_user:
971
        assert env_user == "__token__"
972
 
973
    repository_argument_or_none = ""
974
    if username is not None and password is not None:
975
        repository_argument_or_none = (
976
            ""
977
            if username
978
            else (
979
                "--repository strictdoc_test"
980
                if test_pypi
981
                else "--repository strictdoc_release"
982
            )
983
        )
984
    user_password = f"-u{username} -p{password}" if username is not None else ""
985
 
986
    run_invoke(
987
        context,
988
        """
989
            rm -rfv dist/
990
        """,
991
    )
992
    run_invoke_with_tox(
993
        context,
994
        ToxEnvironment.RELEASE,
995
        """
996
            python3 -m build
997
        """,
998
    )
999
    run_invoke_with_tox(
1000
        context,
1001
        ToxEnvironment.RELEASE,
1002
        """
1003
            twine check dist/*
1004
        """,
1005
    )
1006
    # The token is in a core developer's .pypirc file.
1007
    # https://test.pypi.org/manage/account/token/
1008
    # https://packaging.python.org/en/latest/specifications/pypirc/#pypirc
1009
    run_invoke_with_tox(
1010
        context,
1011
        ToxEnvironment.RELEASE,
1012
        f"""
1013
            twine upload dist/strictdoc-*.tar.gz dist/strictdoc-*.whl
1014
                {repository_argument_or_none}
1015
                {user_password}
1016
        """,
1017
    )
1018
 
1019
 
1020
@task
1021
def release_pyinstaller(context):
1022
    path_to_pyi_dist = "/tmp/strictdoc"
1023
    html_template_data_options = get_pyinstaller_html_template_data_options()
1024
    html_static_data_options = get_pyinstaller_html_static_data_options()
1025
 
1026
    # The --hidden-import strictdoc.server.app flag is needed because without
1027
    # it, the following is produced:
1028
    # ERROR: Error loading ASGI app. Could not import
1029
    # module "strictdoc.server.app".
1030
    # Solution found here: https://stackoverflow.com/a/71340437/598057
1031
    # This behavior is not surprising because that's how the uvicorn loads the
1032
    # application separately from the parent process.
1033
    #
1034
    # Compatibility modules can be imported by user-provided statistics
1035
    # generators at runtime. PyInstaller cannot discover these imports
1036
    # statically because the generators live outside of StrictDoc's package.
1037
    command = f"""
1038
        pyinstaller
1039
            --clean
1040
            --name strictdoc
1041
            --noconfirm
1042
            --additional-hooks-dir developer/pyinstaller_hooks
1043
            --distpath {path_to_pyi_dist}
1044
            --hidden-import strictdoc.backend.rst.strictdoc_lexer
1045
            --hidden-import strictdoc.core.statistics.metric
1046
            --hidden-import strictdoc.export.html.generators.project_statistics
1047
            --hidden-import strictdoc.export.html.generators.view_objects.project_statistics_view_object
1048
            --hidden-import strictdoc.export.html.generators.view_objects.project_tree_stats
1049
            --hidden-import strictdoc.server.app
1050
            {html_template_data_options}
1051
            {html_static_data_options}
1052
            --add-data strictdoc/backend/rst/templates:templates/rst
1053
            --add-data strictdoc/export/html/_static_extra:_static_extra
1054
            strictdoc/cli/main.py
1055
    """
1056
 
1057
    run_invoke_with_tox(
1058
        context,
1059
        ToxEnvironment.PYINSTALLER,
1060
        """
1061
    pyinstaller --version
1062
    """,
1063
    )
1064
 
1065
    run_invoke_with_tox(context, ToxEnvironment.PYINSTALLER, command)
1066
 
1067
 
1068
@task
1069
def watch(context, sdocs_path="."):
1070
    strictdoc_command = f"""
1071
        python -m strictdoc.cli.main
1072
            export
1073
            {sdocs_path}
1074
            --output-dir output/
1075
    """
1076
 
1077
    run_invoke_with_tox(
1078
        context,
1079
        ToxEnvironment.DEVELOPMENT,
1080
        f"""
1081
            {strictdoc_command}
1082
        """,
1083
    )
1084
 
1085
    paths_to_watch = "."
1086
    run_invoke_with_tox(
1087
        context,
1088
        ToxEnvironment.DEVELOPMENT,
1089
        f"""
1090
            watchmedo shell-command
1091
            --patterns="*.py;*.sdoc;*.jinja;*.html;*.css;*.js"
1092
            --recursive
1093
            --ignore-pattern='output/;tests/integration'
1094
            --command='{strictdoc_command}'
1095
            --drop
1096
            {paths_to_watch}
1097
        """,
1098
    )
1099
 
1100
 
1101
@task
1102
def run(context, command):
1103
    run_invoke_with_tox(
1104
        context,
1105
        ToxEnvironment.DEVELOPMENT,
1106
        f"""
1107
            {command}
1108
        """,
1109
    )
1110
 
1111
 
1112
@task
1113
def nuitka(context):
1114
    html_template_data_options = get_nuitka_html_template_data_options()
1115
    html_static_data_options = get_nuitka_html_static_data_options()
1116
 
1117
    run_invoke(
1118
        context,
1119
        f"""
1120
        PYTHONPATH="{os.getcwd()}"
1121
        python -m nuitka
1122
            --static-libpython=no
1123
            --standalone
1124
            --include-module=textx
1125
            --include-module=strictdoc.server.app
1126
            --include-module=docutils
1127
            --include-module=docutils.readers.standalone
1128
            --include-module=docutils.parsers.rst
1129
            {html_template_data_options}
1130
            {html_static_data_options}
1131
            --include-data-dir=strictdoc/backend/rst/templates=templates/rst
1132
            --include-data-dir=strictdoc/export/html/_static_extra/mathjax=_static_extra/mathjax
1133
            --include-package-data=docutils
1134
            strictdoc/cli/main.py
1135
        """,
1136
    )
1137
 
1138
 
1139
# https://github.com/jrfonseca/gprof2dot
1140
# pip install gprof2dot
1141
@task()
1142
def performance(context):
1143
    command = """
1144
        python -m cProfile -o output/profile.prof
1145
            -m strictdoc.cli.main export . --no-parallelization &&
1146
        gprof2dot -f pstats output/profile.prof | dot -Tpng -o output/output.png
1147
    """
1148
    run_invoke(context, command)
1149
 
1150
 
1151
@task(performance)
1152
def performance_snakeviz(context):
1153
    command = """
1154
        snakeviz output/profile.prof
1155
    """
1156
    run_invoke(context, command)
1157
 
1158
 
1159
@task(aliases=["bd"])
1160
def build_docker(
1161
    context,
1162
    image: str = "strictdoc:latest",
1163
    no_cache: bool = False,
1164
    source="pypi",
1165
):
1166
    no_cache_argument = "--no-cache" if no_cache else ""
1167
    run_invoke(
1168
        context,
1169
        f"""
1170
        docker build .
1171
            --build-arg STRICTDOC_SOURCE={source}
1172
            -t {image}
1173
            {no_cache_argument}
1174
        """,
1175
    )
1176
 
1177
 
1178
@task(aliases=["rd"])
1179
def run_docker(
1180
    context, image: str = "strictdoc:latest", command: Optional[str] = None
1181
):
1182
    command_argument = (
1183
        f'/bin/bash -c "{command}"' if command is not None else ""
1184
    )
1185
 
1186
    run_invoke(
1187
        context,
1188
        f"""
1189
        docker run
1190
            --name strictdoc
1191
            --rm
1192
            -it
1193
            -e HOST_UID=$(id -u) -e HOST_GID=$(id -g)
1194
            -v "$(pwd):/data"
1195
            {image}
1196
            {command_argument}
1197
        """,
1198
        pty=True,
1199
    )
1200
 
1201
 
1202
@task(aliases=["td"])
1203
def test_docker(context, image: str = "strictdoc:latest"):
1204
    run_invoke(
1205
        context,
1206
        """
1207
        rm -rf output/ && mkdir -p output/ && chmod 777 output/
1208
        """,
1209
    )
1210
    run_docker(
1211
        context,
1212
        image=image,
1213
        command="strictdoc export --formats=html,html2pdf .",
1214
    )
1215
 
1216
    def check_file_owner(filepath):
1217
        import pwd  # noqa: PLC0415
1218
 
1219
        file_owner = pwd.getpwuid(os.stat(filepath).st_uid).pw_name
1220
        current_user = os.environ.get("USER", "")
1221
        return file_owner == current_user
1222
 
1223
    assert check_file_owner(
1224
        "output/html2pdf/pdf/docs/strictdoc_01_user_guide.pdf"
1225
    )
1226
 
1227
 
1228
@task(aliases=["q"])
1229
def qualification(context):
1230
    test_all(context, coverage=True, headless=True)
1231
    coverage_combine(context)
1232
 
1233
 
1234
@task()
1235
def drawio(context):
1236
    if sys.platform == "darwin":
1237
        path_to_drawio = "/Applications/draw.io.app/Contents/MacOS/draw.io"
1238
    elif sys.platform.startswith("linux"):
1239
        path_to_drawio = "drawio"
1240
    else:
1241
        raise NotImplementedError(
1242
            "drawio task is supported only on macOS and Linux."
1243
        )
1244
 
1245
    artifacts = [
1246
        (
1247
            "developer/drawio/Architecture.drawio",
1248
            "docs/_assets/StrictDoc_Workspace-Architecture.drawio.png",
1249
        ),
1250
        (
1251
            "developer/drawio/Backlog.drawio",
1252
            "docs/_assets/StrictDoc_Workspace-Backlog.drawio.png",
1253
        ),
1254
        (
1255
            "developer/drawio/Roadmap.drawio",
1256
            "docs/_assets/StrictDoc_Workspace-Roadmap.drawio.png",
1257
        ),
1258
    ]
1259
 
1260
    for path_to_drawio_, path_to_png_ in artifacts:
1261
        print(f"Copying: {path_to_drawio_} -> {path_to_png_}")  # noqa: T201
1262
 
1263
        # Basic safety for now to avoid writing wrong files.
1264
        assert os.path.isfile(path_to_drawio_), path_to_drawio_
1265
        assert os.path.isfile(path_to_png_), path_to_png_
1266
 
1267
        run_invoke(
1268
            context,
1269
            f"""
1270
            {path_to_drawio}
1271
                --export
1272
                --format png
1273
                -o {path_to_png_}
1274
                --page-index 0
1275
                {path_to_drawio_}
1276
            """,
1277
            pty=True,
1278
        )