[Testing] Add the LLVM MIR RUN executor

1 files changed, 682 insertions, 0 deletions

diff --git a/backend/tools/MIRTool/LLVMRunTestExecutor.py b/backend/tools/MIRTool/LLVMRunTestExecutor.py
new file mode 100644
index 0000000..38ffd3c
--- /dev/null
+++ b/backend/tools/MIRTool/LLVMRunTestExecutor.py
@@ -0,0 +1,682 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+from __future__ import annotations
+
+import argparse
+import csv
+import dataclasses
+import json
+import os
+import re
+import shlex
+import subprocess
+import sys
+import tempfile
+from collections import Counter
+from concurrent.futures import ThreadPoolExecutor
+from pathlib import Path
+from typing import Iterable
+
+
+IN_SCOPE_CATEGORIES = {"RV64GC_EXACT", "RV64GC_SEMANTIC", "RV64GC_INPUT"}
+OUT_OF_SCOPE_CATEGORIES = {
+    "OUT_RV32": "rv32",
+    "OUT_RVV": "rvv",
+    "OUT_EXTENSION": "extension",
+}
+RV64GC_FEATURES = {"i", "m", "a", "f", "d", "c", "zicsr", "zifencei"}
+NON_ISA_FEATURES = {
+    "disable-postmisched-load-clustering",
+    "disable-postmisched-store-clustering",
+    "fusion-add-load",
+    "fusion-add-mem",
+    "fusion-addi-load",
+    "fusion-auipc-addi",
+    "fusion-auipc-load",
+    "fusion-ld-add",
+    "fusion-logic-imm-reg",
+    "fusion-logic-reg-imm",
+    "fusion-logic-reg-reg",
+    "fusion-lui-addi",
+    "fusion-lui-load",
+    "fusion-mul-add",
+    "fusion-shift-bit-extract",
+    "fusion-shifted-zextw",
+    "fusion-zexth",
+    "fusion-zextw",
+    "prefer-w-inst",
+    "reserve-x6",
+    "reserve-x7",
+    "save-restore",
+    "tagged-globals",
+    "unaligned-scalar-mem",
+    "use-postra-scheduler",
+}
+RUN_PATTERN = re.compile(r"^\s*#\s*RUN:\s?(.*)$")
+FILECHECK_PATTERN = re.compile(r"(?<![\w./-])FileCheck(?=\s|$)")
+LLC_PATTERN = re.compile(r"(?<![\w./-])%?llc(?=\s|$)")
+NOT_PATTERN = re.compile(r"(?<![\w./-])not(?=\s|$)")
+SUBSTITUTION_PATTERN = re.compile(r"%([A-Za-z][A-Za-z0-9_/.:+-]*)")
+
+
+@dataclasses.dataclass(frozen=True)
+class RunDirective:
+    line: int
+    command: str
+
+
+@dataclasses.dataclass(frozen=True)
+class TestDescription:
+    relative_path: str
+    path: Path
+    category: str
+    reason: str
+    runs: tuple[RunDirective, ...]
+
+
+@dataclasses.dataclass
+class GateResult:
+    status: str
+    diagnostic: str = ""
+
+
+@dataclasses.dataclass
+class RunResult:
+    line: int
+    command: str
+    scope: str
+    scope_reason: str
+    production: str = "not-applicable"
+    filecheck: str = "not-applicable"
+    diagnostic: str = ""
+
+
+@dataclasses.dataclass
+class FileResult:
+    path: str
+    category: str
+    reason: str
+    input_gate: GateResult
+    runs: list[RunResult]
+
+
+def parse_arguments() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(
+        description="Execute unmodified LLVM RISC-V MIR RUN directives with RCC"
+    )
+    parser.add_argument("--tool", type=Path, required=True)
+    parser.add_argument("--corpus", type=Path, required=True)
+    parser.add_argument("--manifest", type=Path, required=True)
+    parser.add_argument("--filecheck", type=Path, required=True)
+    parser.add_argument("--not-tool", type=Path)
+    parser.add_argument("--llvm-bin", type=Path)
+    parser.add_argument("--report", type=Path)
+    parser.add_argument("--jobs", type=int, default=max(1, min(8, os.cpu_count() or 1)))
+    parser.add_argument("--timeout", type=float, default=60.0)
+    parser.add_argument("--validate-only", action="store_true")
+    parser.add_argument("--strict", action="store_true")
+    parser.add_argument("--max-diagnostics", type=int, default=20)
+    return parser.parse_args()
+
+
+def read_manifest(path: Path) -> dict[str, tuple[str, str]]:
+    lines = [line for line in path.read_text(encoding="utf-8").splitlines() if not line.startswith("#")]
+    reader = csv.DictReader(lines, delimiter="\t")
+    if reader.fieldnames != ["path", "category", "reason"]:
+        raise ValueError("classification manifest has an invalid header")
+    result: dict[str, tuple[str, str]] = {}
+    for row in reader:
+        relative_path = row["path"]
+        category = row["category"]
+        reason = row["reason"]
+        if not relative_path or category not in IN_SCOPE_CATEGORIES | set(OUT_OF_SCOPE_CATEGORIES):
+            raise ValueError(f"classification manifest has an invalid row for '{relative_path}'")
+        if relative_path in result:
+            raise ValueError(f"classification manifest repeats '{relative_path}'")
+        result[relative_path] = (category, reason)
+    return result
+
+
+def read_run_directives(path: Path) -> tuple[RunDirective, ...]:
+    result: list[RunDirective] = []
+    command = ""
+    start_line = 0
+    for line_number, line in enumerate(path.read_text(encoding="utf-8").splitlines(), 1):
+        match = RUN_PATTERN.match(line)
+        if match is None:
+            if command:
+                raise ValueError(f"{path}:{start_line}: unterminated RUN continuation")
+            continue
+        fragment = match.group(1).rstrip()
+        if not command:
+            start_line = line_number
+        continued = fragment.endswith("\\")
+        if continued:
+            fragment = fragment[:-1].rstrip()
+        command = f"{command} {fragment}".strip()
+        if not continued:
+            if not command:
+                raise ValueError(f"{path}:{start_line}: empty RUN directive")
+            result.append(RunDirective(start_line, command))
+            command = ""
+    if command:
+        raise ValueError(f"{path}:{start_line}: unterminated RUN continuation")
+    return tuple(result)
+
+
+def load_tests(corpus: Path, manifest_path: Path) -> list[TestDescription]:
+    manifest = read_manifest(manifest_path)
+    corpus_paths = {
+        path.relative_to(corpus).as_posix(): path for path in corpus.rglob("*.mir") if path.is_file()
+    }
+    missing = sorted(set(manifest) - set(corpus_paths))
+    unclassified = sorted(set(corpus_paths) - set(manifest))
+    if missing or unclassified:
+        details = []
+        if missing:
+            details.append(f"missing files: {', '.join(missing[:5])}")
+        if unclassified:
+            details.append(f"unclassified files: {', '.join(unclassified[:5])}")
+        raise ValueError("classification and corpus differ: " + "; ".join(details))
+    tests = []
+    for relative_path in sorted(corpus_paths):
+        category, reason = manifest[relative_path]
+        runs = read_run_directives(corpus_paths[relative_path])
+        if not runs:
+            raise ValueError(f"{corpus_paths[relative_path]}: no RUN directives")
+        tests.append(
+            TestDescription(relative_path, corpus_paths[relative_path], category, reason, runs)
+        )
+    return tests
+
+
+def option_values(tokens: list[str], names: Iterable[str]) -> list[str]:
+    result: list[str] = []
+    name_set = set(names)
+    index = 0
+    while index < len(tokens):
+        token = tokens[index]
+        if token in name_set:
+            if index + 1 < len(tokens):
+                result.append(tokens[index + 1])
+                index += 2
+                continue
+        for name in name_set:
+            prefix = name + "="
+            if token.startswith(prefix):
+                result.append(token[len(prefix) :])
+                break
+        index += 1
+    return result
+
+
+def command_scope(test: TestDescription, command: str) -> tuple[str, str, bool]:
+    if test.category in OUT_OF_SCOPE_CATEGORIES:
+        scope = OUT_OF_SCOPE_CATEGORIES[test.category]
+        return scope, f"file classification {test.category}: {test.reason}", True
+    try:
+        tokens = shlex.split(command)
+    except ValueError as error:
+        return "invalid", f"cannot tokenize RUN directive: {error}", True
+
+    triples = option_values(tokens, ("-mtriple", "--mtriple"))
+    for triple in triples:
+        if triple.startswith("riscv32"):
+            return "rv32", f"RUN selects {triple}", True
+        if not triple.startswith("riscv64"):
+            return "target", f"RUN selects unsupported triple {triple}", True
+
+    abis = option_values(tokens, ("-target-abi", "--target-abi"))
+    for abi in abis:
+        if abi.startswith("ilp32"):
+            return "rv32", f"RUN selects {abi}", True
+        if not abi.startswith("lp64"):
+            return "target", f"RUN selects unsupported ABI {abi}", True
+
+    cpus = option_values(tokens, ("-mcpu", "--mcpu"))
+    for cpu in cpus:
+        if cpu not in {"generic-rv64", "generic-rv64gc"}:
+            return "cpu", f"RUN selects unsupported CPU {cpu}", True
+
+    attributes = option_values(tokens, ("-mattr", "--mattr"))
+    for attribute_list in attributes:
+        for attribute in attribute_list.split(","):
+            feature = attribute.strip().lstrip("+-")
+            if not feature:
+                continue
+            if feature == "v" or feature.startswith(("zv", "xsfv", "xsfmm")):
+                return "rvv", f"RUN selects vector feature {feature}", True
+            if feature not in RV64GC_FEATURES | NON_ISA_FEATURES:
+                return "extension", f"RUN selects non-RV64GC feature {feature}", True
+
+    has_target_context = bool(triples or abis or cpus or attributes or LLC_PATTERN.search(command))
+    return "selected", "RV64GC-compatible RUN", has_target_context
+
+
+def shell_quote(path: Path | str) -> str:
+    return shlex.quote(str(path))
+
+
+def validate_substitutions(command: str, path: Path, line: int) -> None:
+    supported = {"s", "t"}
+    for substitution in SUBSTITUTION_PATTERN.findall(command):
+        if substitution in supported or re.fullmatch(r"t[0-9]+", substitution):
+            continue
+        raise ValueError(f"{path}:{line}: unsupported lit substitution %{substitution}")
+
+
+def substitute_command(
+    command: str,
+    source: Path,
+    temporary_prefix: Path,
+    tool: Path,
+    filecheck: Path,
+    not_tool: Path | None,
+) -> str:
+    result = LLC_PATTERN.sub(shell_quote(tool), command)
+    result = FILECHECK_PATTERN.sub(shell_quote(filecheck), result)
+    if not_tool is not None:
+        result = NOT_PATTERN.sub(shell_quote(not_tool), result)
+    result = re.sub(
+        r"%t([0-9]+)", lambda match: shell_quote(f"{temporary_prefix}{match.group(1)}"), result
+    )
+    result = result.replace("%t", shell_quote(temporary_prefix))
+    return result.replace("%s", shell_quote(source))
+
+
+def filecheck_pipe(command: str) -> int | None:
+    match = FILECHECK_PATTERN.search(command)
+    if match is None:
+        return None
+    filecheck_position = match.start()
+    quote = ""
+    escaped = False
+    pipe_positions: list[int] = []
+    for index, character in enumerate(command[:filecheck_position]):
+        if escaped:
+            escaped = False
+            continue
+        if character == "\\" and quote != "'":
+            escaped = True
+            continue
+        if quote:
+            if character == quote:
+                quote = ""
+            continue
+        if character in {"'", '"'}:
+            quote = character
+            continue
+        if character == "|" and (index + 1 >= len(command) or command[index + 1] != "|"):
+            pipe_positions.append(index)
+    return pipe_positions[-1] if pipe_positions else None
+
+
+def run_shell(
+    command: str,
+    cwd: Path,
+    environment: dict[str, str],
+    timeout: float,
+    standard_input: bytes | None = None,
+) -> subprocess.CompletedProcess[bytes]:
+    return subprocess.run(
+        ["/bin/bash", "-o", "pipefail", "-c", command],
+        cwd=cwd,
+        env=environment,
+        input=standard_input,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+        timeout=timeout,
+        check=False,
+    )
+
+
+def diagnostic_for(process: subprocess.CompletedProcess[bytes], command: str) -> str:
+    output = process.stderr.decode("utf-8", errors="replace")
+    if not output:
+        output = process.stdout.decode("utf-8", errors="replace")
+    return f"command: {command}\n{output}"[:8000].rstrip()
+
+
+def execute_run(
+    directive: RunDirective,
+    scope: str,
+    scope_reason: str,
+    test: TestDescription,
+    temporary_prefix: Path,
+    arguments: argparse.Namespace,
+    environment: dict[str, str],
+) -> RunResult:
+    result = RunResult(directive.line, directive.command, scope, scope_reason)
+    if scope != "selected":
+        return result
+    pipe = filecheck_pipe(directive.command)
+    try:
+        if pipe is not None:
+            production_command = substitute_command(
+                directive.command[:pipe].strip(),
+                test.path,
+                temporary_prefix,
+                arguments.tool,
+                arguments.filecheck,
+                arguments.not_tool,
+            )
+            filecheck_command = substitute_command(
+                directive.command[pipe + 1 :].strip(),
+                test.path,
+                temporary_prefix,
+                arguments.tool,
+                arguments.filecheck,
+                arguments.not_tool,
+            )
+            production = run_shell(
+                production_command, test.path.parent, environment, arguments.timeout
+            )
+            if production.returncode != 0:
+                result.production = "failed"
+                result.filecheck = "skipped"
+                result.diagnostic = diagnostic_for(production, production_command)
+                return result
+            result.production = "passed"
+            check = run_shell(
+                filecheck_command,
+                test.path.parent,
+                environment,
+                arguments.timeout,
+                production.stdout,
+            )
+            if check.returncode == 0:
+                result.filecheck = "passed"
+            else:
+                result.filecheck = "failed"
+                result.diagnostic = diagnostic_for(check, filecheck_command)
+            return result
+
+        command = substitute_command(
+            directive.command,
+            test.path,
+            temporary_prefix,
+            arguments.tool,
+            arguments.filecheck,
+            arguments.not_tool,
+        )
+        if FILECHECK_PATTERN.search(directive.command):
+            check = run_shell(command, test.path.parent, environment, arguments.timeout)
+            result.filecheck = "passed" if check.returncode == 0 else "failed"
+            if check.returncode != 0:
+                result.diagnostic = diagnostic_for(check, command)
+            return result
+
+        production = run_shell(command, test.path.parent, environment, arguments.timeout)
+        result.production = "passed" if production.returncode == 0 else "failed"
+        if production.returncode != 0:
+            result.diagnostic = diagnostic_for(production, command)
+    except subprocess.TimeoutExpired:
+        result.production = "failed"
+        result.diagnostic = f"command timed out after {arguments.timeout:g} seconds: {command}"
+    except OSError as error:
+        result.production = "failed"
+        result.diagnostic = f"cannot execute command: {error}: {command}"
+    return result
+
+
+def execute_input_gate(
+    test: TestDescription, arguments: argparse.Namespace
+) -> GateResult:
+    if test.category not in IN_SCOPE_CATEGORIES:
+        return GateResult("not-applicable")
+    command = [
+        str(arguments.tool),
+        "--verify",
+        "-mtriple=riscv64",
+        "-mattr=+m,+a,+f,+d,+c",
+        "-verify-machineinstrs",
+        str(test.path),
+    ]
+    try:
+        process = subprocess.run(
+            command,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            timeout=arguments.timeout,
+            check=False,
+        )
+    except subprocess.TimeoutExpired:
+        return GateResult("failed", f"input gate timed out after {arguments.timeout:g} seconds")
+    except OSError as error:
+        return GateResult("failed", f"cannot execute input gate: {error}")
+    if process.returncode == 0:
+        return GateResult("passed")
+    return GateResult("failed", diagnostic_for(process, shlex.join(command)))
+
+
+def execute_file(
+    test: TestDescription,
+    arguments: argparse.Namespace,
+    temporary_root: Path,
+    environment: dict[str, str],
+) -> FileResult:
+    gate = GateResult("not-run") if arguments.validate_only else execute_input_gate(test, arguments)
+    results: list[RunResult] = []
+    inherited_scope: tuple[str, str] | None = None
+    temporary_directory = temporary_root / re.sub(r"[^A-Za-z0-9_.-]", "_", test.relative_path)
+    temporary_directory.mkdir(parents=True, exist_ok=True)
+    temporary_prefix = temporary_directory / "tmp"
+    for directive in test.runs:
+        validate_substitutions(directive.command, test.path, directive.line)
+        scope, reason, has_target_context = command_scope(test, directive.command)
+        if not has_target_context and inherited_scope is not None:
+            scope, reason = inherited_scope
+        elif has_target_context:
+            inherited_scope = (scope, reason)
+        if arguments.validate_only:
+            results.append(RunResult(directive.line, directive.command, scope, reason))
+        else:
+            results.append(
+                execute_run(
+                    directive,
+                    scope,
+                    reason,
+                    test,
+                    temporary_prefix,
+                    arguments,
+                    environment,
+                )
+            )
+    return FileResult(test.relative_path, test.category, test.reason, gate, results)
+
+
+def status_counts(values: Iterable[str]) -> dict[str, int]:
+    return dict(sorted(Counter(values).items()))
+
+
+def exact_file_status(result: FileResult) -> str:
+    if result.input_gate.status == "not-run":
+        return "not-run"
+    if result.input_gate.status == "failed":
+        return "failed"
+    selected = [run for run in result.runs if run.scope == "selected"]
+    if not selected:
+        return "no-compatible-run"
+    for run in selected:
+        if run.production == "failed" or run.filecheck in {"failed", "skipped"}:
+            return "failed"
+        if run.production == "not-applicable" and run.filecheck != "passed":
+            return "failed"
+    return "passed"
+
+
+def semantic_file_status(result: FileResult) -> str:
+    if result.input_gate.status == "not-run":
+        return "not-run"
+    if result.input_gate.status == "failed":
+        return "failed"
+    selected = [run for run in result.runs if run.scope == "selected"]
+    if not selected:
+        return "no-compatible-run"
+    return "failed" if any(run.production == "failed" for run in selected) else "passed"
+
+
+def build_summary(results: list[FileResult]) -> dict[str, object]:
+    runs = [run for result in results for run in result.runs]
+    exact = [result for result in results if result.category == "RV64GC_EXACT"]
+    semantic = [result for result in results if result.category == "RV64GC_SEMANTIC"]
+    input_only = [result for result in results if result.category == "RV64GC_INPUT"]
+    return {
+        "files": len(results),
+        "runs": len(runs),
+        "categories": status_counts(result.category for result in results),
+        "input_gate": status_counts(result.input_gate.status for result in results),
+        "scope": status_counts(run.scope for run in runs),
+        "production": status_counts(run.production for run in runs),
+        "filecheck": status_counts(run.filecheck for run in runs),
+        "exact_files": status_counts(exact_file_status(result) for result in exact),
+        "semantic_files": status_counts(semantic_file_status(result) for result in semantic),
+        "input_files": status_counts(result.input_gate.status for result in input_only),
+    }
+
+
+def count(summary: dict[str, object], section: str, status: str) -> int:
+    values = summary[section]
+    assert isinstance(values, dict)
+    value = values.get(status, 0)
+    assert isinstance(value, int)
+    return value
+
+
+def print_summary(summary: dict[str, object]) -> None:
+    selected = count(summary, "scope", "selected")
+    out_of_scope = int(summary["runs"]) - selected
+    print(
+        f"LLVM RISC-V MIR corpus: files={summary['files']} runs={summary['runs']} "
+        f"selected={selected} out-of-scope={out_of_scope}"
+    )
+    print(
+        "input gate: "
+        f"passed={count(summary, 'input_gate', 'passed')} "
+        f"failed={count(summary, 'input_gate', 'failed')} "
+        f"not-run={count(summary, 'input_gate', 'not-run')}"
+    )
+    print(
+        "production pass: "
+        f"passed={count(summary, 'production', 'passed')} "
+        f"failed={count(summary, 'production', 'failed')}"
+    )
+    print(
+        "FileCheck: "
+        f"passed={count(summary, 'filecheck', 'passed')} "
+        f"failed={count(summary, 'filecheck', 'failed')} "
+        f"skipped={count(summary, 'filecheck', 'skipped')}"
+    )
+    print(
+        "RV64GC_EXACT files: "
+        f"passed={count(summary, 'exact_files', 'passed')} "
+        f"failed={count(summary, 'exact_files', 'failed')} "
+        f"no-compatible-run={count(summary, 'exact_files', 'no-compatible-run')} "
+        f"not-run={count(summary, 'exact_files', 'not-run')}"
+    )
+    print("out-of-scope RUNs: " + json.dumps(summary["scope"], sort_keys=True))
+
+
+def print_diagnostics(results: list[FileResult], maximum: int) -> None:
+    emitted = 0
+    for result in results:
+        if result.input_gate.status == "failed" and emitted < maximum:
+            print(f"FAIL input {result.path}:\n{result.input_gate.diagnostic}", file=sys.stderr)
+            emitted += 1
+        for run in result.runs:
+            if not run.diagnostic or emitted >= maximum:
+                continue
+            print(f"FAIL {result.path}:{run.line}:\n{run.diagnostic}", file=sys.stderr)
+            emitted += 1
+    total = sum(
+        (1 if result.input_gate.status == "failed" else 0)
+        + sum(1 for run in result.runs if run.diagnostic)
+        for result in results
+    )
+    if total > emitted:
+        print(f"{total - emitted} additional failures omitted", file=sys.stderr)
+
+
+def strict_failure(summary: dict[str, object]) -> bool:
+    return (
+        count(summary, "input_gate", "failed") != 0
+        or count(summary, "exact_files", "failed") != 0
+        or count(summary, "exact_files", "no-compatible-run") != 0
+        or count(summary, "semantic_files", "failed") != 0
+        or count(summary, "semantic_files", "no-compatible-run") != 0
+    )
+
+
+def validate_paths(arguments: argparse.Namespace) -> None:
+    for description, path in (
+        ("RCC MIR tool", arguments.tool),
+        ("MIR corpus", arguments.corpus),
+        ("classification manifest", arguments.manifest),
+        ("FileCheck", arguments.filecheck),
+    ):
+        if not path.exists():
+            raise ValueError(f"{description} does not exist: {path}")
+    if arguments.not_tool is not None and not arguments.not_tool.exists():
+        raise ValueError(f"not tool does not exist: {arguments.not_tool}")
+    if arguments.jobs <= 0:
+        raise ValueError("--jobs must be positive")
+    if arguments.timeout <= 0:
+        raise ValueError("--timeout must be positive")
+
+
+def main() -> int:
+    arguments = parse_arguments()
+    try:
+        validate_paths(arguments)
+        arguments.tool = arguments.tool.resolve()
+        arguments.corpus = arguments.corpus.resolve()
+        arguments.manifest = arguments.manifest.resolve()
+        arguments.filecheck = arguments.filecheck.resolve()
+        if arguments.not_tool is not None:
+            arguments.not_tool = arguments.not_tool.resolve()
+        if arguments.llvm_bin is not None:
+            arguments.llvm_bin = arguments.llvm_bin.resolve()
+        tests = load_tests(arguments.corpus, arguments.manifest)
+        environment = os.environ.copy()
+        path_entries = []
+        if arguments.llvm_bin is not None:
+            path_entries.append(str(arguments.llvm_bin.resolve()))
+        path_entries.append(str(arguments.filecheck.resolve().parent))
+        if arguments.not_tool is not None:
+            path_entries.append(str(arguments.not_tool.resolve().parent))
+        environment["PATH"] = os.pathsep.join(path_entries + [environment.get("PATH", "")])
+        with tempfile.TemporaryDirectory(prefix="rcc-llvm-mir-") as temporary_directory:
+            temporary_root = Path(temporary_directory)
+            with ThreadPoolExecutor(max_workers=arguments.jobs) as executor:
+                results = list(
+                    executor.map(
+                        lambda test: execute_file(test, arguments, temporary_root, environment), tests
+                    )
+                )
+        results.sort(key=lambda result: result.path)
+        summary = build_summary(results)
+        print_summary(summary)
+        if not arguments.validate_only:
+            print_diagnostics(results, arguments.max_diagnostics)
+        if arguments.report is not None:
+            arguments.report.parent.mkdir(parents=True, exist_ok=True)
+            arguments.report.write_text(
+                json.dumps(
+                    {
+                        "summary": summary,
+                        "files": [dataclasses.asdict(result) for result in results],
+                    },
+                    indent=2,
+                    sort_keys=True,
+                )
+                + "\n",
+                encoding="utf-8",
+            )
+        return 1 if arguments.strict and strict_failure(summary) else 0
+    except (OSError, ValueError) as error:
+        print(f"LLVMRunTestExecutor.py: {error}", file=sys.stderr)
+        return 2
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())