diff options
Diffstat (limited to 'rcc/utils/git/code-format-helper.py')
| -rwxr-xr-x | rcc/utils/git/code-format-helper.py | 677 |
1 files changed, 677 insertions, 0 deletions
diff --git a/rcc/utils/git/code-format-helper.py b/rcc/utils/git/code-format-helper.py new file mode 100755 index 0000000..e7a82b8 --- /dev/null +++ b/rcc/utils/git/code-format-helper.py @@ -0,0 +1,677 @@ +#!/usr/bin/env python3 +# +# ====- code-format-helper, runs code formatters from the ci or in a hook --*- python -*--==# +# +# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +# See https://llvm.org/LICENSE.txt for license information. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# +# ==--------------------------------------------------------------------------------------==# + +import argparse +import os +import re +import shlex +import subprocess +import sys +from typing import List, Optional + +""" +This script is run by GitHub actions to ensure that the code in PR's conform to +the coding style of LLVM. It can also be installed as a pre-commit git hook to +check the coding style before submitting it. The canonical source of this script +is in the LLVM source tree under llvm/utils/git. + +For C/C++ code it uses clang-format, for Python code it uses darker (which +in turn invokes black), and for LLVM-libc CMake files it uses cmake_format.py. + +You can learn more about the LLVM coding style on llvm.org: +https://llvm.org/docs/CodingStandards.html + +You can install this script as a git hook by symlinking it to the .git/hooks +directory: + +ln -s $(pwd)/llvm/utils/git/code-format-helper.py .git/hooks/pre-commit + +You can control the exact path to clang-format, darker, or cmake_format.py with +the following environment variables: $CLANG_FORMAT_PATH, $DARKER_FORMAT_PATH, +and $CMAKE_FORMAT_PATH. +""" + + +class FormatArgs: + start_rev: str = None + end_rev: str = None + repo: str = None + changed_files: List[str] = [] + token: str = None + verbose: bool = True + issue_number: int = 0 + write_comment_to_file: bool = False + + def __init__(self, args: argparse.Namespace = None) -> None: + if not args is None: + self.start_rev = args.start_rev + self.end_rev = args.end_rev + self.repo = args.repo + self.token = args.token + self.changed_files = args.changed_files + self.issue_number = args.issue_number + self.write_comment_to_file = args.write_comment_to_file + + +class FormatHelper: + COMMENT_TAG = "<!--LLVM CODE FORMAT COMMENT: {fmt}-->" + name: str + friendly_name: str + comment: dict = None + + @property + def comment_tag(self) -> str: + return self.COMMENT_TAG.replace("fmt", self.name) + + @property + def instructions(self) -> str: + raise NotImplementedError() + + def has_tool(self) -> bool: + raise NotImplementedError() + + def format_run(self, changed_files: List[str], args: FormatArgs) -> Optional[str]: + raise NotImplementedError() + + def pr_comment_text_for_diff(self, diff: str) -> str: + return f""" +:warning: {self.friendly_name}, {self.name} found issues in your code. :warning: + +<details> +<summary> +You can test this locally with the following command: +</summary> + +``````````bash +{self.instructions} +`````````` + +:warning: +The reproduction instructions above might return results for more than one PR +in a stack if you are using a stacked PR workflow. You can limit the results by +changing `origin/main` to the base branch/commit you want to compare against. +:warning: + +</details> + +<details> +<summary> +View the diff from {self.name} here. +</summary> + +``````````diff +{diff} +`````````` + +</details> +""" + + # TODO: any type should be replaced with the correct github type, but it requires refactoring to + # not require the github module to be installed everywhere. + def find_comment(self, pr: any) -> any: + for comment in pr.as_issue().get_comments(): + if self.comment_tag in comment.body: + return comment + return None + + def update_pr(self, comment_text: str, args: FormatArgs, create_new: bool) -> None: + import github + from github import IssueComment, PullRequest + + repo = github.Github(auth=github.Auth.Token(args.token)).get_repo(args.repo) + pr = repo.get_issue(args.issue_number).as_pull_request() + + comment_text = self.comment_tag + "\n\n" + comment_text + + existing_comment = self.find_comment(pr) + + if args.write_comment_to_file: + if create_new or existing_comment: + self.comment = {"body": comment_text} + if existing_comment: + self.comment["id"] = existing_comment.id + return + + if existing_comment: + existing_comment.edit(comment_text) + elif create_new: + pr.as_issue().create_comment(comment_text) + + def run(self, changed_files: List[str], args: FormatArgs) -> bool: + changed_files = [arg for arg in changed_files if "third-party" not in arg] + diff = self.format_run(changed_files, args) + should_update_gh = args.token is not None and args.repo is not None + + if diff is None: + if should_update_gh: + comment_text = ( + ":white_check_mark: With the latest revision " + f"this PR passed the {self.friendly_name}." + ) + self.update_pr(comment_text, args, create_new=False) + return True + elif len(diff) > 0: + if should_update_gh: + comment_text = self.pr_comment_text_for_diff(diff) + self.update_pr(comment_text, args, create_new=True) + else: + print( + f"Warning: {self.friendly_name}, {self.name} detected " + "some issues with your code formatting..." + ) + return False + else: + # The formatter failed but didn't output a diff (e.g. some sort of + # infrastructure failure). + comment_text = ( + f":warning: The {self.friendly_name} failed without printing " + "a diff. Check the logs for stderr output. :warning:" + ) + if should_update_gh: + self.update_pr(comment_text, args, create_new=False) + return False + + +class ClangFormatHelper(FormatHelper): + name = "clang-format" + friendly_name = "C/C++ code formatter" + + def _construct_command(self, diff_expression: list[str] | None): + cf_cmd = [self.clang_fmt_path, "--diff"] + + if diff_expression: + cf_cmd.extend(diff_expression) + + # Gather the extension of all modified files and pass them explicitly to git-clang-format. + # This prevents git-clang-format from applying its own filtering rules on top of ours. + extensions = set() + for file in self._cpp_files: + _, ext = os.path.splitext(file) + extensions.add( + ext.strip(".") + ) # Exclude periods since git-clang-format takes extensions without them + cf_cmd.append("--extensions") + cf_cmd.append(",".join(extensions)) + + cf_cmd.append("--") + cf_cmd += self._cpp_files + return cf_cmd + + @property + def instructions(self) -> str: + return ( + " ".join(self._construct_command(["origin/main", "HEAD"])) + + " --diff_from_common_commit" + ) + + def should_include_extensionless_file(self, path: str) -> bool: + return path.startswith("libcxx/include") + + def filter_changed_files(self, changed_files: List[str]) -> List[str]: + filtered_files = [] + for path in changed_files: + _, ext = os.path.splitext(path) + if ext in ( + ".cpp", + ".c", + ".h", + ".hpp", + ".hxx", + ".cxx", + ".inc", + ".cppm", + ".cl", + ): + filtered_files.append(path) + elif ext == "" and self.should_include_extensionless_file(path): + filtered_files.append(path) + return filtered_files + + @property + def clang_fmt_path(self) -> str: + if "CLANG_FORMAT_PATH" in os.environ: + return os.environ["CLANG_FORMAT_PATH"] + return "git-clang-format" + + def has_tool(self) -> bool: + cmd = [self.clang_fmt_path, "-h"] + proc = None + try: + proc = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + except: + return False + return proc.returncode == 0 + + def format_run(self, changed_files: List[str], args: FormatArgs) -> Optional[str]: + self._cpp_files = self.filter_changed_files(changed_files) + if not self._cpp_files: + return None + + diff_expression = [] + if args.start_rev and args.end_rev: + diff_expression.append(args.start_rev) + diff_expression.append(args.end_rev) + + cf_cmd = self._construct_command(diff_expression) + + if args.verbose: + print(f"Running: {' '.join(cf_cmd)}") + proc = subprocess.run(cf_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + sys.stdout.write(proc.stderr.decode("utf-8")) + + if proc.returncode != 0: + # formatting needed, or the command otherwise failed + if args.verbose: + print(f"error: {self.name} exited with code {proc.returncode}") + # Print the diff in the log so that it is viewable there + print(proc.stdout.decode("utf-8")) + return proc.stdout.decode("utf-8") + else: + return None + + +class DarkerFormatHelper(FormatHelper): + name = "darker" + friendly_name = "Python code formatter" + + def _construct_command(self, diff_expression: str | None) -> str: + darker_cmd = [ + self.darker_fmt_path, + "--check", + "--diff", + ] + if diff_expression: + darker_cmd += ["-r", diff_expression] + darker_cmd += self._py_files + return darker_cmd + + @property + def instructions(self) -> str: + return " ".join(self._construct_command("origin/main...HEAD")) + + def filter_changed_files(self, changed_files: List[str]) -> List[str]: + filtered_files = [] + for path in changed_files: + name, ext = os.path.splitext(path) + if ext == ".py": + filtered_files.append(path) + + return filtered_files + + @property + def darker_fmt_path(self) -> str: + if "DARKER_FORMAT_PATH" in os.environ: + return os.environ["DARKER_FORMAT_PATH"] + return "darker" + + def has_tool(self) -> bool: + cmd = [self.darker_fmt_path, "--version"] + proc = None + try: + proc = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + except: + return False + return proc.returncode == 0 + + def format_run(self, changed_files: List[str], args: FormatArgs) -> Optional[str]: + py_files = self.filter_changed_files(changed_files) + if not py_files: + return None + self._py_files = py_files + diff_expression = None + if args.start_rev and args.end_rev: + diff_expression = f"{args.start_rev}...{args.end_rev}" + darker_cmd = self._construct_command(diff_expression) + if args.verbose: + print(f"Running: {' '.join(darker_cmd)}") + proc = subprocess.run( + darker_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE + ) + if args.verbose: + sys.stdout.write(proc.stderr.decode("utf-8")) + + if proc.returncode != 0: + # formatting needed, or the command otherwise failed + if args.verbose: + print(f"error: {self.name} exited with code {proc.returncode}") + # Print the diff in the log so that it is viewable there + print(proc.stdout.decode("utf-8")) + return proc.stdout.decode("utf-8") + else: + sys.stdout.write(proc.stdout.decode("utf-8")) + return None + + +class UndefGetFormatHelper(FormatHelper): + name = "undef deprecator" + friendly_name = "undef deprecator" + + @property + def instructions(self) -> str: + return " ".join(shlex.quote(c) for c in self.cmd) + + def filter_changed_files(self, changed_files: List[str]) -> List[str]: + filtered_files = [] + for path in changed_files: + _, ext = os.path.splitext(path) + if ext in (".cpp", ".c", ".h", ".hpp", ".hxx", ".cxx", ".inc", ".cppm", ".ll"): + filtered_files.append(path) + return filtered_files + + def has_tool(self) -> bool: + return True + + def pr_comment_text_for_diff(self, diff: str) -> str: + return f""" +:warning: {self.name} found issues in your code. :warning: + +<details> +<summary> +You can test this locally with the following command: +</summary> + +``````````bash +{self.instructions} +`````````` + +</details> + +{diff} +""" + + def format_run(self, changed_files: List[str], args: FormatArgs) -> Optional[str]: + files = self.filter_changed_files(changed_files) + if not files: + return None + + # Use git to find files that have had a change in the number of undefs + regex = "([^a-zA-Z0-9#_-]undef([^a-zA-Z0-9_-]|$)|UndefValue::get)" + cmd = ["git", "diff", "-U0", "--pickaxe-regex", "-S", regex] + + if args.start_rev and args.end_rev: + cmd.append(args.start_rev) + cmd.append(args.end_rev) + + cmd += files + self.cmd = cmd + + if args.verbose: + print(f"Running: {self.instructions}") + + proc = subprocess.run( + cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, encoding="utf-8" + ) + sys.stdout.write(proc.stderr) + stdout = proc.stdout + + if not stdout: + return None + + files = [] + + # Split the diff so we have one array entry per file. + # Each file is prefixed like: + # diff --git a/file b/file + for file in re.split("^diff --git ", stdout, 0, re.MULTILINE): + lines = file.splitlines() + match = re.match("a/([^ ]+)", lines[0] if lines else "") + filename = match[1] if match else "" + if filename.endswith(".ll"): + undef_regex = r"^[+][^;\n]*(?<!%)\bundef\b" + else: + undef_regex = r"^[+].*UndefValue::get" + # search for additions of undef + if re.search(undef_regex, file, re.MULTILINE): + files.append(filename) + + if not files: + return None + + files = "\n".join(" - " + f for f in files) + report = f""" +The following files introduce new uses of undef: +{files} + +[Undef](https://llvm.org/docs/LangRef.html#undefined-values) is now deprecated and should only be used in the rare cases where no replacement is possible. For example, a load of uninitialized memory yields `undef`. You should use `poison` values for placeholders instead. + +In tests, avoid using `undef` and having tests that trigger undefined behavior. If you need an operand with some unimportant value, you can add a new argument to the function and use that instead. + +For example, this is considered a bad practice: +```llvm +define void @fn() {{ + ... + br i1 undef, ... +}} +``` + +Please use the following instead: +```llvm +define void @fn(i1 %cond) {{ + ... + br i1 %cond, ... +}} +``` + +Please refer to the [Undefined Behavior Manual](https://llvm.org/docs/UndefinedBehavior.html) for more information. +""" + if args.verbose: + print(f"error: {self.name} failed") + print(report) + return report + + +class CMakeFormatHelper(FormatHelper): + name = "cmake-format" + friendly_name = "CMake code formatter" + + def _construct_command(self) -> List[str]: + return [ + sys.executable, + self.cmake_fmt_path, + "--dry-run", + "--diff", + ] + self._cmake_files + + @property + def instructions(self) -> str: + return " ".join(["python3", self.cmake_fmt_path, "-i"] + self._cmake_files) + + @property + def check_instructions(self) -> str: + return " ".join( + ["python3", self.cmake_fmt_path, "--dry-run", "--diff"] + self._cmake_files + ) + + def pr_comment_text_for_diff(self, diff: str) -> str: + return f""" +:warning: {self.friendly_name}, {self.name} found issues in your code. :warning: + +<details> +<summary> +You can fix this locally with the following command: +</summary> + +``````````bash +{self.instructions} +`````````` + +You can also preview the formatting diff without modifying files: + +``````````bash +{self.check_instructions} +`````````` + +</details> + +<details> +<summary> +View the diff from {self.name} here. +</summary> + +``````````diff +{diff} +`````````` + +</details> +""" + + def filter_changed_files(self, changed_files: List[str]) -> List[str]: + filtered_files = [] + for path in changed_files: + if not path.startswith("libc/"): + continue + if os.path.basename(path) == "CMakeLists.txt" or path.endswith(".cmake"): + if os.path.exists(path): + filtered_files.append(path) + return filtered_files + + @property + def cmake_fmt_path(self) -> str: + if "CMAKE_FORMAT_PATH" in os.environ: + return os.environ["CMAKE_FORMAT_PATH"] + return "libc/utils/cmake_format.py" + + def has_tool(self) -> bool: + return os.path.isfile(self.cmake_fmt_path) + + def format_run(self, changed_files: List[str], args: FormatArgs) -> Optional[str]: + cmake_files = self.filter_changed_files(changed_files) + if not cmake_files: + return None + self._cmake_files = cmake_files + cmake_cmd = self._construct_command() + if args.verbose: + print(f"Running: {' '.join(cmake_cmd)}") + proc = subprocess.run(cmake_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + if args.verbose: + sys.stdout.write(proc.stderr.decode("utf-8")) + + if proc.returncode != 0: + # formatting needed, or the command otherwise failed + stdout = proc.stdout.decode("utf-8") + if args.verbose: + print(f"error: {self.name} exited with code {proc.returncode}") + if stdout: + print( + "error: CMake files are not formatted according to " + "LLVM-libc style." + ) + print(f"To fix this locally, run:\n {self.instructions}") + print( + "To preview the formatting diff locally, run:\n" + f" {self.check_instructions}" + ) + # Print the diff in the log so that it is viewable there + print(stdout) + elif stdout: + print(f"To fix CMake formatting, run:\n {self.instructions}") + return stdout + else: + sys.stdout.write(proc.stdout.decode("utf-8")) + return None + + +ALL_FORMATTERS = ( + DarkerFormatHelper(), + ClangFormatHelper(), + CMakeFormatHelper(), + UndefGetFormatHelper(), +) + + +def hook_main(): + # fill out args + args = FormatArgs() + args.verbose = os.getenv("FORMAT_HOOK_VERBOSE", False) + + # find the changed files + cmd = ["git", "diff", "--cached", "--name-only", "--diff-filter=d"] + proc = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + output = proc.stdout.decode("utf-8") + for line in output.splitlines(): + args.changed_files.append(line) + + failed_fmts = [] + for fmt in ALL_FORMATTERS: + if fmt.has_tool(): + if not fmt.run(args.changed_files, args): + failed_fmts.append(fmt.name) + else: + print(f"Couldn't find {fmt.name}, can't check " + fmt.friendly_name.lower()) + + if len(failed_fmts) > 0: + print( + "Pre-commit format hook failed, rerun with FORMAT_HOOK_VERBOSE=1 environment for verbose output" + ) + sys.exit(1) + + sys.exit(0) + + +if __name__ == "__main__": + script_path = os.path.abspath(__file__) + if ".git/hooks" in script_path: + hook_main() + sys.exit(0) + + parser = argparse.ArgumentParser() + parser.add_argument( + "--token", type=str, required=True, help="GitHub authentication token" + ) + parser.add_argument( + "--repo", + type=str, + default=os.getenv("GITHUB_REPOSITORY", "llvm/llvm-project"), + help="The GitHub repository that we are working with in the form of <owner>/<repo> (e.g. llvm/llvm-project)", + ) + parser.add_argument("--issue-number", type=int, required=True) + parser.add_argument( + "--start-rev", + type=str, + required=True, + help="Compute changes from this revision.", + ) + parser.add_argument( + "--end-rev", type=str, required=True, help="Compute changes to this revision" + ) + parser.add_argument( + "--changed-files", + type=str, + help="Comma separated list of files that has been changed", + ) + parser.add_argument( + "--write-comment-to-file", + action="store_true", + help="Don't post comments on the PR, instead write the comments and metadata a file called 'comment'", + ) + + args = FormatArgs(parser.parse_args()) + + changed_files = [] + if args.changed_files: + changed_files = args.changed_files.split(",") + + failed_formatters = [] + comments = [] + for fmt in ALL_FORMATTERS: + if not fmt.run(changed_files, args): + failed_formatters.append(fmt.name) + if fmt.comment: + comments.append(fmt.comment) + + if len(comments): + with open("comments", "w") as f: + import json + + json.dump(comments, f) + + if len(failed_formatters) > 0: + print(f"error: some formatters failed: {' '.join(failed_formatters)}") + sys.exit(1) |