diff options
| -rw-r--r-- | rcc/CMakeLists.txt | 2 | ||||
| -rw-r--r-- | rcc/utils/CMakeLists.txt | 43 | ||||
| -rwxr-xr-x | rcc/utils/clang-format/git-clang-format | 891 | ||||
| -rwxr-xr-x | rcc/utils/clang-tidy/clang-tidy-diff.py | 442 | ||||
| -rwxr-xr-x | rcc/utils/clang-tidy/run-clang-tidy.py | 796 | ||||
| -rw-r--r-- | rcc/utils/dev.py | 277 | ||||
| -rwxr-xr-x | rcc/utils/git/code-format-helper.py | 677 |
7 files changed, 2806 insertions, 322 deletions
diff --git a/rcc/CMakeLists.txt b/rcc/CMakeLists.txt index f80833c..86f1c80 100644 --- a/rcc/CMakeLists.txt +++ b/rcc/CMakeLists.txt @@ -68,8 +68,6 @@ install(DIRECTORY "${RCC_BINARY_DIR}/include/rcc" DESTINATION "${CMAKE_INSTALL_I add_subdirectory(lib) add_subdirectory(tools) -add_subdirectory(utils) - foreach(rccProject IN LISTS RCC_ENABLE_PROJECTS) if(NOT rccProject STREQUAL "cfront") message(FATAL_ERROR "Unknown RCC project: ${rccProject}. Available: cfront") diff --git a/rcc/utils/CMakeLists.txt b/rcc/utils/CMakeLists.txt deleted file mode 100644 index bb13e46..0000000 --- a/rcc/utils/CMakeLists.txt +++ /dev/null @@ -1,43 +0,0 @@ -# SPDX-License-Identifier: GPL-3.0-or-later - -# Resolve developer tools only when the corresponding target is invoked. -set(RCC_PYTHON_EXECUTABLE python3 CACHE STRING "Python interpreter for developer tools") -set(RCC_CLANG_FORMAT_EXECUTABLE clang-format CACHE STRING "clang-format executable") -set(RCC_CLANG_TIDY_EXECUTABLE clang-tidy CACHE STRING "clang-tidy executable") -set(RCC_ANALYZER_EXECUTABLE clang++ CACHE STRING "Clang static analyzer executable") -set(RCC_CLANG_FORMAT_VERSION "" CACHE STRING "Required clang-format version, if specified") -set(RCC_CLANG_TIDY_VERSION "" CACHE STRING "Required clang-tidy version, if specified") -set(RCC_ANALYZER_VERSION "" CACHE STRING "Required Clang analyzer version, if specified") -set(RCC_DEV_BASE HEAD CACHE STRING "Git revision used for changed-file checks") - -set(rccDevScript "${CMAKE_CURRENT_SOURCE_DIR}/dev.py") -add_custom_target(format-check - COMMAND "${RCC_PYTHON_EXECUTABLE}" "${rccDevScript}" format - --build-dir "${CMAKE_BINARY_DIR}" - --base "${RCC_DEV_BASE}" - --tool "${RCC_CLANG_FORMAT_EXECUTABLE}" - --tool-version "${RCC_CLANG_FORMAT_VERSION}" - WORKING_DIRECTORY "${RCC_SOURCE_DIR}/.." - USES_TERMINAL VERBATIM) -add_custom_target(format - COMMAND "${RCC_PYTHON_EXECUTABLE}" "${rccDevScript}" format --fix - --build-dir "${CMAKE_BINARY_DIR}" - --base "${RCC_DEV_BASE}" - --tool "${RCC_CLANG_FORMAT_EXECUTABLE}" - --tool-version "${RCC_CLANG_FORMAT_VERSION}" - WORKING_DIRECTORY "${RCC_SOURCE_DIR}/.." - USES_TERMINAL VERBATIM) -add_custom_target(tidy - COMMAND "${RCC_PYTHON_EXECUTABLE}" "${rccDevScript}" tidy - --build-dir "${CMAKE_BINARY_DIR}" --base "${RCC_DEV_BASE}" - --tool "${RCC_CLANG_TIDY_EXECUTABLE}" - --tool-version "${RCC_CLANG_TIDY_VERSION}" - WORKING_DIRECTORY "${RCC_SOURCE_DIR}/.." - USES_TERMINAL VERBATIM) -add_custom_target(analyze - COMMAND "${RCC_PYTHON_EXECUTABLE}" "${rccDevScript}" analyze - --build-dir "${CMAKE_BINARY_DIR}" --base "${RCC_DEV_BASE}" - --tool "${RCC_ANALYZER_EXECUTABLE}" - --tool-version "${RCC_ANALYZER_VERSION}" - WORKING_DIRECTORY "${RCC_SOURCE_DIR}/.." - USES_TERMINAL VERBATIM) diff --git a/rcc/utils/clang-format/git-clang-format b/rcc/utils/clang-format/git-clang-format new file mode 100755 index 0000000..d9b2d50 --- /dev/null +++ b/rcc/utils/clang-format/git-clang-format @@ -0,0 +1,891 @@ +#!/usr/bin/env python3 +# +# ===- git-clang-format - ClangFormat Git Integration -------*- python -*--=== # +# +# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +# See https://llvm.org/LICENSE.txt for license information. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# +# ===----------------------------------------------------------------------=== # + +r""" +clang-format git integration +============================ + +This file provides a clang-format integration for git. Put it somewhere in your +path and ensure that it is executable. Then, "git clang-format" will invoke +clang-format on the changes in current files or a specific commit. + +For further details, run: +git clang-format -h + +Requires Python version >=3.8 +""" + +from __future__ import absolute_import, division, print_function +import argparse +import collections +import contextlib +import errno +import os +import re +import subprocess +import sys +import tempfile + +usage = "git clang-format [OPTIONS] [<commit>] [<commit>|--staged] [--] [<file>...]" + +desc = """ +If zero or one commits are given, run clang-format on all lines that differ +between the working directory and <commit>, which defaults to HEAD. Changes are +only applied to the working directory, or in the stage/index. + +Examples: + To format staged changes, i.e everything that's been `git add`ed: + git clang-format + + To also format everything touched in the most recent commit: + git clang-format HEAD~1 + + If you're on a branch off main, to format everything touched on your branch: + git clang-format main + +If two commits are given (requires --diff), run clang-format on all lines in the +second <commit> that differ from the first <commit>. + +The following git-config settings set the default of the corresponding option: + clangFormat.binary + clangFormat.commit + clangFormat.extensions + clangFormat.style + +When running inside a pre-commit CI environment (i.e., no commits are given +on the command line and the PRE_COMMIT_FROM_REF and PRE_COMMIT_TO_REF +environment variables are set), the diff range is taken from those variables +and --diff is implied. +""" + +# Name of the temporary index file in which save the output of clang-format. +# This file is created within the .git directory. +temp_index_basename = "clang-format-index" + + +Range = collections.namedtuple("Range", "start, count") + + +def main(): + config = load_git_config() + + # In order to keep '--' yet allow options after positionals, we need to + # check for '--' ourselves. (Setting nargs='*' throws away the '--', while + # nargs=argparse.REMAINDER disallows options after positionals.) + argv = sys.argv[1:] + try: + idx = argv.index("--") + except ValueError: + dash_dash = [] + else: + dash_dash = argv[idx:] + argv = argv[:idx] + + default_extensions = ",".join( + [ + # From clang/lib/Frontend/FrontendOptions.cpp, all lower case + "c", + "h", # C + "m", # ObjC + "mm", # ObjC++ + "cc", + "cp", + "cpp", + "c++", + "cxx", + "hh", + "hpp", + "hxx", + "inc", # C++ + "ccm", + "cppm", + "cxxm", + "c++m", # C++ Modules + "cu", + "cuh", # CUDA + "cl", # OpenCL + # Other languages that clang-format supports + "proto", + "protodevel", # Protocol Buffers + "java", # Java + "js", + "mjs", + "cjs", # JavaScript + "ts", # TypeScript + "cs", # C Sharp + "json", + "ipynb", # JSON + "sv", + "svh", + "v", + "vh", # Verilog + "td", # TableGen + "txtpb", + "textpb", + "pb.txt", + "textproto", + "asciipb", # TextProto + ] + ) + + p = argparse.ArgumentParser( + usage=usage, + formatter_class=argparse.RawDescriptionHelpFormatter, + description=desc, + ) + p.add_argument( + "--binary", + default=config.get("clangformat.binary", "clang-format"), + help="path to clang-format", + ), + p.add_argument( + "--commit", + default=config.get("clangformat.commit", "HEAD"), + help="default commit to use if none is specified", + ), + p.add_argument( + "--diff", + action="store_true", + help="print a diff instead of applying the changes", + ) + p.add_argument( + "--diffstat", + action="store_true", + help="print a diffstat instead of applying the changes", + ) + p.add_argument( + "--extensions", + default=config.get("clangformat.extensions", default_extensions), + help=( + "comma-separated list of file extensions to format, " + "excluding the period and case-insensitive" + ), + ), + p.add_argument( + "-f", + "--force", + action="store_true", + help="allow changes to unstaged files", + ) + p.add_argument( + "-p", "--patch", action="store_true", help="select hunks interactively" + ) + p.add_argument( + "-q", + "--quiet", + action="count", + default=0, + help="print less information", + ) + p.add_argument( + "--staged", + "--cached", + action="store_true", + help="format lines in the stage instead of the working dir", + ) + p.add_argument( + "--style", + default=config.get("clangformat.style", None), + help="passed to clang-format", + ), + p.add_argument( + "-v", + "--verbose", + action="count", + default=0, + help="print extra information", + ) + p.add_argument( + "--whole-file", + action="store_true", + help="format whole file instead of only modified lines", + ) + p.add_argument( + "--diff_from_common_commit", + action="store_true", + help=( + "diff from the last common commit for commits in " + "separate branches rather than the exact point of the " + "commits" + ), + ) + # We gather all the remaining positional arguments into 'args' since we need + # to use some heuristics to determine whether or not <commit> was present. + # However, to print pretty messages, we make use of metavar and help. + p.add_argument( + "args", + nargs="*", + metavar="<commit>", + help="revision from which to compute the diff", + ) + p.add_argument( + "ignored", + nargs="*", + metavar="<file>...", + help="if specified, only consider differences in these files", + ) + opts = p.parse_args(argv) + + # When no commits are given explicitly and the pre-commit CI framework's + # environment variables are set, use them to define the diff range. + if not opts.args and not dash_dash: + from_ref = os.environ.get("PRE_COMMIT_FROM_REF") + to_ref = os.environ.get("PRE_COMMIT_TO_REF") + if from_ref and to_ref: + opts.args = [from_ref, to_ref] + if not opts.diff and not opts.diffstat: + opts.diff = True + + opts.verbose -= opts.quiet + del opts.quiet + + commits, files = interpret_args(opts.args, dash_dash, opts.commit) + if len(commits) > 2: + die("at most two commits allowed; %d given" % len(commits)) + if len(commits) == 2: + if opts.staged: + die("--staged is not allowed when two commits are given") + if not opts.diff: + die("--diff is required when two commits are given") + elif opts.diff_from_common_commit: + die("--diff_from_common_commit is only allowed when two commits are given") + + if os.path.dirname(opts.binary): + opts.binary = os.path.abspath(opts.binary) + + changed_lines = compute_diff_and_extract_lines( + commits, files, opts.staged, opts.whole_file, opts.diff_from_common_commit + ) + if opts.verbose >= 1: + ignored_files = set(changed_lines) + filter_by_extension(changed_lines, opts.extensions.lower().split(",")) + # The computed diff outputs absolute paths, so we must cd before accessing + # those files. + cd_to_toplevel() + filter_symlinks(changed_lines) + filter_ignored_files(changed_lines, binary=opts.binary) + if opts.verbose >= 1: + ignored_files.difference_update(changed_lines) + if ignored_files: + print( + "Ignoring the following files (wrong extension, symlink, or " + "ignored by clang-format):" + ) + for filename in ignored_files: + print(" %s" % filename) + if changed_lines: + print("Running clang-format on the following files:") + for filename in changed_lines: + print(" %s" % filename) + + if not changed_lines: + if opts.verbose >= 0: + print("no modified files to format") + return 0 + + if len(commits) > 1: + old_tree = commits[1] + revision = old_tree + elif opts.staged: + old_tree = create_tree_from_index(changed_lines) + revision = "" + else: + old_tree = create_tree_from_workdir(changed_lines) + revision = None + new_tree = run_clang_format_and_save_to_tree( + changed_lines, revision, binary=opts.binary, style=opts.style + ) + if opts.verbose >= 1: + print("old tree: %s" % old_tree) + print("new tree: %s" % new_tree) + + if old_tree == new_tree: + if opts.verbose >= 0: + print("clang-format did not modify any files") + return 0 + + if opts.diff: + return print_diff(old_tree, new_tree) + if opts.diffstat: + return print_diffstat(old_tree, new_tree) + + changed_files = apply_changes( + old_tree, new_tree, force=opts.force, patch_mode=opts.patch + ) + if (opts.verbose >= 0 and not opts.patch) or opts.verbose >= 1: + print("changed files:") + for filename in changed_files: + print(" %s" % filename) + + return 1 + + +def load_git_config(non_string_options=None): + """Return the git configuration as a dictionary. + + All options are assumed to be strings unless in `non_string_options`, in + which is a dictionary mapping option name (in lower case) to either "--bool" + or "--int".""" + if non_string_options is None: + non_string_options = {} + out = {} + for entry in run("git", "config", "--list", "--null").split("\0"): + if entry: + if "\n" in entry: + name, value = entry.split("\n", 1) + else: + # A setting with no '=' ('\n' with --null) is implicitly 'true' + name = entry + value = "true" + if name in non_string_options: + value = run("git", "config", non_string_options[name], name) + out[name] = value + return out + + +def interpret_args(args, dash_dash, default_commit): + """Interpret `args` as "[commits] [--] [files]" and return (commits, files). + + It is assumed that "--" and everything that follows has been removed from + args and placed in `dash_dash`. + + If "--" is present (i.e., `dash_dash` is non-empty), the arguments to its + left (if present) are taken as commits. Otherwise, the arguments are + checked from left to right if they are commits or files. If commits are not + given, a list with `default_commit` is used.""" + if dash_dash: + if len(args) == 0: + commits = [default_commit] + else: + commits = args + for commit in commits: + object_type = get_object_type(commit) + if object_type not in ("commit", "tag"): + if object_type is None: + die("'%s' is not a commit" % commit) + else: + die( + "'%s' is a %s, but a commit was expected" + % (commit, object_type) + ) + files = dash_dash[1:] + elif args: + commits = [] + while args: + if not disambiguate_revision(args[0]): + break + commits.append(args.pop(0)) + if not commits: + commits = [default_commit] + files = args + else: + commits = [default_commit] + files = [] + return commits, files + + +def disambiguate_revision(value): + """Returns True if `value` is a revision, False if it is a file, or dies.""" + # If `value` is ambiguous (neither a commit nor a file), the following + # command will die with an appropriate error message. + run("git", "rev-parse", value, verbose=False) + object_type = get_object_type(value) + if object_type is None: + return False + if object_type in ("commit", "tag"): + return True + die("`%s` is a %s, but a commit or filename was expected" % (value, object_type)) + + +def get_object_type(value): + """Returns a string description of an object's type, or None if it is not + a valid git object.""" + cmd = ["git", "cat-file", "-t", value] + p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + stdout, stderr = p.communicate() + if p.returncode != 0: + return None + return convert_string(stdout.strip()) + + +def compute_diff_and_extract_lines( + commits, files, staged, whole_file, diff_common_commit +): + """Calls compute_diff() followed by extract_lines().""" + diff_process = compute_diff(commits, files, staged, diff_common_commit) + changed_lines = extract_lines(diff_process.stdout, whole_file) + diff_process.stdout.close() + diff_process.wait() + if diff_process.returncode != 0: + # Assume error was already printed to stderr. + sys.exit(2) + return changed_lines + + +def compute_diff(commits, files, staged, diff_common_commit): + """Return a subprocess object producing the diff from `commits`. + + The return value's `stdin` file object will produce a patch with the + differences between the working directory (or stage if --staged is used) and + the first commit if a single one was specified, or the difference between + both specified commits, filtered on `files` (if non-empty). + Zero context lines are used in the patch.""" + git_tool = "diff-index" + extra_args = [] + if len(commits) == 2: + git_tool = "diff-tree" + if diff_common_commit: + extra_args += ["--merge-base"] + elif staged: + extra_args += ["--cached"] + + cmd = ["git", git_tool, "-p", "-U0"] + extra_args + commits + ["--"] + cmd.extend(files) + p = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE) + p.stdin.close() + return p + + +def extract_lines(patch_file, whole_file): + """Extract the changed lines in `patch_file`. + + The return value is a dictionary mapping filename to a list of (start_line, + line_count) pairs. + + The input must have been produced with ``-U0``, meaning unidiff format with + zero lines of context. The return value is a dict mapping filename to a + list of line `Range`s.""" + matches = {} + for line in patch_file: + line = convert_string(line) + match = re.search(r"^\+\+\+\ [^/]+/(.*)", line) + if match: + filename = match.group(1).rstrip("\r\n\t") + if whole_file: + # Initialize the key for filename but ensure ranges is an empty + # list. This will prevent any filtering in clang_format_to_blob(). + matches[filename] = [] + continue + if not whole_file: + match = re.search(r"^@@ -[0-9,]+ \+(\d+)(,(\d+))?", line) + if match: + start_line = int(match.group(1)) + line_count = 1 + if match.group(3): + line_count = int(match.group(3)) + # A hunk that adds no lines is a pure deletion. start_line + # refers to the preceding line (0 when the deletion is at the + # start of a file), which the deletion left alone. + if line_count == 0: + continue + matches.setdefault(filename, []).append(Range(start_line, line_count)) + return matches + + +def filter_by_extension(dictionary, allowed_extensions): + """Delete every key in `dictionary` that doesn't have an allowed extension. + + `allowed_extensions` must be a collection of lowercase file extensions, + excluding the period.""" + allowed_extensions = frozenset(allowed_extensions) + for filename in list(dictionary.keys()): + base_ext = filename.rsplit(".", 1) + if len(base_ext) == 1 and "" in allowed_extensions: + continue + if len(base_ext) == 1 or base_ext[1].lower() not in allowed_extensions: + del dictionary[filename] + + +def filter_symlinks(dictionary): + """Delete every key in `dictionary` that is a symlink.""" + for filename in list(dictionary.keys()): + if os.path.islink(filename): + del dictionary[filename] + + +def filter_ignored_files(dictionary, binary): + """Delete every key in `dictionary` that is ignored by clang-format.""" + ignored_files = run(binary, "-list-ignored", *dictionary.keys()) + if not ignored_files: + return + ignored_files = ignored_files.split("\n") + for filename in ignored_files: + del dictionary[filename] + + +def cd_to_toplevel(): + """Change to the top level of the git repository.""" + toplevel = run("git", "rev-parse", "--show-toplevel") + os.chdir(toplevel) + + +def create_tree_from_workdir(filenames): + """Create a new git tree with the given files from the working directory. + + Returns the object ID (SHA-1) of the created tree.""" + return create_tree(filenames, "--stdin") + + +def create_tree_from_index(filenames): + # Copy the environment, because the files have to be read from the original + # index. + env = os.environ.copy() + + def index_contents_generator(): + for filename in filenames: + git_ls_files_cmd = [ + "git", + "ls-files", + "--stage", + "-z", + "--", + filename, + ] + git_ls_files = subprocess.Popen( + git_ls_files_cmd, + env=env, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + ) + stdout = git_ls_files.communicate()[0] + yield convert_string(stdout.split(b"\0")[0]) + + return create_tree(index_contents_generator(), "--index-info") + + +def run_clang_format_and_save_to_tree( + changed_lines, revision=None, binary="clang-format", style=None +): + """Run clang-format on each file and save the result to a git tree. + + Returns the object ID (SHA-1) of the created tree.""" + # Copy the environment when formatting the files in the index, because the + # files have to be read from the original index. + env = os.environ.copy() if revision == "" else None + + def iteritems(container): + try: + return container.iteritems() # Python 2 + except AttributeError: + return container.items() # Python 3 + + def index_info_generator(): + for filename, line_ranges in iteritems(changed_lines): + if revision is not None: + if len(revision) > 0: + git_metadata_cmd = [ + "git", + "ls-tree", + "%s:%s" % (revision, os.path.dirname(filename)), + os.path.basename(filename), + ] + else: + git_metadata_cmd = [ + "git", + "ls-files", + "--stage", + "--", + filename, + ] + git_metadata = subprocess.Popen( + git_metadata_cmd, + env=env, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + ) + stdout = git_metadata.communicate()[0] + mode = oct(int(stdout.split()[0], 8)) + else: + mode = oct(os.stat(filename).st_mode) + # Adjust python3 octal format so that it matches what git expects + if mode.startswith("0o"): + mode = "0" + mode[2:] + blob_id = clang_format_to_blob( + filename, + line_ranges, + revision=revision, + binary=binary, + style=style, + env=env, + ) + yield "%s %s\t%s" % (mode, blob_id, filename) + + return create_tree(index_info_generator(), "--index-info") + + +def create_tree(input_lines, mode): + """Create a tree object from the given input. + + If mode is '--stdin', it must be a list of filenames. If mode is + '--index-info' is must be a list of values suitable for "git update-index + --index-info", such as "<mode> <SP> <sha1> <TAB> <filename>". Any other + mode is invalid.""" + assert mode in ("--stdin", "--index-info") + cmd = ["git", "update-index", "--add", "-z", mode] + with temporary_index_file(): + p = subprocess.Popen(cmd, stdin=subprocess.PIPE) + for line in input_lines: + p.stdin.write(to_bytes("%s\0" % line)) + p.stdin.close() + if p.wait() != 0: + die("`%s` failed" % " ".join(cmd)) + tree_id = run("git", "write-tree") + return tree_id + + +def clang_format_to_blob( + filename, + line_ranges, + revision=None, + binary="clang-format", + style=None, + env=None, +): + """Run clang-format on the given file and save the result to a git blob. + + Runs on the file in `revision` if not None, or on the file in the working + directory if `revision` is None. Revision can be set to an empty string to + run clang-format on the file in the index. + + Returns the object ID (SHA-1) of the created blob.""" + clang_format_cmd = [binary] + if style: + clang_format_cmd.extend(["--style=" + style]) + clang_format_cmd.extend( + [ + "--lines=%s:%s" % (start_line, start_line + line_count - 1) + for start_line, line_count in line_ranges + ] + ) + if revision is not None: + clang_format_cmd.extend(["--assume-filename=" + filename]) + git_show_cmd = [ + "git", + "cat-file", + "blob", + "%s:%s" % (revision, filename), + ] + git_show = subprocess.Popen( + git_show_cmd, env=env, stdin=subprocess.PIPE, stdout=subprocess.PIPE + ) + git_show.stdin.close() + clang_format_stdin = git_show.stdout + else: + clang_format_cmd.extend([filename]) + git_show = None + clang_format_stdin = subprocess.PIPE + try: + clang_format = subprocess.Popen( + clang_format_cmd, stdin=clang_format_stdin, stdout=subprocess.PIPE + ) + if clang_format_stdin == subprocess.PIPE: + clang_format_stdin = clang_format.stdin + except OSError as e: + if e.errno == errno.ENOENT: + die('cannot find executable "%s"' % binary) + else: + raise + clang_format_stdin.close() + hash_object_cmd = [ + "git", + "hash-object", + "-w", + "--path=" + filename, + "--stdin", + ] + hash_object = subprocess.Popen( + hash_object_cmd, stdin=clang_format.stdout, stdout=subprocess.PIPE + ) + clang_format.stdout.close() + stdout = hash_object.communicate()[0] + if hash_object.returncode != 0: + die("`%s` failed" % " ".join(hash_object_cmd)) + if clang_format.wait() != 0: + die("`%s` failed" % " ".join(clang_format_cmd)) + if git_show and git_show.wait() != 0: + die("`%s` failed" % " ".join(git_show_cmd)) + return convert_string(stdout).rstrip("\r\n") + + +@contextlib.contextmanager +def temporary_index_file(tree=None): + """Context manager for setting GIT_INDEX_FILE to a temporary file and + deleting the file afterward.""" + index_path = create_temporary_index(tree) + old_index_path = os.environ.get("GIT_INDEX_FILE") + os.environ["GIT_INDEX_FILE"] = index_path + try: + yield + finally: + if old_index_path is None: + del os.environ["GIT_INDEX_FILE"] + else: + os.environ["GIT_INDEX_FILE"] = old_index_path + os.remove(index_path) + + +def create_temporary_index(tree=None): + """Create a temporary index file and return the created file's path. + + If `tree` is not None, use that as the tree to read in. Otherwise, an + empty index is created.""" + gitdir = run("git", "rev-parse", "--git-dir") + # Use a unique filename to avoid stale .lock files when concurrent git + # operations (e.g. pre-commit hooks) race with this process. + fd, path = tempfile.mkstemp(prefix=temp_index_basename + "-", dir=gitdir) + os.close(fd) + if tree is None: + tree = "--empty" + run("git", "read-tree", "--index-output=" + path, tree) + return path + + +def print_diff(old_tree, new_tree): + """Print the diff between the two trees to stdout.""" + # We use the porcelain 'diff' and not plumbing 'diff-tree' because the + # output is expected to be viewed by the user, and only the former does nice + # things like color and pagination. + # + # We also only print modified files since `new_tree` only contains the files + # that were modified, so unmodified files would show as deleted without the + # filter. + return subprocess.run( + ["git", "diff", "--diff-filter=M", "--exit-code", old_tree, new_tree] + ).returncode + + +def print_diffstat(old_tree, new_tree): + """Print the diffstat between the two trees to stdout.""" + # We use the porcelain 'diff' and not plumbing 'diff-tree' because the + # output is expected to be viewed by the user, and only the former does nice + # things like color and pagination. + # + # We also only print modified files since `new_tree` only contains the files + # that were modified, so unmodified files would show as deleted without the + # filter. + return subprocess.run( + [ + "git", + "diff", + "--diff-filter=M", + "--exit-code", + "--stat", + old_tree, + new_tree, + ] + ).returncode + + +def apply_changes(old_tree, new_tree, force=False, patch_mode=False): + """Apply the changes in `new_tree` to the working directory. + + Bails if there are local changes in those files and not `force`. If + `patch_mode`, runs `git checkout --patch` to select hunks interactively.""" + changed_files = ( + run( + "git", + "diff-tree", + "--diff-filter=M", + "-r", + "-z", + "--name-only", + old_tree, + new_tree, + ) + .rstrip("\0") + .split("\0") + ) + if not force: + unstaged_files = run("git", "diff-files", "--name-status", *changed_files) + if unstaged_files: + print( + "The following files would be modified but have unstaged changes:", + file=sys.stderr, + ) + print(unstaged_files, file=sys.stderr) + print("Please commit, stage, or stash them first.", file=sys.stderr) + sys.exit(2) + if patch_mode: + # In patch mode, we could just as well create an index from the new tree + # and checkout from that, but then the user will be presented with a + # message saying "Discard ... from worktree". Instead, we use the old + # tree as the index and checkout from new_tree, which gives the slightly + # better message, "Apply ... to index and worktree". This is not quite + # right, since it won't be applied to the user's index, but oh well. + with temporary_index_file(old_tree): + subprocess.run(["git", "checkout", "--patch", new_tree], check=True) + index_tree = old_tree + else: + with temporary_index_file(new_tree): + run("git", "checkout-index", "-f", "--", *changed_files) + return changed_files + + +def run(*args, **kwargs): + stdin = kwargs.pop("stdin", "") + verbose = kwargs.pop("verbose", True) + strip = kwargs.pop("strip", True) + for name in kwargs: + raise TypeError("run() got an unexpected keyword argument '%s'" % name) + p = subprocess.Popen( + args, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + stdin=subprocess.PIPE, + ) + stdout, stderr = p.communicate(input=stdin) + + stdout = convert_string(stdout) + stderr = convert_string(stderr) + + if p.returncode == 0: + if stderr: + if verbose: + print("`%s` printed to stderr:" % " ".join(args), file=sys.stderr) + print(stderr.rstrip(), file=sys.stderr) + if strip: + stdout = stdout.rstrip("\r\n") + return stdout + if verbose: + print("`%s` returned %s" % (" ".join(args), p.returncode), file=sys.stderr) + if stderr: + print(stderr.rstrip(), file=sys.stderr) + sys.exit(2) + + +def die(message): + print("error:", message, file=sys.stderr) + sys.exit(2) + + +def to_bytes(str_input): + # Encode to UTF-8 to get binary data. + if isinstance(str_input, bytes): + return str_input + return str_input.encode("utf-8") + + +def to_string(bytes_input): + if isinstance(bytes_input, str): + return bytes_input + return bytes_input.encode("utf-8") + + +def convert_string(bytes_input): + try: + return to_string(bytes_input.decode("utf-8")) + except AttributeError: # 'str' object has no attribute 'decode'. + return str(bytes_input) + except UnicodeError: + return str(bytes_input) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/rcc/utils/clang-tidy/clang-tidy-diff.py b/rcc/utils/clang-tidy/clang-tidy-diff.py new file mode 100755 index 0000000..059b2a2 --- /dev/null +++ b/rcc/utils/clang-tidy/clang-tidy-diff.py @@ -0,0 +1,442 @@ +#!/usr/bin/env python3 +# +# ===-----------------------------------------------------------------------===# +# +# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +# See https://llvm.org/LICENSE.txt for license information. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# +# ===-----------------------------------------------------------------------===# + +r""" +ClangTidy Diff Checker +====================== + +This script reads input from a unified diff, runs clang-tidy on all changed +files and outputs clang-tidy warnings in changed lines only. This is useful to +detect clang-tidy regressions in the lines touched by a specific patch. +Example usage for git/svn users: + + git diff -U0 HEAD^ | clang-tidy-diff.py -p1 + svn diff --diff-cmd=diff -x-U0 | \ + clang-tidy-diff.py -fix -checks=-*,modernize-use-override + +""" + +import argparse +import glob +import json +import multiprocessing +import os +import queue +import re +import shutil +import subprocess +import sys +import tempfile +import threading +import traceback +from pathlib import Path + +try: + import yaml +except ImportError: + yaml = None + + +def run_tidy(task_queue, lock, timeout, failed_files): + watchdog = None + while True: + command = task_queue.get() + try: + proc = subprocess.Popen( + command, stdout=subprocess.PIPE, stderr=subprocess.PIPE + ) + + if timeout is not None: + watchdog = threading.Timer(timeout, proc.kill) + watchdog.start() + + stdout, stderr = proc.communicate() + if proc.returncode != 0: + if proc.returncode < 0: + msg = "Terminated by signal %d : %s\n" % ( + -proc.returncode, + " ".join(command), + ) + stderr += msg.encode("utf-8") + failed_files.append(command) + + with lock: + if stdout: + sys.stdout.write(stdout.decode("utf-8") + "\n") + sys.stdout.flush() + if stderr: + sys.stderr.write(stderr.decode("utf-8") + "\n") + sys.stderr.flush() + except Exception as e: + with lock: + sys.stderr.write("Failed: " + str(e) + ": ".join(command) + "\n") + finally: + with lock: + if not (timeout is None or watchdog is None): + if not watchdog.is_alive(): + sys.stderr.write( + "Terminated by timeout: " + " ".join(command) + "\n" + ) + watchdog.cancel() + task_queue.task_done() + + +def start_workers(max_tasks, tidy_caller, arguments): + for _ in range(max_tasks): + t = threading.Thread(target=tidy_caller, args=arguments) + t.daemon = True + t.start() + + +def merge_replacement_files(tmpdir, mergefile): + """Merge all replacement files in a directory into a single file""" + # The fixes suggested by clang-tidy >= 4.0.0 are given under + # the top level key 'Diagnostics' in the output yaml files + mergekey = "Diagnostics" + merged = [] + for replacefile in glob.iglob(os.path.join(tmpdir, "*.yaml")): + content = yaml.safe_load(open(replacefile, "r")) + if not content: + continue # Skip empty files. + merged.extend(content.get(mergekey, [])) + + if merged: + # MainSourceFile: The key is required by the definition inside + # include/clang/Tooling/ReplacementsYaml.h, but the value + # is actually never used inside clang-apply-replacements, + # so we set it to '' here. + output = {"MainSourceFile": "", mergekey: merged} + with open(mergefile, "w") as out: + yaml.safe_dump(output, out) + else: + # Empty the file: + open(mergefile, "w").close() + + +def get_compiling_files(args): + """Read a compile_commands.json database and return a set of file paths""" + current_dir = Path.cwd() + compile_commands_json = ( + (current_dir / args.build_path) if args.build_path else current_dir + ) + compile_commands_json = compile_commands_json / "compile_commands.json" + files = set() + with open(compile_commands_json) as db_file: + db_json = json.load(db_file) + for entry in db_json: + if "file" not in entry: + continue + files.add(Path(entry["file"])) + return files + + +def main(): + parser = argparse.ArgumentParser( + description="Run clang-tidy against changed files, and " + "output diagnostics only for modified " + "lines." + ) + parser.add_argument( + "-clang-tidy-binary", + metavar="PATH", + default="clang-tidy", + help="path to clang-tidy binary", + ) + parser.add_argument( + "-p", + metavar="NUM", + default=0, + help="strip the smallest prefix containing P slashes", + ) + parser.add_argument( + "-regex", + metavar="PATTERN", + default=None, + help="custom pattern selecting file paths to check " + "(case sensitive, overrides -iregex)", + ) + parser.add_argument( + "-iregex", + metavar="PATTERN", + default=r".*\.(cpp|cc|c\+\+|cxx|c|cl|h|hpp|m|mm|inc)", + help="custom pattern selecting file paths to check " + "(case insensitive, overridden by -regex)", + ) + parser.add_argument( + "-j", + type=int, + default=0, + help="number of tidy instances to be run in parallel.", + ) + parser.add_argument( + "-timeout", type=int, default=None, help="timeout per each file in seconds." + ) + parser.add_argument( + "-fix", action="store_true", default=False, help="apply suggested fixes" + ) + parser.add_argument( + "-checks", + help="checks filter, when not specified, use clang-tidy " "default", + default="", + ) + parser.add_argument( + "-config-file", + dest="config_file", + help="Specify the path of .clang-tidy or custom config file", + default="", + ) + parser.add_argument("-use-color", action="store_true", help="Use colors in output") + parser.add_argument( + "-path", dest="build_path", help="Path used to read a compile command database." + ) + if yaml: + parser.add_argument( + "-export-fixes", + metavar="FILE_OR_DIRECTORY", + dest="export_fixes", + help="A directory or a yaml file to store suggested fixes in, " + "which can be applied with clang-apply-replacements. If the " + "parameter is a directory, the fixes of each compilation unit are " + "stored in individual yaml files in the directory.", + ) + else: + parser.add_argument( + "-export-fixes", + metavar="DIRECTORY", + dest="export_fixes", + help="A directory to store suggested fixes in, which can be applied " + "with clang-apply-replacements. The fixes of each compilation unit are " + "stored in individual yaml files in the directory.", + ) + parser.add_argument( + "-extra-arg", + dest="extra_arg", + action="append", + default=[], + help="Additional argument to append to the compiler " "command line.", + ) + parser.add_argument( + "-extra-arg-before", + dest="extra_arg_before", + action="append", + default=[], + help="Additional argument to prepend to the compiler " "command line.", + ) + parser.add_argument( + "-removed-arg", + dest="removed_arg", + action="append", + default=[], + help="Arguments to remove from the compiler command line.", + ) + parser.add_argument( + "-quiet", + action="store_true", + default=False, + help="Run clang-tidy in quiet mode", + ) + parser.add_argument( + "-load", + dest="plugins", + action="append", + default=[], + help="Load the specified plugin in clang-tidy.", + ) + parser.add_argument( + "-allow-no-checks", + action="store_true", + help="Allow empty enabled checks.", + ) + parser.add_argument( + "-only-check-in-db", + dest="skip_non_compiling", + default=False, + action="store_true", + help="Only check files in the compilation database", + ) + parser.add_argument( + "-warnings-as-errors", + help="Upgrades clang-tidy warnings to errors. Same format as '-checks'.", + default="", + ) + parser.add_argument( + "-hide-progress", + action="store_true", + help="Hide progress", + ) + + clang_tidy_args = [] + argv = sys.argv[1:] + if "--" in argv: + clang_tidy_args.extend(argv[argv.index("--") :]) + argv = argv[: argv.index("--")] + + args = parser.parse_args(argv) + + compiling_files = get_compiling_files(args) if args.skip_non_compiling else None + + # Extract changed lines for each file. + filename = None + lines_by_file = {} + for line in sys.stdin: + match = re.search(r'^\+\+\+\ "?(.*?/){%s}([^ \t\n"]*)' % args.p, line) + if match: + filename = match.group(2) + if filename is None: + continue + + if args.regex is not None: + if not re.match("^%s$" % args.regex, filename): + continue + else: + if not re.match("^%s$" % args.iregex, filename, re.IGNORECASE): + continue + + # Skip any files not in the compiling list + if ( + compiling_files is not None + and (Path.cwd() / filename) not in compiling_files + ): + continue + + match = re.search(r"^@@.*\+(\d+)(,(\d+))?", line) + if match: + start_line = int(match.group(1)) + line_count = 1 + if match.group(3): + line_count = int(match.group(3)) + if line_count == 0: + continue + end_line = start_line + line_count - 1 + lines_by_file.setdefault(filename, []).append([start_line, end_line]) + + if not any(lines_by_file): + print("No relevant changes found.") + sys.exit(0) + + max_task_count = args.j + if max_task_count == 0: + max_task_count = multiprocessing.cpu_count() + max_task_count = min(len(lines_by_file), max_task_count) + if not args.hide_progress: + print(f"Running clang-tidy in {max_task_count} threads...") + + combine_fixes = False + export_fixes_dir = None + delete_fixes_dir = False + if args.export_fixes is not None: + # if a directory is given, create it if it does not exist + if args.export_fixes.endswith(os.path.sep) and not os.path.isdir( + args.export_fixes + ): + os.makedirs(args.export_fixes) + + if not os.path.isdir(args.export_fixes): + if not yaml: + raise RuntimeError( + "Cannot combine fixes in one yaml file. Either install PyYAML or specify an output directory." + ) + + combine_fixes = True + + if os.path.isdir(args.export_fixes): + export_fixes_dir = args.export_fixes + + if combine_fixes: + export_fixes_dir = tempfile.mkdtemp() + delete_fixes_dir = True + + # Tasks for clang-tidy. + task_queue = queue.Queue(max_task_count) + # A lock for console output. + lock = threading.Lock() + + # List of files with a non-zero return code. + failed_files = [] + + # Run a pool of clang-tidy workers. + start_workers( + max_task_count, run_tidy, (task_queue, lock, args.timeout, failed_files) + ) + + # Form the common args list. + common_clang_tidy_args = [] + if args.fix: + common_clang_tidy_args.append("-fix") + if args.checks != "": + common_clang_tidy_args.append("-checks=" + args.checks) + if args.config_file != "": + common_clang_tidy_args.append("-config-file=" + args.config_file) + if args.quiet: + common_clang_tidy_args.append("-quiet") + if args.build_path is not None: + common_clang_tidy_args.append("-p=%s" % args.build_path) + if args.use_color: + common_clang_tidy_args.append("--use-color") + if args.allow_no_checks: + common_clang_tidy_args.append("--allow-no-checks") + for arg in args.extra_arg: + common_clang_tidy_args.append("-extra-arg=%s" % arg) + for arg in args.extra_arg_before: + common_clang_tidy_args.append("-extra-arg-before=%s" % arg) + for arg in args.removed_arg: + common_clang_tidy_args.append("-removed-arg=%s" % arg) + for plugin in args.plugins: + common_clang_tidy_args.append("-load=%s" % plugin) + if args.warnings_as_errors: + common_clang_tidy_args.append("-warnings-as-errors=" + args.warnings_as_errors) + + for name in lines_by_file: + line_filter_json = json.dumps( + [{"name": name, "lines": lines_by_file[name]}], separators=(",", ":") + ) + + # Run clang-tidy on files containing changes. + command = [args.clang_tidy_binary] + command.append("-line-filter=" + line_filter_json) + if args.export_fixes is not None: + # Get a temporary file. We immediately close the handle so clang-tidy can + # overwrite it. + (handle, tmp_name) = tempfile.mkstemp(suffix=".yaml", dir=export_fixes_dir) + os.close(handle) + command.append("-export-fixes=" + tmp_name) + command.extend(common_clang_tidy_args) + command.append(name) + command.extend(clang_tidy_args) + + task_queue.put(command) + + # Application return code + return_code = 0 + + # Wait for all threads to be done. + task_queue.join() + # Application return code + return_code = 0 + if failed_files: + return_code = 1 + + if combine_fixes: + if not args.hide_progress: + print(f"Writing fixes to {args.export_fixes} ...") + try: + merge_replacement_files(export_fixes_dir, args.export_fixes) + except Exception: + sys.stderr.write("Error exporting fixes.\n") + traceback.print_exc() + return_code = 1 + + if delete_fixes_dir: + shutil.rmtree(export_fixes_dir) + sys.exit(return_code) + + +if __name__ == "__main__": + main() diff --git a/rcc/utils/clang-tidy/run-clang-tidy.py b/rcc/utils/clang-tidy/run-clang-tidy.py new file mode 100755 index 0000000..f827ef4 --- /dev/null +++ b/rcc/utils/clang-tidy/run-clang-tidy.py @@ -0,0 +1,796 @@ +#!/usr/bin/env python3 +# +# ===-----------------------------------------------------------------------===# +# +# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +# See https://llvm.org/LICENSE.txt for license information. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# +# ===-----------------------------------------------------------------------===# +# FIXME: Integrate with clang-tidy-diff.py + + +""" +Parallel clang-tidy runner +========================== + +Runs clang-tidy over all files in a compilation database. Requires clang-tidy +and clang-apply-replacements in $PATH. + +Example invocations. +- Run clang-tidy on all files in the current working directory with a default + set of checks and show warnings in the cpp files and all project headers. + run-clang-tidy.py $PWD + +- Fix all header guards. + run-clang-tidy.py -fix -checks=-*,llvm-header-guard + +- Fix all header guards included from clang-tidy and header guards + for clang-tidy headers. + run-clang-tidy.py -fix -checks=-*,llvm-header-guard extra/clang-tidy \ + -header-filter=extra/clang-tidy + +Compilation database setup: +https://clang.llvm.org/docs/HowToSetupToolingForLLVM.html +""" + +import argparse +import asyncio +from dataclasses import dataclass +import glob +import json +import multiprocessing +import os +import re +import shlex +import shutil +import subprocess +import sys +import tempfile +import time +import traceback +from types import ModuleType +from typing import Any, Awaitable, Callable, Dict, List, Optional, Tuple, TypeVar + + +yaml: Optional[ModuleType] = None +try: + import yaml +except ImportError: + yaml = None + + +def strtobool(val: str) -> bool: + """Convert a string representation of truth to a bool following LLVM's CLI argument parsing.""" + + val = val.lower() + if val in ["", "true", "1"]: + return True + elif val in ["false", "0"]: + return False + + # Return ArgumentTypeError so that argparse does not substitute its own error message + raise argparse.ArgumentTypeError( + f"'{val}' is invalid value for boolean argument! Try 0 or 1." + ) + + +def find_compilation_database(path: str) -> str: + """Adjusts the directory until a compilation database is found.""" + result = os.path.realpath("./") + while not os.path.isfile(os.path.join(result, path)): + parent = os.path.dirname(result) + if result == parent: + print("Error: could not find compilation database.") + sys.exit(1) + result = parent + return result + + +def get_tidy_invocation( + f: Optional[str], + clang_tidy_binary: str, + checks: str, + tmpdir: Optional[str], + build_path: str, + header_filter: Optional[str], + allow_enabling_alpha_checkers: bool, + extra_arg: List[str], + extra_arg_before: List[str], + removed_arg: List[str], + quiet: bool, + config_file_path: str, + config: str, + line_filter: Optional[str], + use_color: bool, + plugins: List[str], + warnings_as_errors: Optional[str], + exclude_header_filter: Optional[str], + allow_no_checks: bool, + store_check_profile: Optional[str], +) -> List[str]: + """Gets a command line for clang-tidy.""" + start = [clang_tidy_binary] + if allow_enabling_alpha_checkers: + start.append("-allow-enabling-analyzer-alpha-checkers") + if exclude_header_filter is not None: + start.append(f"--exclude-header-filter={exclude_header_filter}") + if header_filter is not None: + start.append(f"-header-filter={header_filter}") + if line_filter is not None: + start.append(f"-line-filter={line_filter}") + if use_color is not None: + if use_color: + start.append("--use-color") + else: + start.append("--use-color=false") + if checks: + start.append(f"-checks={checks}") + if tmpdir is not None: + start.append("-export-fixes") + # Get a temporary file. We immediately close the handle so clang-tidy can + # overwrite it. + (handle, name) = tempfile.mkstemp(suffix=".yaml", dir=tmpdir) + os.close(handle) + start.append(name) + for arg in extra_arg: + start.append(f"-extra-arg={arg}") + for arg in extra_arg_before: + start.append(f"-extra-arg-before={arg}") + for arg in removed_arg: + start.append(f"-removed-arg={arg}") + start.append(f"-p={build_path}") + if quiet: + start.append("-quiet") + if config_file_path: + start.append(f"--config-file={config_file_path}") + elif config: + start.append(f"-config={config}") + for plugin in plugins: + start.append(f"-load={plugin}") + if warnings_as_errors: + start.append(f"--warnings-as-errors={warnings_as_errors}") + if allow_no_checks: + start.append("--allow-no-checks") + if store_check_profile: + start.append("--enable-check-profile") + start.append(f"--store-check-profile={store_check_profile}") + if f: + start.append(f) + return start + + +def merge_replacement_files(tmpdir: str, mergefile: str) -> None: + """Merge all replacement files in a directory into a single file""" + assert yaml + # The fixes suggested by clang-tidy >= 4.0.0 are given under + # the top level key 'Diagnostics' in the output yaml files + mergekey = "Diagnostics" + merged = [] + for replacefile in glob.iglob(os.path.join(tmpdir, "*.yaml")): + content = yaml.safe_load(open(replacefile, "r")) + if not content: + continue # Skip empty files. + merged.extend(content.get(mergekey, [])) + + if merged: + # MainSourceFile: The key is required by the definition inside + # include/clang/Tooling/ReplacementsYaml.h, but the value + # is actually never used inside clang-apply-replacements, + # so we set it to '' here. + output = {"MainSourceFile": "", mergekey: merged} + with open(mergefile, "w") as out: + yaml.safe_dump(output, out) + else: + # Empty the file: + open(mergefile, "w").close() + + +def aggregate_profiles(profile_dir: str) -> Dict[str, float]: + """Aggregate timing data from multiple profile JSON files""" + aggregated: Dict[str, float] = {} + + for profile_file in glob.iglob(os.path.join(profile_dir, "*.json")): + try: + with open(profile_file, "r", encoding="utf-8") as f: + data = json.load(f) + profile_data: Dict[str, float] = data.get("profile", {}) + + for key, value in profile_data.items(): + if key.startswith("time.clang-tidy."): + if key in aggregated: + aggregated[key] += value + else: + aggregated[key] = value + except (json.JSONDecodeError, KeyError, IOError) as e: + print(f"Error: invalid json file {profile_file}: {e}", file=sys.stderr) + continue + + return aggregated + + +def print_profile_data(aggregated_data: Dict[str, float]) -> None: + """Print aggregated checks profile data in the same format as clang-tidy""" + if not aggregated_data: + return + + # Extract checker names and their timing data + checkers: Dict[str, Dict[str, float]] = {} + for key, value in aggregated_data.items(): + parts = key.split(".") + if len(parts) >= 4 and parts[0] == "time" and parts[1] == "clang-tidy": + checker_name = ".".join( + parts[2:-1] + ) # Everything between "clang-tidy" and the timing type + timing_type = parts[-1] # wall, user, or sys + + if checker_name not in checkers: + checkers[checker_name] = {"wall": 0.0, "user": 0.0, "sys": 0.0} + + checkers[checker_name][timing_type] = value + + if not checkers: + return + + total_user = sum(data["user"] for data in checkers.values()) + total_sys = sum(data["sys"] for data in checkers.values()) + total_wall = sum(data["wall"] for data in checkers.values()) + + sorted_checkers: List[Tuple[str, Dict[str, float]]] = sorted( + checkers.items(), key=lambda x: x[1]["user"] + x[1]["sys"], reverse=True + ) + + def print_stderr(*args: Any, **kwargs: Any) -> None: + print(*args, file=sys.stderr, **kwargs) + + print_stderr( + "===-------------------------------------------------------------------------===" + ) + print_stderr(" clang-tidy checks profiling") + print_stderr( + "===-------------------------------------------------------------------------===" + ) + print_stderr( + f" Total Execution Time: {total_user + total_sys:.4f} seconds ({total_wall:.4f} wall clock)\n" + ) + + # Calculate field widths based on the Total line which has the largest values + total_combined = total_user + total_sys + user_width = len(f"{total_user:.4f}") + sys_width = len(f"{total_sys:.4f}") + combined_width = len(f"{total_combined:.4f}") + wall_width = len(f"{total_wall:.4f}") + + # Header with proper alignment + additional_width = 9 # for " (100.0%)" + user_header = "---User Time---".center(user_width + additional_width) + sys_header = "--System Time--".center(sys_width + additional_width) + combined_header = "--User+System--".center(combined_width + additional_width) + wall_header = "---Wall Time---".center(wall_width + additional_width) + + print_stderr( + f" {user_header} {sys_header} {combined_header} {wall_header} --- Name ---" + ) + + for checker_name, data in sorted_checkers: + user_time = data["user"] + sys_time = data["sys"] + wall_time = data["wall"] + combined_time = user_time + sys_time + + user_percent = (user_time / total_user * 100) if total_user > 0 else 0 + sys_percent = (sys_time / total_sys * 100) if total_sys > 0 else 0 + combined_percent = ( + (combined_time / total_combined * 100) if total_combined > 0 else 0 + ) + wall_percent = (wall_time / total_wall * 100) if total_wall > 0 else 0 + + user_str = f"{user_time:{user_width}.4f} ({user_percent:5.1f}%)" + sys_str = f"{sys_time:{sys_width}.4f} ({sys_percent:5.1f}%)" + combined_str = f"{combined_time:{combined_width}.4f} ({combined_percent:5.1f}%)" + wall_str = f"{wall_time:{wall_width}.4f} ({wall_percent:5.1f}%)" + + print_stderr( + f" {user_str} {sys_str} {combined_str} {wall_str} {checker_name}" + ) + + user_total_str = f"{total_user:{user_width}.4f} (100.0%)" + sys_total_str = f"{total_sys:{sys_width}.4f} (100.0%)" + combined_total_str = f"{total_combined:{combined_width}.4f} (100.0%)" + wall_total_str = f"{total_wall:{wall_width}.4f} (100.0%)" + + print_stderr( + f" {user_total_str} {sys_total_str} {combined_total_str} {wall_total_str} Total" + ) + + +def find_binary(arg: str, name: str, build_path: str) -> str: + """Get the path for a binary or exit""" + if arg: + if shutil.which(arg): + return arg + else: + raise SystemExit( + f"error: passed binary '{arg}' was not found or is not executable" + ) + + built_path = os.path.join(build_path, "bin", name) + binary = shutil.which(name) or shutil.which(built_path) + if binary: + return binary + else: + raise SystemExit(f"error: failed to find {name} in $PATH or at {built_path}") + + +def apply_fixes( + args: argparse.Namespace, clang_apply_replacements_binary: str, tmpdir: str +) -> None: + """Calls clang-apply-fixes on a given directory.""" + invocation = [clang_apply_replacements_binary] + invocation.append("-ignore-insert-conflict") + if args.format: + invocation.append("-format") + if args.style: + invocation.append(f"-style={args.style}") + invocation.append(tmpdir) + subprocess.call(invocation) + + +# FIXME Python 3.12: This can be simplified out with run_with_semaphore[T](...). +T = TypeVar("T") + + +async def run_with_semaphore( + semaphore: asyncio.Semaphore, + f: Callable[..., Awaitable[T]], + *args: Any, + **kwargs: Any, +) -> T: + async with semaphore: + return await f(*args, **kwargs) + + +@dataclass +class ClangTidyResult: + filename: str + invocation: List[str] + returncode: int + stdout: str + stderr: str + elapsed: float + + +async def run_tidy( + args: argparse.Namespace, + name: str, + clang_tidy_binary: str, + tmpdir: str, + build_path: str, + store_check_profile: Optional[str], +) -> ClangTidyResult: + """ + Runs clang-tidy on a single file and returns the result. + """ + invocation = get_tidy_invocation( + name, + clang_tidy_binary, + args.checks, + tmpdir, + build_path, + args.header_filter, + args.allow_enabling_alpha_checkers, + args.extra_arg, + args.extra_arg_before, + args.removed_arg, + args.quiet, + args.config_file, + args.config, + args.line_filter, + args.use_color, + args.plugins, + args.warnings_as_errors, + args.exclude_header_filter, + args.allow_no_checks, + store_check_profile, + ) + + try: + process = await asyncio.create_subprocess_exec( + *invocation, stdout=subprocess.PIPE, stderr=subprocess.PIPE + ) + start = time.time() + stdout, stderr = await process.communicate() + end = time.time() + except asyncio.CancelledError: + process.terminate() + await process.wait() + raise + + assert process.returncode is not None + return ClangTidyResult( + name, + invocation, + process.returncode, + stdout.decode("UTF-8"), + stderr.decode("UTF-8"), + end - start, + ) + + +async def main() -> None: + parser = argparse.ArgumentParser( + description="Runs clang-tidy over all files " + "in a compilation database. Requires " + "clang-tidy and clang-apply-replacements in " + "$PATH or in your build directory." + ) + parser.add_argument( + "-allow-enabling-alpha-checkers", + action="store_true", + help="Allow alpha checkers from clang-analyzer.", + ) + parser.add_argument( + "-clang-tidy-binary", metavar="PATH", help="Path to clang-tidy binary." + ) + parser.add_argument( + "-clang-apply-replacements-binary", + metavar="PATH", + help="Path to clang-apply-replacements binary.", + ) + parser.add_argument( + "-checks", + default=None, + help="Checks filter, when not specified, use clang-tidy default.", + ) + config_group = parser.add_mutually_exclusive_group() + config_group.add_argument( + "-config", + default=None, + help="Specifies a configuration in YAML/JSON format: " + " -config=\"{Checks: '*', " + ' CheckOptions: {x: y}}" ' + "When the value is empty, clang-tidy will " + "attempt to find a file named .clang-tidy for " + "each source file in its parent directories.", + ) + config_group.add_argument( + "-config-file", + default=None, + help="Specify the path of .clang-tidy or custom config " + "file: e.g. -config-file=/some/path/myTidyConfigFile. " + "This option internally works exactly the same way as " + "-config option after reading specified config file. " + "Use either -config-file or -config, not both.", + ) + parser.add_argument( + "-exclude-header-filter", + default=None, + help="Regular expression matching the names of the " + "headers to exclude diagnostics from. Diagnostics from " + "the main file of each translation unit are always " + "displayed.", + ) + parser.add_argument( + "-header-filter", + default=None, + help="Regular expression matching the names of the " + "headers to output diagnostics from. Diagnostics from " + "the main file of each translation unit are always " + "displayed.", + ) + parser.add_argument( + "-source-filter", + default=None, + help="Regular expression matching the names of the " + "source files from compilation database to output " + "diagnostics from.", + ) + parser.add_argument( + "-line-filter", + default=None, + help="List of files and line ranges to output diagnostics from.", + ) + if yaml: + parser.add_argument( + "-export-fixes", + metavar="file_or_directory", + dest="export_fixes", + help="A directory or a yaml file to store suggested fixes in, " + "which can be applied with clang-apply-replacements. If the " + "parameter is a directory, the fixes of each compilation unit are " + "stored in individual yaml files in the directory.", + ) + else: + parser.add_argument( + "-export-fixes", + metavar="directory", + dest="export_fixes", + help="A directory to store suggested fixes in, which can be applied " + "with clang-apply-replacements. The fixes of each compilation unit are " + "stored in individual yaml files in the directory.", + ) + parser.add_argument( + "-j", + type=int, + default=0, + help="Number of tidy instances to be run in parallel.", + ) + parser.add_argument( + "files", + nargs="*", + default=[".*"], + help="Files to be processed (regex on path).", + ) + parser.add_argument("-fix", action="store_true", help="apply fix-its.") + parser.add_argument( + "-format", action="store_true", help="Reformat code after applying fixes." + ) + parser.add_argument( + "-style", + default="file", + help="The style of reformat code after applying fixes.", + ) + parser.add_argument( + "-use-color", + type=strtobool, + nargs="?", + const=True, + help="Use colors in diagnostics, overriding clang-tidy's" + " default behavior. This option overrides the 'UseColor" + "' option in .clang-tidy file, if any.", + ) + parser.add_argument( + "-p", dest="build_path", help="Path used to read a compile command database." + ) + parser.add_argument( + "-extra-arg", + dest="extra_arg", + action="append", + default=[], + help="Additional argument to append to the compiler command line.", + ) + parser.add_argument( + "-extra-arg-before", + dest="extra_arg_before", + action="append", + default=[], + help="Additional argument to prepend to the compiler command line.", + ) + parser.add_argument( + "-removed-arg", + dest="removed_arg", + action="append", + default=[], + help="Arguments to remove from the compiler command line.", + ) + parser.add_argument( + "-quiet", action="store_true", help="Run clang-tidy in quiet mode." + ) + parser.add_argument( + "-load", + dest="plugins", + action="append", + default=[], + help="Load the specified plugin in clang-tidy.", + ) + parser.add_argument( + "-warnings-as-errors", + default=None, + help="Upgrades warnings to errors. Same format as '-checks'.", + ) + parser.add_argument( + "-allow-no-checks", + action="store_true", + help="Allow empty enabled checks.", + ) + parser.add_argument( + "-enable-check-profile", + action="store_true", + help="Enable per-check timing profiles, and print a report", + ) + parser.add_argument( + "-hide-progress", + action="store_true", + help="Hide progress", + ) + args = parser.parse_args() + + db_path = "compile_commands.json" + + if args.build_path is not None: + build_path = args.build_path + else: + # Find our database + build_path = find_compilation_database(db_path) + + clang_tidy_binary = find_binary(args.clang_tidy_binary, "clang-tidy", build_path) + + if args.fix: + clang_apply_replacements_binary = find_binary( + args.clang_apply_replacements_binary, "clang-apply-replacements", build_path + ) + + combine_fixes = False + export_fixes_dir: Optional[str] = None + delete_fixes_dir = False + if args.export_fixes is not None: + # if a directory is given, create it if it does not exist + if args.export_fixes.endswith(os.path.sep) and not os.path.isdir( + args.export_fixes + ): + os.makedirs(args.export_fixes) + + if not os.path.isdir(args.export_fixes): + if not yaml: + raise RuntimeError( + "Cannot combine fixes in one yaml file. Either install PyYAML or specify an output directory." + ) + + combine_fixes = True + + if os.path.isdir(args.export_fixes): + export_fixes_dir = args.export_fixes + + if export_fixes_dir is None and (args.fix or combine_fixes): + export_fixes_dir = tempfile.mkdtemp() + delete_fixes_dir = True + + profile_dir: Optional[str] = None + if args.enable_check_profile: + profile_dir = tempfile.mkdtemp() + + try: + invocation = get_tidy_invocation( + None, + clang_tidy_binary, + args.checks, + None, + build_path, + args.header_filter, + args.allow_enabling_alpha_checkers, + args.extra_arg, + args.extra_arg_before, + args.removed_arg, + args.quiet, + args.config_file, + args.config, + args.line_filter, + args.use_color, + args.plugins, + args.warnings_as_errors, + args.exclude_header_filter, + args.allow_no_checks, + None, # No profiling for the list-checks invocation + ) + invocation.append("-list-checks") + invocation.append("-") + # Even with -quiet we still want to check if we can call clang-tidy. + subprocess.check_call( + invocation, stdout=subprocess.DEVNULL if args.quiet else None + ) + except Exception: + print("Unable to run clang-tidy.", file=sys.stderr) + sys.exit(1) + + # Load the database and extract all files. + with open(os.path.join(build_path, db_path)) as f: + database = json.load(f) + files = {os.path.abspath(os.path.join(e["directory"], e["file"])) for e in database} + number_files_in_database = len(files) + + # Filter source files from compilation database. + if args.source_filter: + try: + source_filter_re = re.compile(args.source_filter) + except Exception: + print( + "Error: unable to compile regex from arg -source-filter:", + file=sys.stderr, + ) + traceback.print_exc() + sys.exit(1) + files = {f for f in files if source_filter_re.match(f)} + + max_task = args.j + if max_task == 0: + max_task = multiprocessing.cpu_count() + + # Build up a big regexy filter from all command line arguments. + file_name_re = re.compile("|".join(args.files)) + files = {f for f in files if file_name_re.search(f)} + + if not args.hide_progress: + print( + f"Running clang-tidy in {max_task} threads for {len(files)} files " + f"out of {number_files_in_database} in compilation database ..." + ) + + returncode = 0 + semaphore = asyncio.Semaphore(max_task) + tasks = [ + asyncio.create_task( + run_with_semaphore( + semaphore, + run_tidy, + args, + f, + clang_tidy_binary, + export_fixes_dir, + build_path, + profile_dir, + ) + ) + for f in files + ] + + try: + for i, coro in enumerate(asyncio.as_completed(tasks)): + result = await coro + if result.returncode != 0: + returncode = 1 + if result.returncode < 0: + result.stderr += f"{result.filename}: terminated by signal {-result.returncode}\n" + progress = f"[{i + 1: >{len(f'{len(files)}')}}/{len(files)}]" + runtime = f"[{result.elapsed:.1f}s]" + if not args.hide_progress: + print(f"{progress}{runtime} {shlex.join(result.invocation)}") + if result.stdout: + print(result.stdout, end=("" if result.stderr else "\n")) + if result.stderr: + print(result.stderr) + except asyncio.CancelledError: + if not args.hide_progress: + print("\nCtrl-C detected, goodbye.") + for task in tasks: + task.cancel() + if delete_fixes_dir: + assert export_fixes_dir + shutil.rmtree(export_fixes_dir) + if profile_dir: + shutil.rmtree(profile_dir) + return + + if args.enable_check_profile and profile_dir: + # Ensure all clang-tidy stdout is flushed before printing profiling + sys.stdout.flush() + aggregated_data = aggregate_profiles(profile_dir) + if aggregated_data: + print_profile_data(aggregated_data) + else: + print("No profiling data found.") + + if combine_fixes: + if not args.hide_progress: + print(f"Writing fixes to {args.export_fixes} ...") + try: + assert export_fixes_dir + merge_replacement_files(export_fixes_dir, args.export_fixes) + except Exception: + print("Error exporting fixes.\n", file=sys.stderr) + traceback.print_exc() + returncode = 1 + + if args.fix: + if not args.hide_progress: + print("Applying fixes ...") + try: + assert export_fixes_dir + apply_fixes(args, clang_apply_replacements_binary, export_fixes_dir) + except Exception: + print("Error applying fixes.\n", file=sys.stderr) + traceback.print_exc() + returncode = 1 + + if delete_fixes_dir: + assert export_fixes_dir + shutil.rmtree(export_fixes_dir) + if profile_dir: + shutil.rmtree(profile_dir) + sys.exit(returncode) + + +if __name__ == "__main__": + try: + asyncio.run(main()) + except KeyboardInterrupt: + pass diff --git a/rcc/utils/dev.py b/rcc/utils/dev.py deleted file mode 100644 index b14276a..0000000 --- a/rcc/utils/dev.py +++ /dev/null @@ -1,277 +0,0 @@ -#!/usr/bin/env python3 -# SPDX-License-Identifier: GPL-3.0-or-later - -"""Run formatting, clang-tidy, or the Clang analyzer for RCC and CFront.""" - -import argparse -import difflib -import hashlib -import json -from pathlib import Path -import re -import shlex -import subprocess -import sys - - -ROOT = Path(__file__).resolve().parents[2] -PROJECTS = ("rcc", "cfront") -SOURCE_SUFFIXES = {".c", ".cc", ".cpp", ".cxx", ".h", ".hh", ".hpp", ".hxx", ".inc"} -HEADER_SUFFIXES = {".h", ".hh", ".hpp", ".hxx", ".inc"} -HUNK = re.compile(r"^@@ -\d+(?:,\d+)? \+(\d+)(?:,(\d+))? @@", re.MULTILINE) - - -def git_output(*arguments): - result = subprocess.run( - ["git", "-C", str(ROOT), *arguments], - check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, - ) - return result.stdout - - -def is_source(path, build_dir): - # The allowlist prevents checks from reaching independent or legacy projects. - resolved = path.resolve() - if not any(resolved.is_relative_to(ROOT / project) for project in PROJECTS): - return False - if build_dir is not None and resolved.is_relative_to(build_dir.resolve()): - return False - relative = resolved.relative_to(ROOT) - for part in relative.parts[:-1]: - if part in {"generated", "CMakeFiles", "rcc-legacy"}: - return False - return path.suffix in SOURCE_SUFFIXES and path.is_file() - - -def git_files(*arguments): - names = git_output(*arguments, "--", *PROJECTS).split(b"\0") - return {ROOT / name.decode() for name in names if name} - - -def selected_files(args): - scopes = [Path(name).resolve() for name in args.paths] - for scope in scopes: - if scope != ROOT and not any(scope.is_relative_to(ROOT / p) for p in PROJECTS): - raise ValueError(f"Selection must be inside rcc/ or cfront/: {scope}") - - whole_files = args.all or (args.paths and not args.changed) - if whole_files: - candidates = git_files("ls-files", "--cached", "--others", "--exclude-standard", "-z") - untracked = set() - else: - candidates = git_files( - "diff", "--no-ext-diff", "--no-renames", "--name-only", "-z", args.base - ) - untracked = git_files("ls-files", "--others", "--exclude-standard", "-z") - candidates |= untracked - - selected = {} - for path in sorted(candidates): - if not is_source(path, args.build_dir): - continue - if scopes and not any(path == scope or path.is_relative_to(scope) for scope in scopes): - continue - if whole_files or path in untracked: - selected[path] = None - continue - patch = git_output( - "diff", "--no-color", "--no-ext-diff", "--no-textconv", "--no-renames", "--unified=0", - args.base, "--", str(path.relative_to(ROOT)), - ).decode() - ranges = [] - for start, count in HUNK.findall(patch): - length = int(count) if count else 1 - if length == 0: - continue - first = int(start) - ranges.append((first, first + length - 1)) - selected[path] = ranges - return selected - - -def check_tool(executable, expected_version, directory=ROOT): - result = subprocess.run( - [executable, "--version"], cwd=directory, - check=True, text=True, stdout=subprocess.PIPE, - ) - print(result.stdout.strip(), flush=True) - if expected_version: - version = re.search(r"\bversion\s+(\d+(?:\.\d+)*)", result.stdout) - if version is None or version.group(1) != expected_version: - raise ValueError(f"Expected {executable} version {expected_version}") - - -def format_sources(args): - files = selected_files(args) - if not files: - print("No matching source files.") - return 0 - check_tool(args.tool, args.tool_version) - different = False - for path, ranges in files.items(): - if ranges == []: - continue - command = [args.tool, "--style=file"] - if ranges is not None: - command.extend(f"--lines={start}:{end}" for start, end in ranges) - command.append(str(path)) - original = path.read_text() - formatted = subprocess.run( - command, check=True, text=True, stdout=subprocess.PIPE, - ).stdout - if original == formatted: - continue - different = True - name = str(path.relative_to(ROOT)) - if args.fix: - path.write_text(formatted) - print(f"Formatted {name}") - else: - sys.stdout.writelines(difflib.unified_diff( - original.splitlines(keepends=True), formatted.splitlines(keepends=True), - fromfile=f"a/{name}", tofile=f"b/{name}", - )) - return int(different and not args.fix) - - -def compilation_entries(args, files): - database_dir = args.build_dir.resolve() - with (database_dir / "compile_commands.json").open() as stream: - database = json.load(stream) - entries = [] - for entry in database: - directory = (database_dir / entry["directory"]).resolve() - source = (directory / entry["file"]).resolve() - if not is_source(source, args.build_dir): - continue - if source in files: - entries.append((source, directory, entry)) - return entries - - -def tidy_sources(args): - files = selected_files(args) - if not files: - print("No matching source files.") - return 0 - if args.all: - sources = {source for source, _, _ in compilation_entries(args, files)} - else: - # Like clang-tidy-diff, pass selected headers directly to clang-tidy. - # The tool infers their compile commands from the compilation database. - sources = set(files) - if not sources: - raise ValueError("No compilation database entries match the selected sources.") - check_tool(args.tool, args.tool_version) - line_filter = [] - if args.line_filter: - for path, ranges in files.items(): - if ranges == []: - continue - item = {"name": str(path)} - if ranges is not None: - item["lines"] = ranges - line_filter.append(item) - if not line_filter: - print("No changed lines to report.") - return 0 - - failed = False - for source in sorted(sources): - command = [args.tool, str(source), "-p", str(args.build_dir.resolve())] - if args.line_filter: - command.append("--line-filter=" + json.dumps(line_filter)) - command.extend(args.extra_arg) - print(shlex.join(command), flush=True) - result = subprocess.run(command, cwd=ROOT) - failed |= result.returncode != 0 - return int(failed) - - -def compilation_command(entry): - arguments = entry.get("arguments") - if arguments is None: - arguments = shlex.split(entry["command"]) - return arguments - - -def analyze_sources(args): - files = selected_files(args) - if not files: - print("No matching source files.") - return 0 - entries = compilation_entries(args, files) - if not entries: - raise ValueError( - "No selected translation units are in the compilation database. " - "Select source files or use --all to analyze all project translation units." - ) - if any(path.suffix in HEADER_SUFFIXES for path in files): - print("Header changes do not expand the selected translation units; use --all for full analysis.") - reports = (args.output_dir or args.build_dir / "analyzer-reports").resolve() - failed = False - check_tool(args.tool, args.tool_version) - for source, directory, entry in entries: - arguments = compilation_command(entry) - driver_index = 1 if Path(arguments[0]).name in {"ccache", "sccache"} else 0 - # Distinguish compilation variants of the same source file. - identity = json.dumps(entry, sort_keys=True).encode() - suffix = hashlib.sha256(identity).hexdigest()[:12] - output = reports / f"{source.stem}-{suffix}" - output.mkdir(parents=True, exist_ok=True) - # Run the selected Clang analyzer with the recorded compilation options. - # Appending -o selects the report directory instead of the object output. - command = [args.tool, *arguments[driver_index + 1:], - "--analyze", "--analyzer-output", "html", "-o", str(output)] - command.extend(args.extra_arg) - print(shlex.join(command), flush=True) - result = subprocess.run(command, cwd=directory) - failed |= result.returncode != 0 - print(f"Analyzer reports: {reports}") - return int(failed) - - -def main(): - parser = argparse.ArgumentParser(description=__doc__) - commands = parser.add_subparsers(dest="command", required=True) - for name, handler, executable in ( - ("format", format_sources, "clang-format"), - ("tidy", tidy_sources, "clang-tidy"), - ("analyze", analyze_sources, "clang++"), - ): - command = commands.add_parser(name) - command.set_defaults(handler=handler) - command.add_argument("paths", nargs="*", help="Files or module directories; paths select whole files by default") - mode = command.add_mutually_exclusive_group() - mode.add_argument("--all", action="store_true", help="Select all tracked and untracked source files") - mode.add_argument("--changed", action="store_true", help="Select changes since --base, also within explicit paths") - command.add_argument("--base", default="HEAD", help="Diff base for the working tree (default: HEAD)") - command.add_argument("--tool", default=executable, help="Tool executable or absolute path") - command.add_argument("--tool-version", default="", help="Require this exact tool version when provided") - if name == "format": - command.add_argument("--build-dir", type=Path, help="Build directory to exclude from source selection") - command.add_argument("--fix", action="store_true", help="Write formatting changes; otherwise only print a diff") - else: - command.add_argument("--build-dir", type=Path, required=True, help="Directory containing compile_commands.json") - command.add_argument("--extra-arg", action="append", default=[], help="Additional tool option; use --extra-arg=VALUE") - if name == "tidy": - command.add_argument("--line-filter", action="store_true", help="Report diagnostics on selected lines; translation units are still fully analyzed") - if name == "analyze": - command.description = "Analyze selected translation units with Clang using their recorded compilation options. Use --all for full analysis." - command.add_argument("--output-dir", type=Path, help="HTML report directory (default: BUILD/analyzer-reports)") - - args = parser.parse_args() - try: - return args.handler(args) - except subprocess.CalledProcessError as error: - if error.stderr: - sys.stderr.write(error.stderr.decode()) - print(f"Command failed: {shlex.join(error.cmd)}", file=sys.stderr) - return 1 - except (OSError, ValueError) as error: - print(str(error), file=sys.stderr) - return 1 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/rcc/utils/git/code-format-helper.py b/rcc/utils/git/code-format-helper.py new file mode 100755 index 0000000..e7a82b8 --- /dev/null +++ b/rcc/utils/git/code-format-helper.py @@ -0,0 +1,677 @@ +#!/usr/bin/env python3 +# +# ====- code-format-helper, runs code formatters from the ci or in a hook --*- python -*--==# +# +# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +# See https://llvm.org/LICENSE.txt for license information. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# +# ==--------------------------------------------------------------------------------------==# + +import argparse +import os +import re +import shlex +import subprocess +import sys +from typing import List, Optional + +""" +This script is run by GitHub actions to ensure that the code in PR's conform to +the coding style of LLVM. It can also be installed as a pre-commit git hook to +check the coding style before submitting it. The canonical source of this script +is in the LLVM source tree under llvm/utils/git. + +For C/C++ code it uses clang-format, for Python code it uses darker (which +in turn invokes black), and for LLVM-libc CMake files it uses cmake_format.py. + +You can learn more about the LLVM coding style on llvm.org: +https://llvm.org/docs/CodingStandards.html + +You can install this script as a git hook by symlinking it to the .git/hooks +directory: + +ln -s $(pwd)/llvm/utils/git/code-format-helper.py .git/hooks/pre-commit + +You can control the exact path to clang-format, darker, or cmake_format.py with +the following environment variables: $CLANG_FORMAT_PATH, $DARKER_FORMAT_PATH, +and $CMAKE_FORMAT_PATH. +""" + + +class FormatArgs: + start_rev: str = None + end_rev: str = None + repo: str = None + changed_files: List[str] = [] + token: str = None + verbose: bool = True + issue_number: int = 0 + write_comment_to_file: bool = False + + def __init__(self, args: argparse.Namespace = None) -> None: + if not args is None: + self.start_rev = args.start_rev + self.end_rev = args.end_rev + self.repo = args.repo + self.token = args.token + self.changed_files = args.changed_files + self.issue_number = args.issue_number + self.write_comment_to_file = args.write_comment_to_file + + +class FormatHelper: + COMMENT_TAG = "<!--LLVM CODE FORMAT COMMENT: {fmt}-->" + name: str + friendly_name: str + comment: dict = None + + @property + def comment_tag(self) -> str: + return self.COMMENT_TAG.replace("fmt", self.name) + + @property + def instructions(self) -> str: + raise NotImplementedError() + + def has_tool(self) -> bool: + raise NotImplementedError() + + def format_run(self, changed_files: List[str], args: FormatArgs) -> Optional[str]: + raise NotImplementedError() + + def pr_comment_text_for_diff(self, diff: str) -> str: + return f""" +:warning: {self.friendly_name}, {self.name} found issues in your code. :warning: + +<details> +<summary> +You can test this locally with the following command: +</summary> + +``````````bash +{self.instructions} +`````````` + +:warning: +The reproduction instructions above might return results for more than one PR +in a stack if you are using a stacked PR workflow. You can limit the results by +changing `origin/main` to the base branch/commit you want to compare against. +:warning: + +</details> + +<details> +<summary> +View the diff from {self.name} here. +</summary> + +``````````diff +{diff} +`````````` + +</details> +""" + + # TODO: any type should be replaced with the correct github type, but it requires refactoring to + # not require the github module to be installed everywhere. + def find_comment(self, pr: any) -> any: + for comment in pr.as_issue().get_comments(): + if self.comment_tag in comment.body: + return comment + return None + + def update_pr(self, comment_text: str, args: FormatArgs, create_new: bool) -> None: + import github + from github import IssueComment, PullRequest + + repo = github.Github(auth=github.Auth.Token(args.token)).get_repo(args.repo) + pr = repo.get_issue(args.issue_number).as_pull_request() + + comment_text = self.comment_tag + "\n\n" + comment_text + + existing_comment = self.find_comment(pr) + + if args.write_comment_to_file: + if create_new or existing_comment: + self.comment = {"body": comment_text} + if existing_comment: + self.comment["id"] = existing_comment.id + return + + if existing_comment: + existing_comment.edit(comment_text) + elif create_new: + pr.as_issue().create_comment(comment_text) + + def run(self, changed_files: List[str], args: FormatArgs) -> bool: + changed_files = [arg for arg in changed_files if "third-party" not in arg] + diff = self.format_run(changed_files, args) + should_update_gh = args.token is not None and args.repo is not None + + if diff is None: + if should_update_gh: + comment_text = ( + ":white_check_mark: With the latest revision " + f"this PR passed the {self.friendly_name}." + ) + self.update_pr(comment_text, args, create_new=False) + return True + elif len(diff) > 0: + if should_update_gh: + comment_text = self.pr_comment_text_for_diff(diff) + self.update_pr(comment_text, args, create_new=True) + else: + print( + f"Warning: {self.friendly_name}, {self.name} detected " + "some issues with your code formatting..." + ) + return False + else: + # The formatter failed but didn't output a diff (e.g. some sort of + # infrastructure failure). + comment_text = ( + f":warning: The {self.friendly_name} failed without printing " + "a diff. Check the logs for stderr output. :warning:" + ) + if should_update_gh: + self.update_pr(comment_text, args, create_new=False) + return False + + +class ClangFormatHelper(FormatHelper): + name = "clang-format" + friendly_name = "C/C++ code formatter" + + def _construct_command(self, diff_expression: list[str] | None): + cf_cmd = [self.clang_fmt_path, "--diff"] + + if diff_expression: + cf_cmd.extend(diff_expression) + + # Gather the extension of all modified files and pass them explicitly to git-clang-format. + # This prevents git-clang-format from applying its own filtering rules on top of ours. + extensions = set() + for file in self._cpp_files: + _, ext = os.path.splitext(file) + extensions.add( + ext.strip(".") + ) # Exclude periods since git-clang-format takes extensions without them + cf_cmd.append("--extensions") + cf_cmd.append(",".join(extensions)) + + cf_cmd.append("--") + cf_cmd += self._cpp_files + return cf_cmd + + @property + def instructions(self) -> str: + return ( + " ".join(self._construct_command(["origin/main", "HEAD"])) + + " --diff_from_common_commit" + ) + + def should_include_extensionless_file(self, path: str) -> bool: + return path.startswith("libcxx/include") + + def filter_changed_files(self, changed_files: List[str]) -> List[str]: + filtered_files = [] + for path in changed_files: + _, ext = os.path.splitext(path) + if ext in ( + ".cpp", + ".c", + ".h", + ".hpp", + ".hxx", + ".cxx", + ".inc", + ".cppm", + ".cl", + ): + filtered_files.append(path) + elif ext == "" and self.should_include_extensionless_file(path): + filtered_files.append(path) + return filtered_files + + @property + def clang_fmt_path(self) -> str: + if "CLANG_FORMAT_PATH" in os.environ: + return os.environ["CLANG_FORMAT_PATH"] + return "git-clang-format" + + def has_tool(self) -> bool: + cmd = [self.clang_fmt_path, "-h"] + proc = None + try: + proc = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + except: + return False + return proc.returncode == 0 + + def format_run(self, changed_files: List[str], args: FormatArgs) -> Optional[str]: + self._cpp_files = self.filter_changed_files(changed_files) + if not self._cpp_files: + return None + + diff_expression = [] + if args.start_rev and args.end_rev: + diff_expression.append(args.start_rev) + diff_expression.append(args.end_rev) + + cf_cmd = self._construct_command(diff_expression) + + if args.verbose: + print(f"Running: {' '.join(cf_cmd)}") + proc = subprocess.run(cf_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + sys.stdout.write(proc.stderr.decode("utf-8")) + + if proc.returncode != 0: + # formatting needed, or the command otherwise failed + if args.verbose: + print(f"error: {self.name} exited with code {proc.returncode}") + # Print the diff in the log so that it is viewable there + print(proc.stdout.decode("utf-8")) + return proc.stdout.decode("utf-8") + else: + return None + + +class DarkerFormatHelper(FormatHelper): + name = "darker" + friendly_name = "Python code formatter" + + def _construct_command(self, diff_expression: str | None) -> str: + darker_cmd = [ + self.darker_fmt_path, + "--check", + "--diff", + ] + if diff_expression: + darker_cmd += ["-r", diff_expression] + darker_cmd += self._py_files + return darker_cmd + + @property + def instructions(self) -> str: + return " ".join(self._construct_command("origin/main...HEAD")) + + def filter_changed_files(self, changed_files: List[str]) -> List[str]: + filtered_files = [] + for path in changed_files: + name, ext = os.path.splitext(path) + if ext == ".py": + filtered_files.append(path) + + return filtered_files + + @property + def darker_fmt_path(self) -> str: + if "DARKER_FORMAT_PATH" in os.environ: + return os.environ["DARKER_FORMAT_PATH"] + return "darker" + + def has_tool(self) -> bool: + cmd = [self.darker_fmt_path, "--version"] + proc = None + try: + proc = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + except: + return False + return proc.returncode == 0 + + def format_run(self, changed_files: List[str], args: FormatArgs) -> Optional[str]: + py_files = self.filter_changed_files(changed_files) + if not py_files: + return None + self._py_files = py_files + diff_expression = None + if args.start_rev and args.end_rev: + diff_expression = f"{args.start_rev}...{args.end_rev}" + darker_cmd = self._construct_command(diff_expression) + if args.verbose: + print(f"Running: {' '.join(darker_cmd)}") + proc = subprocess.run( + darker_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE + ) + if args.verbose: + sys.stdout.write(proc.stderr.decode("utf-8")) + + if proc.returncode != 0: + # formatting needed, or the command otherwise failed + if args.verbose: + print(f"error: {self.name} exited with code {proc.returncode}") + # Print the diff in the log so that it is viewable there + print(proc.stdout.decode("utf-8")) + return proc.stdout.decode("utf-8") + else: + sys.stdout.write(proc.stdout.decode("utf-8")) + return None + + +class UndefGetFormatHelper(FormatHelper): + name = "undef deprecator" + friendly_name = "undef deprecator" + + @property + def instructions(self) -> str: + return " ".join(shlex.quote(c) for c in self.cmd) + + def filter_changed_files(self, changed_files: List[str]) -> List[str]: + filtered_files = [] + for path in changed_files: + _, ext = os.path.splitext(path) + if ext in (".cpp", ".c", ".h", ".hpp", ".hxx", ".cxx", ".inc", ".cppm", ".ll"): + filtered_files.append(path) + return filtered_files + + def has_tool(self) -> bool: + return True + + def pr_comment_text_for_diff(self, diff: str) -> str: + return f""" +:warning: {self.name} found issues in your code. :warning: + +<details> +<summary> +You can test this locally with the following command: +</summary> + +``````````bash +{self.instructions} +`````````` + +</details> + +{diff} +""" + + def format_run(self, changed_files: List[str], args: FormatArgs) -> Optional[str]: + files = self.filter_changed_files(changed_files) + if not files: + return None + + # Use git to find files that have had a change in the number of undefs + regex = "([^a-zA-Z0-9#_-]undef([^a-zA-Z0-9_-]|$)|UndefValue::get)" + cmd = ["git", "diff", "-U0", "--pickaxe-regex", "-S", regex] + + if args.start_rev and args.end_rev: + cmd.append(args.start_rev) + cmd.append(args.end_rev) + + cmd += files + self.cmd = cmd + + if args.verbose: + print(f"Running: {self.instructions}") + + proc = subprocess.run( + cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, encoding="utf-8" + ) + sys.stdout.write(proc.stderr) + stdout = proc.stdout + + if not stdout: + return None + + files = [] + + # Split the diff so we have one array entry per file. + # Each file is prefixed like: + # diff --git a/file b/file + for file in re.split("^diff --git ", stdout, 0, re.MULTILINE): + lines = file.splitlines() + match = re.match("a/([^ ]+)", lines[0] if lines else "") + filename = match[1] if match else "" + if filename.endswith(".ll"): + undef_regex = r"^[+][^;\n]*(?<!%)\bundef\b" + else: + undef_regex = r"^[+].*UndefValue::get" + # search for additions of undef + if re.search(undef_regex, file, re.MULTILINE): + files.append(filename) + + if not files: + return None + + files = "\n".join(" - " + f for f in files) + report = f""" +The following files introduce new uses of undef: +{files} + +[Undef](https://llvm.org/docs/LangRef.html#undefined-values) is now deprecated and should only be used in the rare cases where no replacement is possible. For example, a load of uninitialized memory yields `undef`. You should use `poison` values for placeholders instead. + +In tests, avoid using `undef` and having tests that trigger undefined behavior. If you need an operand with some unimportant value, you can add a new argument to the function and use that instead. + +For example, this is considered a bad practice: +```llvm +define void @fn() {{ + ... + br i1 undef, ... +}} +``` + +Please use the following instead: +```llvm +define void @fn(i1 %cond) {{ + ... + br i1 %cond, ... +}} +``` + +Please refer to the [Undefined Behavior Manual](https://llvm.org/docs/UndefinedBehavior.html) for more information. +""" + if args.verbose: + print(f"error: {self.name} failed") + print(report) + return report + + +class CMakeFormatHelper(FormatHelper): + name = "cmake-format" + friendly_name = "CMake code formatter" + + def _construct_command(self) -> List[str]: + return [ + sys.executable, + self.cmake_fmt_path, + "--dry-run", + "--diff", + ] + self._cmake_files + + @property + def instructions(self) -> str: + return " ".join(["python3", self.cmake_fmt_path, "-i"] + self._cmake_files) + + @property + def check_instructions(self) -> str: + return " ".join( + ["python3", self.cmake_fmt_path, "--dry-run", "--diff"] + self._cmake_files + ) + + def pr_comment_text_for_diff(self, diff: str) -> str: + return f""" +:warning: {self.friendly_name}, {self.name} found issues in your code. :warning: + +<details> +<summary> +You can fix this locally with the following command: +</summary> + +``````````bash +{self.instructions} +`````````` + +You can also preview the formatting diff without modifying files: + +``````````bash +{self.check_instructions} +`````````` + +</details> + +<details> +<summary> +View the diff from {self.name} here. +</summary> + +``````````diff +{diff} +`````````` + +</details> +""" + + def filter_changed_files(self, changed_files: List[str]) -> List[str]: + filtered_files = [] + for path in changed_files: + if not path.startswith("libc/"): + continue + if os.path.basename(path) == "CMakeLists.txt" or path.endswith(".cmake"): + if os.path.exists(path): + filtered_files.append(path) + return filtered_files + + @property + def cmake_fmt_path(self) -> str: + if "CMAKE_FORMAT_PATH" in os.environ: + return os.environ["CMAKE_FORMAT_PATH"] + return "libc/utils/cmake_format.py" + + def has_tool(self) -> bool: + return os.path.isfile(self.cmake_fmt_path) + + def format_run(self, changed_files: List[str], args: FormatArgs) -> Optional[str]: + cmake_files = self.filter_changed_files(changed_files) + if not cmake_files: + return None + self._cmake_files = cmake_files + cmake_cmd = self._construct_command() + if args.verbose: + print(f"Running: {' '.join(cmake_cmd)}") + proc = subprocess.run(cmake_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + if args.verbose: + sys.stdout.write(proc.stderr.decode("utf-8")) + + if proc.returncode != 0: + # formatting needed, or the command otherwise failed + stdout = proc.stdout.decode("utf-8") + if args.verbose: + print(f"error: {self.name} exited with code {proc.returncode}") + if stdout: + print( + "error: CMake files are not formatted according to " + "LLVM-libc style." + ) + print(f"To fix this locally, run:\n {self.instructions}") + print( + "To preview the formatting diff locally, run:\n" + f" {self.check_instructions}" + ) + # Print the diff in the log so that it is viewable there + print(stdout) + elif stdout: + print(f"To fix CMake formatting, run:\n {self.instructions}") + return stdout + else: + sys.stdout.write(proc.stdout.decode("utf-8")) + return None + + +ALL_FORMATTERS = ( + DarkerFormatHelper(), + ClangFormatHelper(), + CMakeFormatHelper(), + UndefGetFormatHelper(), +) + + +def hook_main(): + # fill out args + args = FormatArgs() + args.verbose = os.getenv("FORMAT_HOOK_VERBOSE", False) + + # find the changed files + cmd = ["git", "diff", "--cached", "--name-only", "--diff-filter=d"] + proc = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + output = proc.stdout.decode("utf-8") + for line in output.splitlines(): + args.changed_files.append(line) + + failed_fmts = [] + for fmt in ALL_FORMATTERS: + if fmt.has_tool(): + if not fmt.run(args.changed_files, args): + failed_fmts.append(fmt.name) + else: + print(f"Couldn't find {fmt.name}, can't check " + fmt.friendly_name.lower()) + + if len(failed_fmts) > 0: + print( + "Pre-commit format hook failed, rerun with FORMAT_HOOK_VERBOSE=1 environment for verbose output" + ) + sys.exit(1) + + sys.exit(0) + + +if __name__ == "__main__": + script_path = os.path.abspath(__file__) + if ".git/hooks" in script_path: + hook_main() + sys.exit(0) + + parser = argparse.ArgumentParser() + parser.add_argument( + "--token", type=str, required=True, help="GitHub authentication token" + ) + parser.add_argument( + "--repo", + type=str, + default=os.getenv("GITHUB_REPOSITORY", "llvm/llvm-project"), + help="The GitHub repository that we are working with in the form of <owner>/<repo> (e.g. llvm/llvm-project)", + ) + parser.add_argument("--issue-number", type=int, required=True) + parser.add_argument( + "--start-rev", + type=str, + required=True, + help="Compute changes from this revision.", + ) + parser.add_argument( + "--end-rev", type=str, required=True, help="Compute changes to this revision" + ) + parser.add_argument( + "--changed-files", + type=str, + help="Comma separated list of files that has been changed", + ) + parser.add_argument( + "--write-comment-to-file", + action="store_true", + help="Don't post comments on the PR, instead write the comments and metadata a file called 'comment'", + ) + + args = FormatArgs(parser.parse_args()) + + changed_files = [] + if args.changed_files: + changed_files = args.changed_files.split(",") + + failed_formatters = [] + comments = [] + for fmt in ALL_FORMATTERS: + if not fmt.run(changed_files, args): + failed_formatters.append(fmt.name) + if fmt.comment: + comments.append(fmt.comment) + + if len(comments): + with open("comments", "w") as f: + import json + + json.dump(comments, f) + + if len(failed_formatters) > 0: + print(f"error: some formatters failed: {' '.join(failed_formatters)}") + sys.exit(1) |