#!/usr/bin/env python3 import argparse import os from pathlib import Path import queue import re import shutil import subprocess import sys import tempfile import threading import click import colorama from helpers import ( basepath, build_all_include, filter_changed, filter_grep, get_binary, get_usable_cpu_count, git_ls_files, load_idedata, print_error_for_file, print_file_list, root_path, temp_header_file, ) def clang_options(idedata): cmd = [] # extract target architecture from triplet in g++ filename triplet = Path(idedata["cxx_path"]).name[:-4] if triplet.startswith("xtensa-"): # clang doesn't support Xtensa (yet?), so compile in 32-bit mode and pretend we're the Xtensa compiler cmd.append("-m32") cmd.append("-D__XTENSA__") cmd.append("-D_LIBC") else: cmd.append(f"--target={triplet}") omit_flags = ( "-free", "-fipa-pta", "-fstrict-volatile-bitfields", "-mlongcalls", "-mtext-section-literals", "-mdisable-hardware-atomics", "-mfix-esp32-psram-cache-issue", "-mfix-esp32-psram-cache-strategy=memw", "-fno-tree-switch-conversion", ) if "zephyr" in triplet: omit_flags += ( "-fno-reorder-functions", "-mfp16-format=ieee", "--param=min-pagesize=0", ) else: cmd.extend( [ # disable built-in include directories from the host "-nostdinc++", ] ) # set flags cmd.extend( [ # disable built-in include directories from the host "-nostdinc", # replace pgmspace.h, as it uses GNU extensions clang doesn't support # https://github.com/earlephilhower/newlib-xtensa/pull/18 "-D_PGMSPACE_H_", "-Dpgm_read_byte(s)=(*(const uint8_t *)(s))", "-Dpgm_read_byte_near(s)=(*(const uint8_t *)(s))", "-Dpgm_read_word(s)=(*(const uint16_t *)(s))", "-Dpgm_read_dword(s)=(*(const uint32_t *)(s))", "-Dpgm_read_ptr(s)=(*(const void *const *)(s))", "-DPROGMEM=", "-DPGM_P=const char *", "-DPSTR(s)=(s)", # this next one is also needed with upstream pgmspace.h # suppress warning about identifier naming in expansion of this macro "-DPSTRN(s, n)=(s)", # suppress warning about attribute cannot be applied to type # https://github.com/esp8266/Arduino/pull/8258 "-Ddeprecated(x)=", # allow to condition code on the presence of clang-tidy "-DCLANG_TIDY", # (esp-idf) Fix __once_callable in some libstdc++ headers "-D_GLIBCXX_HAVE_TLS", ] ) # copy compiler flags, except those clang doesn't understand. cmd.extend(flag for flag in idedata["cxx_flags"] if flag not in omit_flags) # defines cmd.extend(f"-D{define}" for define in idedata["defines"]) # add toolchain include directories using -isystem to suppress their errors # idedata contains include directories for all toolchains of this platform, only use those from the one in use toolchain_dir = os.path.normpath(f"{idedata['cxx_path']}/../../") for directory in idedata["includes"]["toolchain"]: if directory.startswith(toolchain_dir) and "picolibc" not in directory: cmd.extend(["-isystem", directory]) # add library include directories using -isystem to suppress their errors for directory in list(idedata["includes"]["build"]): # skip our own directories, we add those later if ( not directory.startswith(f"{root_path}") or directory.startswith( ( f"{root_path}/.platformio", f"{root_path}/.temp", f"{root_path}/managed_components", ) ) or (directory.startswith(f"{root_path}") and "/.pio/" in directory) ): cmd.extend(["-isystem", directory]) # add the esphome include directory using -I cmd.extend(["-I", root_path]) return cmd pids = set() def run_tidy(executable, args, options, tmpdir, path_queue, lock, failed_files): while True: path = path_queue.get() invocation = [executable] if tmpdir is not None: invocation.append("--export-fixes") # Get a temporary file. We immediately close the handle so clang-tidy can # overwrite it. (handle, name) = tempfile.mkstemp(suffix=".yaml", dir=tmpdir) os.close(handle) invocation.append(name) if args.quiet: invocation.append("--quiet") if sys.stdout.isatty(): invocation.append("--use-color") invocation.append(f"--header-filter={Path(basepath).resolve()}/.*") invocation.append(str(Path(path).resolve())) invocation.append("--") invocation.extend(options) proc = subprocess.run( invocation, capture_output=True, encoding="utf-8", check=False, close_fds=False, ) if proc.returncode != 0: with lock: print_error_for_file(path, proc.stdout) failed_files.append(path) path_queue.task_done() def progress_bar_show(value): if value is None: return "" return None def split_list(a, n): k, m = divmod(len(a), n) return [a[i * k + min(i, m) : (i + 1) * k + min(i + 1, m)] for i in range(n)] def main(): colorama.init() parser = argparse.ArgumentParser() parser.add_argument( "-j", "--jobs", type=int, default=get_usable_cpu_count(), help="number of tidy instances to be run in parallel.", ) parser.add_argument( "-e", "--environment", default="esp32-arduino-tidy", help="the PlatformIO environment to use (as defined in platformio.ini)", ) parser.add_argument( "files", nargs="*", default=[], help="files to be processed (regex on path)" ) parser.add_argument("--fix", action="store_true", help="apply fix-its") parser.add_argument( "-q", "--quiet", action="store_false", help="run clang-tidy in quiet mode" ) parser.add_argument( "-c", "--changed", action="store_true", help="only run on changed files" ) parser.add_argument( "-g", "--grep", action="append", help="only run on files containing value", ) parser.add_argument( "--split-num", type=int, help="split the files into X jobs.", default=None ) parser.add_argument( "--split-at", type=int, help="which split is this? starts at 1", default=None ) parser.add_argument( "--all-headers", action="store_true", help="create a dummy file that checks all headers", ) args = parser.parse_args() cwd = Path.cwd() files = [os.path.relpath(path, cwd) for path in git_ls_files(["*.cpp"])] # Exclude benchmark files — they require google benchmark headers not # available in the ESP32 toolchain and use different naming conventions. files = [f for f in files if not f.startswith("tests/benchmarks/")] # Print initial file count if it's large if len(files) > 50: print(f"Found {len(files)} total files to process") if args.files: # Match against files specified on command-line file_name_re = re.compile("|".join(args.files)) files = [p for p in files if file_name_re.search(p)] if args.changed: files = filter_changed(files) if args.grep: files = filter_grep(files, args.grep) files.sort() if args.split_num: files = split_list(files, args.split_num)[args.split_at - 1] print(f"Split {args.split_at}/{args.split_num}: checking {len(files)} files") # Print file count before adding header file print(f"\nTotal cpp files to check: {len(files)}") # Add header file for checking (before early exit check) if args.all_headers and args.split_at in (None, 1): # When --changed is used, only include changed headers instead of all headers if args.changed: all_headers = [ os.path.relpath(p, cwd) for p in git_ls_files(["esphome/**/*.h"]) ] changed_headers = filter_changed(all_headers) if changed_headers: build_all_include(changed_headers) files.insert(0, temp_header_file) else: print("No changed headers to check") else: build_all_include() files.insert(0, temp_header_file) print(f"Added all-include header file, new total: {len(files)}") # Early exit if no files to check if not files: print("No files to check - exiting early") return 0 # Print final file list before loading idedata print_file_list(files, "Final files to process:") # Load idedata and options only if we have files to check idedata = load_idedata(args.environment) options = clang_options(idedata) tmpdir = None if args.fix: tmpdir = tempfile.mkdtemp() failed_files = [] try: executable = get_binary("clang-tidy", 22) task_queue = queue.Queue(args.jobs) lock = threading.Lock() for _ in range(args.jobs): t = threading.Thread( target=run_tidy, args=( executable, args, options, tmpdir, task_queue, lock, failed_files, ), ) t.daemon = True t.start() # Fill the queue with files. with click.progressbar( files, width=30, file=sys.stderr, item_show_func=progress_bar_show ) as progress_bar: for name in progress_bar: task_queue.put(name) # Wait for all threads to be done. task_queue.join() except FileNotFoundError: return 1 except KeyboardInterrupt: print() print("Ctrl-C detected, goodbye.") if tmpdir: shutil.rmtree(tmpdir) # Kill subprocesses (and ourselves!) # No simple, clean alternative appears to be available. os.kill(0, 9) return 2 # Will not execute. if args.fix and failed_files: print("Applying fixes ...") try: try: subprocess.call( ["clang-apply-replacements-22", tmpdir], close_fds=False ) except FileNotFoundError: subprocess.call(["clang-apply-replacements", tmpdir], close_fds=False) except FileNotFoundError: print( "Error please install clang-apply-replacements-22 or clang-apply-replacements.\n", file=sys.stderr, ) except: print("Error applying fixes.\n", file=sys.stderr) raise return len(failed_files) if __name__ == "__main__": sys.exit(main())