#!/usr/bin/env python3 import argparse import os from pathlib import Path import queue import re import shutil import subprocess import sys import tempfile import threading import click import colorama from helpers import ( basepath, build_all_include, filter_changed, filter_grep, get_binary, get_usable_cpu_count, git_ls_files, load_idedata, print_error_for_file, print_file_list, root_path, temp_header_file, ) def clang_options(idedata): cmd = [] # extract target architecture from triplet in g++ filename triplet = Path(idedata["cxx_path"]).name[:-4] if triplet.startswith("xtensa-"): # clang has an Xtensa frontend, but only a generic core -- the esp32 IDF # toolchain headers (xtruntime, xtensa/config) need the GCC core config # (XCHAL_*) it doesn't ship, so we still compile in 32-bit x86 mode and # just pretend to be Xtensa. Undefine the host x86 arch macros -m32 sets, # so libraries with x86 SIMD paths (FastLED's fl/math/simd, simd_x86.hpp) # fall back to their scalar implementation instead of an incomplete # host-x86 one, and define the xtensa endianness macro newlib's # machine/ieeefp.h then needs in their place. cmd.append("-m32") cmd.append("-U__i386__") cmd.append("-U__x86_64__") cmd.append("-D__XTENSA__") cmd.append("-D__XTENSA_EL__") cmd.append("-D_LIBC") else: # RISC-V (and other non-Xtensa targets) have a real clang backend, so # compile for the actual triplet. Espressif's RISC-V GCC -march adds # vendor extensions (xesploop, xespv) upstream clang doesn't know; those # are stripped from the copied cxx_flags below. cmd.append(f"--target={triplet}") # The GCC build passes flags (e.g. -fno-plt) that clang accepts for some # targets but not others; don't error on the ones unused for this target. cmd.append("-Qunused-arguments") omit_flags = ( "-free", "-fipa-pta", "-fstrict-volatile-bitfields", "-mlongcalls", "-mtext-section-literals", "-mdisable-hardware-atomics", "-mfix-esp32-psram-cache-issue", "-mfix-esp32-psram-cache-strategy=memw", "-fno-tree-switch-conversion", # GCC-only flags emitted by the native ESP-IDF toolchain build "-freorder-blocks", "-fno-jump-tables", "-fno-shrink-wrap", "-mno-target-align", ) if "zephyr" in triplet: omit_flags += ( "-fno-reorder-functions", "-mfp16-format=ieee", "--param=min-pagesize=0", ) else: cmd.extend( [ # disable built-in include directories from the host "-nostdinc++", ] ) # set flags cmd.extend( [ # disable built-in include directories from the host "-nostdinc", # replace pgmspace.h, as it uses GNU extensions clang doesn't support # https://github.com/earlephilhower/newlib-xtensa/pull/18 "-D_PGMSPACE_H_", "-Dpgm_read_byte(s)=(*(const uint8_t *)(s))", "-Dpgm_read_byte_near(s)=(*(const uint8_t *)(s))", "-Dpgm_read_word(s)=(*(const uint16_t *)(s))", "-Dpgm_read_dword(s)=(*(const uint32_t *)(s))", "-Dpgm_read_ptr(s)=(*(const void *const *)(s))", "-DPROGMEM=", "-DPGM_P=const char *", "-DPSTR(s)=(s)", # this next one is also needed with upstream pgmspace.h # suppress warning about identifier naming in expansion of this macro "-DPSTRN(s, n)=(s)", # suppress warning about attribute cannot be applied to type # https://github.com/esp8266/Arduino/pull/8258 "-Ddeprecated(x)=", # allow to condition code on the presence of clang-tidy "-DCLANG_TIDY", # (esp-idf) Fix __once_callable in some libstdc++ headers "-D_GLIBCXX_HAVE_TLS", ] ) # Copy compiler flags, dropping: ones clang doesn't understand; -Werror* # (clang-tidy enforces .clang-tidy's WarningsAsErrors, and a build -Werror # would bypass the -clang-diagnostic-* suppressions); and -std= (the native # ESP-IDF build defaults to gnu++2b, but ESPHome compiles with gnu++20 per # platformio.ini -- analyzing as C++23 flags code that doesn't build under # gnu++20). Force gnu++20 to match the real build. # Strip Espressif's non-standard RISC-V -march extensions (e.g. xesploop, # xespv); clang rejects the whole arch string otherwise. def strip_esp_march(flag): if flag.startswith("-march=") and triplet.startswith("riscv"): return re.sub(r"_xesp\w+", "", flag) return flag cmd.extend( strip_esp_march(flag) for flag in idedata["cxx_flags"] if flag not in omit_flags and not flag.startswith("-Werror") and not flag.startswith("-std=") and not flag.startswith("-mtune=esp") ) cmd.append("-std=gnu++20") # defines cmd.extend(f"-D{define}" for define in idedata["defines"]) # add toolchain include directories using -isystem to suppress their errors # idedata contains include directories for all toolchains of this platform, only use those from the one in use toolchain_dir = os.path.normpath(f"{idedata['cxx_path']}/../../") for directory in idedata["includes"]["toolchain"]: if directory.startswith(toolchain_dir) and "picolibc" not in directory: cmd.extend(["-isystem", directory]) # add library include directories using -isystem to suppress their errors for directory in list(idedata["includes"]["build"]): # skip our own directories, we add those later if ( not directory.startswith(f"{root_path}") or directory.startswith( ( f"{root_path}/.platformio", f"{root_path}/.temp", f"{root_path}/managed_components", ) ) or (directory.startswith(f"{root_path}") and "/.pio/" in directory) ): cmd.extend(["-isystem", directory]) # add the esphome include directory using -I cmd.extend(["-I", root_path]) return cmd pids = set() def run_tidy(executable, args, options, tmpdir, path_queue, lock, failed_files): while True: path = path_queue.get() invocation = [executable] if tmpdir is not None: invocation.append("--export-fixes") # Get a temporary file. We immediately close the handle so clang-tidy can # overwrite it. (handle, name) = tempfile.mkstemp(suffix=".yaml", dir=tmpdir) os.close(handle) invocation.append(name) if args.quiet: invocation.append("--quiet") if sys.stdout.isatty(): invocation.append("--use-color") invocation.append(f"--header-filter={Path(basepath).resolve()}/.*") invocation.append(str(Path(path).resolve())) invocation.append("--") invocation.extend(options) proc = subprocess.run( invocation, capture_output=True, encoding="utf-8", check=False, close_fds=False, ) if proc.returncode != 0: with lock: print_error_for_file(path, proc.stdout) failed_files.append(path) path_queue.task_done() def progress_bar_show(value): if value is None: return "" return None def split_list(a, n): k, m = divmod(len(a), n) return [a[i * k + min(i, m) : (i + 1) * k + min(i + 1, m)] for i in range(n)] def main(): colorama.init() parser = argparse.ArgumentParser() parser.add_argument( "-j", "--jobs", type=int, default=get_usable_cpu_count(), help="number of tidy instances to be run in parallel.", ) parser.add_argument( "-e", "--environment", default="esp32-arduino-tidy", help="the PlatformIO environment to use (as defined in platformio.ini)", ) parser.add_argument( "files", nargs="*", default=[], help="files to be processed (regex on path)" ) parser.add_argument("--fix", action="store_true", help="apply fix-its") parser.add_argument( "-q", "--quiet", action="store_false", help="run clang-tidy in quiet mode" ) parser.add_argument( "-c", "--changed", action="store_true", help="only run on changed files" ) parser.add_argument( "-g", "--grep", action="append", help="only run on files containing value", ) parser.add_argument( "-x", "--exclude-grep", action="append", help="skip files containing value", ) parser.add_argument( "--split-num", type=int, help="split the files into X jobs.", default=None ) parser.add_argument( "--split-at", type=int, help="which split is this? starts at 1", default=None ) parser.add_argument( "--all-headers", action="store_true", help="create a dummy file that checks all headers", ) args = parser.parse_args() cwd = Path.cwd() files = [os.path.relpath(path, cwd) for path in git_ls_files(["*.cpp"])] # Exclude benchmark files — they require google benchmark headers not # available in the ESP32 toolchain and use different naming conventions. files = [f for f in files if not f.startswith("tests/benchmarks/")] # Print initial file count if it's large if len(files) > 50: print(f"Found {len(files)} total files to process") if args.files: # Match against files specified on command-line file_name_re = re.compile("|".join(args.files)) files = [p for p in files if file_name_re.search(p)] if args.changed: files = filter_changed(files) if args.grep: files = filter_grep(files, args.grep) if args.exclude_grep: excluded = set(filter_grep(files, args.exclude_grep)) files = [f for f in files if f not in excluded] files.sort() if args.split_num: files = split_list(files, args.split_num)[args.split_at - 1] print(f"Split {args.split_at}/{args.split_num}: checking {len(files)} files") # Print file count before adding header file print(f"\nTotal cpp files to check: {len(files)}") # Add header file for checking (before early exit check) if args.all_headers and args.split_at in (None, 1): # When --changed is used, only include changed headers instead of all headers if args.changed: all_headers = [ os.path.relpath(p, cwd) for p in git_ls_files(["esphome/**/*.h"]) ] changed_headers = filter_changed(all_headers) if changed_headers: build_all_include(changed_headers) files.insert(0, temp_header_file) else: print("No changed headers to check") else: build_all_include() files.insert(0, temp_header_file) print(f"Added all-include header file, new total: {len(files)}") # Early exit if no files to check if not files: print("No files to check - exiting early") return 0 # Print final file list before loading idedata print_file_list(files, "Final files to process:") # Load idedata and options only if we have files to check idedata = load_idedata(args.environment) options = clang_options(idedata) tmpdir = None if args.fix: tmpdir = tempfile.mkdtemp() failed_files = [] try: executable = get_binary("clang-tidy", 22) task_queue = queue.Queue(args.jobs) lock = threading.Lock() for _ in range(args.jobs): t = threading.Thread( target=run_tidy, args=( executable, args, options, tmpdir, task_queue, lock, failed_files, ), ) t.daemon = True t.start() # Fill the queue with files. with click.progressbar( files, width=30, file=sys.stderr, item_show_func=progress_bar_show ) as progress_bar: for name in progress_bar: task_queue.put(name) # Wait for all threads to be done. task_queue.join() except FileNotFoundError: return 1 except KeyboardInterrupt: print() print("Ctrl-C detected, goodbye.") if tmpdir: shutil.rmtree(tmpdir) # Kill subprocesses (and ourselves!) # No simple, clean alternative appears to be available. os.kill(0, 9) return 2 # Will not execute. if args.fix and failed_files: print("Applying fixes ...") try: try: subprocess.call( ["clang-apply-replacements-22", tmpdir], close_fds=False ) except FileNotFoundError: subprocess.call(["clang-apply-replacements", tmpdir], close_fds=False) except FileNotFoundError: print( "Error please install clang-apply-replacements-22 or clang-apply-replacements.\n", file=sys.stderr, ) except: print("Error applying fixes.\n", file=sys.stderr) raise return len(failed_files) if __name__ == "__main__": sys.exit(main())