diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 371debaab1..1cf59f073f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -893,14 +893,20 @@ jobs: matrix: include: - id: clang-tidy - name: Run script/clang-tidy for ESP32 IDF 1/3 - options: --environment esp32-idf-tidy --split-num 3 --split-at 1 + name: Run script/clang-tidy for ESP32 IDF 1/5 + options: --environment esp32-idf-tidy --split-num 5 --split-at 1 - id: clang-tidy - name: Run script/clang-tidy for ESP32 IDF 2/3 - options: --environment esp32-idf-tidy --split-num 3 --split-at 2 + name: Run script/clang-tidy for ESP32 IDF 2/5 + options: --environment esp32-idf-tidy --split-num 5 --split-at 2 - id: clang-tidy - name: Run script/clang-tidy for ESP32 IDF 3/3 - options: --environment esp32-idf-tidy --split-num 3 --split-at 3 + name: Run script/clang-tidy for ESP32 IDF 3/5 + options: --environment esp32-idf-tidy --split-num 5 --split-at 3 + - id: clang-tidy + name: Run script/clang-tidy for ESP32 IDF 4/5 + options: --environment esp32-idf-tidy --split-num 5 --split-at 4 + - id: clang-tidy + name: Run script/clang-tidy for ESP32 IDF 5/5 + options: --environment esp32-idf-tidy --split-num 5 --split-at 5 steps: - name: Check out code from GitHub diff --git a/script/clang-tidy b/script/clang-tidy index ad6c99d637..f97e9a482e 100755 --- a/script/clang-tidy +++ b/script/clang-tidy @@ -1,6 +1,7 @@ #!/usr/bin/env python3 import argparse +import heapq import os from pathlib import Path import queue @@ -11,6 +12,7 @@ import subprocess import sys import tempfile import threading +import time from typing import Any import click @@ -278,9 +280,12 @@ def clang_options(idedata, environment): pids = set() -def run_tidy(executable, args, options, tmpdir, path_queue, lock, failed_files): +def run_tidy( + executable, args, options, tmpdir, path_queue, lock, failed_files, timings +): while True: path = path_queue.get() + started = time.monotonic() invocation = [executable] if tmpdir is not None: @@ -319,8 +324,9 @@ def run_tidy(executable, args, options, tmpdir, path_queue, lock, failed_files): check=False, close_fds=False, ) - if proc.returncode != 0: - with lock: + with lock: + timings.append((time.monotonic() - started, path)) + if proc.returncode != 0: print_error_for_file(path, proc.stdout) failed_files.append(path) path_queue.task_done() @@ -333,8 +339,27 @@ def progress_bar_show(value): def split_list(a, n): - k, m = divmod(len(a), n) - return [a[i * k + min(i, m) : (i + 1) * k + min(i + 1, m)] for i in range(n)] + """Deal the sorted files out to n splits in turn. + + Contiguous slices put whole directories on one split; expensive + neighbours such as esphome/core then land together and that split takes + several times longer than the others. Dealing spreads them out. + """ + return [a[i::n] for i in range(n)] + + +SLOWEST_FILES_SHOWN = 15 + + +def print_slowest_files(timings): + """Print the files clang-tidy spent the longest on, so the splits can be + rebalanced from CI logs.""" + if not timings: + return + total = sum(seconds for seconds, _ in timings) + print(f"\nclang-tidy spent {total:.0f} s on {len(timings)} files; slowest:") + for seconds, path in heapq.nlargest(SLOWEST_FILES_SHOWN, timings): + print(f" {seconds:6.1f} s {path}") def main(): @@ -458,6 +483,7 @@ def main(): tmpdir = tempfile.mkdtemp() failed_files = [] + timings = [] try: executable = get_binary("clang-tidy", 22) task_queue = queue.Queue(args.jobs) @@ -473,6 +499,7 @@ def main(): task_queue, lock, failed_files, + timings, ), ) t.daemon = True @@ -487,6 +514,7 @@ def main(): # Wait for all threads to be done. task_queue.join() + print_slowest_files(timings) except FileNotFoundError: return 1 diff --git a/script/helpers.py b/script/helpers.py index fb2c964bdd..11605fcef0 100644 --- a/script/helpers.py +++ b/script/helpers.py @@ -1616,7 +1616,7 @@ def lpt_partition( Heaviest item first into the lightest group. Ties keep input order, so pass pre-sorted items for deterministic output. script/clang-tidy's - split_list is the unweighted contiguous sibling. + split_list is the unweighted sibling that deals files out in turn. """ groups: list[list[str]] = [[] for _ in range(count)] group_weights = [0.0] * count