From ce163b82585ea29d67333f57202665dd9b4b37fe Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 28 Aug 2026 20:41:57 -0500 Subject: [PATCH] [ci] Cache clang-tidy idedata and key ESP-IDF cache on Python version (#18868) --- .../cache-clang-tidy-idedata/action.yml | 50 ++++++++++++++++ .github/actions/cache-esp-idf/action.yml | 16 ++++-- .github/workflows/ci.yml | 24 ++++++++ script/clang_tidy_hash.py | 57 ++++++++++++++++--- script/determine-jobs.py | 24 ++------ script/helpers.py | 11 ++-- tests/script/test_clang_tidy_hash.py | 37 ++++++++++++ 7 files changed, 180 insertions(+), 39 deletions(-) create mode 100644 .github/actions/cache-clang-tidy-idedata/action.yml diff --git a/.github/actions/cache-clang-tidy-idedata/action.yml b/.github/actions/cache-clang-tidy-idedata/action.yml new file mode 100644 index 0000000000..18f3c2b31a --- /dev/null +++ b/.github/actions/cache-clang-tidy-idedata/action.yml @@ -0,0 +1,50 @@ +name: Cache clang-tidy idedata +description: > + Cache the clang-tidy idedata and the headers it references under .temp + (headers only, about 30MB per env). Run after restore-python and cache-esp-idf. +inputs: + environment: + description: 'clang-tidy environment (e.g. esp32-idf-tidy).' + required: true +runs: + using: composite + steps: + - name: Compute cache key + id: key + shell: bash + run: | + . venv/bin/activate + [ -n "${{ inputs.environment }}" ] || { echo "::error::cache-clang-tidy-idedata: 'environment' input is empty"; exit 1; } + hash=$(python -c 'import sys; sys.path.insert(0, "script"); from clang_tidy_hash import idedata_cache_hash; print(idedata_cache_hash("${{ inputs.environment }}"))') + pyver=$(python -c 'import platform; print(platform.python_version())') + # Generating idedata is what installs ESP-IDF; never skip it over a missing + # install. This also skips the save, so a dev run that installs ESP-IDF + # warms the idedata cache on the next run. + if [ -d ~/.esphome-idf/frameworks ]; then + echo "skip=false" >> "$GITHUB_OUTPUT" + else + echo "ESP-IDF install missing, not using the clang-tidy idedata cache" + echo "skip=true" >> "$GITHUB_OUTPUT" + fi + echo "key=${{ runner.os }}-tidy-idedata-${{ inputs.environment }}-$hash-py$pyver" >> "$GITHUB_OUTPUT" + { + echo "path<> "$GITHUB_OUTPUT" + # Mirror cache-esp-idf: write on dev, restore-only on PRs. The post-step + # save only runs when the job succeeded, so a failed generation is never saved. + # Extend the extension list if a component ships extensionless headers. + - name: Cache clang-tidy idedata (write on dev) + if: (github.ref == 'refs/heads/dev' || contains(github.event.pull_request.labels.*.name, 'ci-cache-write')) && steps.key.outputs.skip != 'true' + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: ${{ steps.key.outputs.path }} + key: ${{ steps.key.outputs.key }} + - name: Cache clang-tidy idedata (restore-only off dev) + if: github.ref != 'refs/heads/dev' && !contains(github.event.pull_request.labels.*.name, 'ci-cache-write') && steps.key.outputs.skip != 'true' + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: ${{ steps.key.outputs.path }} + key: ${{ steps.key.outputs.key }} diff --git a/.github/actions/cache-esp-idf/action.yml b/.github/actions/cache-esp-idf/action.yml index b884e1e4c6..58c9b69cd7 100644 --- a/.github/actions/cache-esp-idf/action.yml +++ b/.github/actions/cache-esp-idf/action.yml @@ -26,6 +26,9 @@ runs: # The native-IDF version is pinned in code, not in any file that feeds the # other cache keys, so resolve it explicitly. Keying on it means the cache # invalidates on a version bump (actions/cache never overwrites a key). + # Also key on the Python version: the cached IDF venv links to the + # runner's toolcache interpreter and is reinstalled every run after a + # runner image bump. id: version shell: bash run: | @@ -36,19 +39,22 @@ runs: version=$(python -c 'from esphome.components.esp32 import ESP_IDF_FRAMEWORK_VERSION_LOOKUP as L; print(L["recommended"])') fi echo "version=$version" >> "$GITHUB_OUTPUT" + echo "python-version=$(python -c 'import platform; print(platform.python_version())')" >> "$GITHUB_OUTPUT" # Mirror the adjacent PlatformIO cache: only dev-branch runs write the # shared cache (so it lives in the default-branch scope readable by all # PRs), and PRs are restore-only -- they never push multi-GB artifacts into - # their own scope / the repo quota (e.g. on a version-bump PR). + # their own scope / the repo quota (e.g. on a version-bump PR). The + # ci-cache-write label lets a PR write into its own scope to test the hit path; + # that costs about 1GB of the repo cache quota per run, so remove it when done. - name: Cache ESP-IDF install (write on dev) - if: github.ref == 'refs/heads/dev' && inputs.restore-only != 'true' + if: (github.ref == 'refs/heads/dev' || contains(github.event.pull_request.labels.*.name, 'ci-cache-write')) && inputs.restore-only != 'true' uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 with: path: ~/.esphome-idf - key: ${{ runner.os }}-esphome-idf-${{ steps.version.outputs.version }} + key: ${{ runner.os }}-esphome-idf-${{ steps.version.outputs.version }}-py${{ steps.version.outputs.python-version }} - name: Cache ESP-IDF install (restore-only off dev) - if: github.ref != 'refs/heads/dev' || inputs.restore-only == 'true' + if: github.ref != 'refs/heads/dev' && !contains(github.event.pull_request.labels.*.name, 'ci-cache-write') || inputs.restore-only == 'true' uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 with: path: ~/.esphome-idf - key: ${{ runner.os }}-esphome-idf-${{ steps.version.outputs.version }} + key: ${{ runner.os }}-esphome-idf-${{ steps.version.outputs.version }}-py${{ steps.version.outputs.python-version }} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cbf6e070b4..b8840f74f8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -649,6 +649,12 @@ jobs: with: framework: arduino + - name: Cache clang-tidy idedata + if: matrix.cache_idf + uses: ./.github/actions/cache-clang-tidy-idedata + with: + environment: esp32-arduino-tidy + - name: Cache nRF Connect SDK install if: matrix.cache_sdk_nrf uses: ./.github/actions/cache-sdk-nrf @@ -730,6 +736,11 @@ jobs: - name: Cache ESP-IDF install uses: ./.github/actions/cache-esp-idf + - name: Cache clang-tidy idedata + uses: ./.github/actions/cache-clang-tidy-idedata + with: + environment: esp32-idf-tidy + - name: Register problem matchers run: | echo "::add-matcher::.github/workflows/matchers/gcc.json" @@ -809,6 +820,11 @@ jobs: - name: Cache ESP-IDF install uses: ./.github/actions/cache-esp-idf + - name: Cache clang-tidy idedata + uses: ./.github/actions/cache-clang-tidy-idedata + with: + environment: esp32-idf-tidy + - name: Register problem matchers run: | echo "::add-matcher::.github/workflows/matchers/gcc.json" @@ -866,16 +882,19 @@ jobs: name: Run script/clang-tidy for ESP32 S3 # yamllint disable-line rule:line-length options: --environment esp32s3-idf-tidy --grep SOC_TEMP_SENSOR_SUPPORTED --grep USE_ESP32_VARIANT_ESP32S3 --grep USE_LOGGER_USB_CDC + tidy_environment: esp32s3-idf-tidy - id: clang-tidy name: Run script/clang-tidy for ESP32 P4 # P4 has no native Wi-Fi/BLE; those run over the hosted co-processor, # so their code paths differ -- lint them under the P4 build too. # yamllint disable-line rule:line-length options: --environment esp32p4-idf-tidy --grep USE_ESP32_VARIANT_ESP32P4 --grep USE_ESP32_HOSTED --grep USE_WIFI --grep USE_BLE + tidy_environment: esp32p4-idf-tidy - id: clang-tidy name: Run script/clang-tidy for ESP32 C6 # yamllint disable-line rule:line-length options: --environment esp32c6-idf-tidy --grep SOC_LP_I2C_SUPPORTED --grep USE_ESP32_VARIANT_ESP32C6 --grep USE_OPENTHREAD --grep USE_ZIGBEE + tidy_environment: esp32c6-idf-tidy steps: - name: Check out code from GitHub @@ -893,6 +912,11 @@ jobs: - name: Cache ESP-IDF install uses: ./.github/actions/cache-esp-idf + - name: Cache clang-tidy idedata + uses: ./.github/actions/cache-clang-tidy-idedata + with: + environment: ${{ matrix.tidy_environment }} + - name: Register problem matchers run: | echo "::add-matcher::.github/workflows/matchers/gcc.json" diff --git a/script/clang_tidy_hash.py b/script/clang_tidy_hash.py index f4fd5a4dff..bdc97bd766 100644 --- a/script/clang_tidy_hash.py +++ b/script/clang_tidy_hash.py @@ -1,13 +1,10 @@ -"""Files that affect clang-tidy results, and a content hash over them. +"""Files that affect clang-tidy results and the idedata built from them. -``CLANG_TIDY_GLOBAL_FILES`` (plus ``SDKCONFIG_DEFAULTS_PREFIX``) is the single -source of truth for which files influence clang-tidy output. A change to any of -them can surface warnings in source files a PR didn't touch, so: - -* ``script/determine-jobs.py`` runs a full clang-tidy scan when one changes, and -* ``calculate_clang_tidy_hash()`` folds them into the idedata cache key used by - ``script/helpers.py`` (a content hash, unlike an mtime check, stays correct - across git checkouts). +``CLANG_TIDY_GLOBAL_FILES`` (plus ``SDKCONFIG_DEFAULTS_PREFIX``) lists the files +that influence clang-tidy output; ``script/determine-jobs.py`` runs a full scan +when one changes. ``ESP_IDF_INFRA_TRIGGER_*`` lists the native ESP-IDF build +code. ``idedata_cache_hash()`` folds the right set into the idedata cache key +used by ``script/helpers.py`` and the CI cache action. """ from __future__ import annotations @@ -31,6 +28,18 @@ CLANG_TIDY_GLOBAL_FILES = ( # this prefix at the repo root. SDKCONFIG_DEFAULTS_PREFIX = "sdkconfig.defaults" +# Native ESP-IDF build infra: determine-jobs forces an esp32 compile when these +# change, and they feed the clang-tidy idedata cache key. +ESP_IDF_INFRA_TRIGGER_PATH_PREFIXES = ("esphome/espidf/", "esphome/build_helpers/") +ESP_IDF_INFRA_TRIGGER_FILES = frozenset( + { + "esphome/build_gen/espidf.py", + "esphome/framework_helpers.py", + "esphome/platformio/library.py", + "esphome/platformio/extra_script.py", + } +) + def read_file_bytes(path: Path) -> bytes: """Read bytes from a file.""" @@ -66,3 +75,33 @@ def calculate_clang_tidy_hash(repo_root: Path | None = None) -> str: hasher.update(read_file_bytes(path)) return hasher.hexdigest() + + +def calculate_idedata_cache_hash(repo_root: Path | None = None) -> str: + """Clang-tidy hash plus the Python that generates the idedata.""" + repo_root = _ensure_repo_root(repo_root) + + hasher = hashlib.sha256() + hasher.update(calculate_clang_tidy_hash(repo_root).encode()) + + paths = {repo_root / name for name in ESP_IDF_INFRA_TRIGGER_FILES} + for prefix in ESP_IDF_INFRA_TRIGGER_PATH_PREFIXES: + # .pyc files appear between the CI key computation and load_idedata's. + paths.update( + path + for path in (repo_root / prefix).rglob("*") + if "__pycache__" not in path.parts + ) + for path in sorted(paths): + if path.is_file(): + hasher.update(str(path.relative_to(repo_root)).encode()) + hasher.update(read_file_bytes(path)) + + return hasher.hexdigest() + + +def idedata_cache_hash(environment: str, repo_root: Path | None = None) -> str: + """Hash gating the cached idedata of one clang-tidy environment.""" + if "esp32" in environment: + return calculate_idedata_cache_hash(repo_root) + return calculate_clang_tidy_hash(repo_root) diff --git a/script/determine-jobs.py b/script/determine-jobs.py index 2bdf7807a9..9eead4b38c 100755 --- a/script/determine-jobs.py +++ b/script/determine-jobs.py @@ -58,7 +58,12 @@ from pathlib import Path import sys from typing import Any -from clang_tidy_hash import CLANG_TIDY_GLOBAL_FILES, SDKCONFIG_DEFAULTS_PREFIX +from clang_tidy_hash import ( + CLANG_TIDY_GLOBAL_FILES, + ESP_IDF_INFRA_TRIGGER_FILES, + ESP_IDF_INFRA_TRIGGER_PATH_PREFIXES, + SDKCONFIG_DEFAULTS_PREFIX, +) from helpers import ( CPP_FILE_EXTENSIONS, ESPHOME_TESTS_COMPONENTS_PATH, @@ -524,23 +529,6 @@ def _esp32_platformio_path_or_file_trigger(files: list[str]) -> bool: return False -# Native-build infra: changes under esphome/espidf/, the shared -# esphome/build_helpers/ package, or the modules the native ESP-IDF build -# imports affect every esp32 IDF build (now the default toolchain) but aren't -# components, so the component matrix wouldn't otherwise force any esp32 -# compile. When they change we fold the `esp32` component into the matrix so -# the default native-IDF build path is still compiled on an infra-only PR. -ESP_IDF_INFRA_TRIGGER_PATH_PREFIXES = ("esphome/espidf/", "esphome/build_helpers/") -ESP_IDF_INFRA_TRIGGER_FILES = frozenset( - { - "esphome/build_gen/espidf.py", - "esphome/framework_helpers.py", - "esphome/platformio/library.py", - "esphome/platformio/extra_script.py", - } -) - - def _esp_idf_infra_changed(files: list[str]) -> bool: """Whether any changed file is ESP-IDF build/runner infrastructure.""" for file in files: diff --git a/script/helpers.py b/script/helpers.py index 9e3969e5ce..e648bb91bb 100644 --- a/script/helpers.py +++ b/script/helpers.py @@ -809,17 +809,14 @@ def load_idedata(environment: str) -> dict[str, Any]: start_time = time.time() print(f"Loading IDE data for environment '{environment}'...") - # Reuse the clang-tidy input hash as the cache key: it already covers every - # file baked into the generated idedata (platformio.ini, sdkconfig.defaults, - # esphome/idf_component.yml), so this can't drift from that file list. A - # content hash -- unlike an mtime comparison -- stays correct across git - # checkouts, which don't preserve mtimes. - from clang_tidy_hash import calculate_clang_tidy_hash + # Content hash of the idedata inputs (data files and the generator code); a + # content hash, unlike mtimes, stays correct across git checkouts. + from clang_tidy_hash import idedata_cache_hash temp_idedata = Path(temp_folder) / f"idedata-{environment}.json" temp_hash = Path(temp_folder) / f"idedata-{environment}.hash" - cache_key = calculate_clang_tidy_hash() + cache_key = idedata_cache_hash(environment) changed = ( not temp_idedata.is_file() or not temp_hash.is_file() diff --git a/tests/script/test_clang_tidy_hash.py b/tests/script/test_clang_tidy_hash.py index b5a9d8ebe9..decae4fd13 100644 --- a/tests/script/test_clang_tidy_hash.py +++ b/tests/script/test_clang_tidy_hash.py @@ -81,3 +81,40 @@ def test_read_file_bytes(tmp_path: Path) -> None: result = clang_tidy_hash.read_file_bytes(test_file) assert result == test_content + + +def test_calculate_idedata_cache_hash_changes_with_infra_code(tmp_path: Path) -> None: + _populate(tmp_path) + infra = tmp_path / "esphome" / "espidf" / "clang_tidy.py" + infra.parent.mkdir(parents=True) + infra.write_text("a") + before = clang_tidy_hash.calculate_idedata_cache_hash(repo_root=tmp_path) + assert before == clang_tidy_hash.calculate_idedata_cache_hash(repo_root=tmp_path) + infra.write_text("b") + assert clang_tidy_hash.calculate_idedata_cache_hash(repo_root=tmp_path) != before + + +def test_calculate_idedata_cache_hash_includes_listed_files(tmp_path: Path) -> None: + _populate(tmp_path) + before = clang_tidy_hash.calculate_idedata_cache_hash(repo_root=tmp_path) + listed = tmp_path / "esphome" / "platformio" / "library.py" + listed.parent.mkdir(parents=True) + listed.write_text("x") + assert clang_tidy_hash.calculate_idedata_cache_hash(repo_root=tmp_path) != before + + +def test_idedata_cache_hash_only_widens_for_esp32(tmp_path: Path) -> None: + _populate(tmp_path) + infra = tmp_path / "esphome" / "espidf" / "clang_tidy.py" + infra.parent.mkdir(parents=True) + infra.write_text("a") + esp32_before = clang_tidy_hash.idedata_cache_hash("esp32-idf-tidy", tmp_path) + other_before = clang_tidy_hash.idedata_cache_hash("esp8266-arduino-tidy", tmp_path) + infra.write_text("b") + assert ( + clang_tidy_hash.idedata_cache_hash("esp32-idf-tidy", tmp_path) != esp32_before + ) + assert ( + clang_tidy_hash.idedata_cache_hash("esp8266-arduino-tidy", tmp_path) + == other_before + )