[ci] Cache clang-tidy idedata and key ESP-IDF cache on Python version (#18868)

.github/actions/cache-clang-tidy-idedata/action.yml

@@ -0,0 +1,50 @@
+name: Cache clang-tidy idedata
+description: >
+  Cache the clang-tidy idedata and the headers it references under .temp
+  (headers only, about 30MB per env). Run after restore-python and cache-esp-idf.
+inputs:
+  environment:
+    description: 'clang-tidy environment (e.g. esp32-idf-tidy).'
+    required: true
+runs:
+  using: composite
+  steps:
+    - name: Compute cache key
+      id: key
+      shell: bash
+      run: |
+        . venv/bin/activate
+        [ -n "${{ inputs.environment }}" ] || { echo "::error::cache-clang-tidy-idedata: 'environment' input is empty"; exit 1; }
+        hash=$(python -c 'import sys; sys.path.insert(0, "script"); from clang_tidy_hash import idedata_cache_hash; print(idedata_cache_hash("${{ inputs.environment }}"))')
+        pyver=$(python -c 'import platform; print(platform.python_version())')
+        # Generating idedata is what installs ESP-IDF; never skip it over a missing
+        # install. This also skips the save, so a dev run that installs ESP-IDF
+        # warms the idedata cache on the next run.
+        if [ -d ~/.esphome-idf/frameworks ]; then
+          echo "skip=false" >> "$GITHUB_OUTPUT"
+        else
+          echo "ESP-IDF install missing, not using the clang-tidy idedata cache"
+          echo "skip=true" >> "$GITHUB_OUTPUT"
+        fi
+        echo "key=${{ runner.os }}-tidy-idedata-${{ inputs.environment }}-$hash-py$pyver" >> "$GITHUB_OUTPUT"
+        {
+          echo "path<<EOF"
+          printf '%s\n' '.temp/idedata-*.json' '.temp/idedata-*.hash'
+          printf '.temp/**/*.%s\n' h hpp hh hxx inc inl ipp tpp
+          echo "EOF"
+        } >> "$GITHUB_OUTPUT"
+    # Mirror cache-esp-idf: write on dev, restore-only on PRs. The post-step
+    # save only runs when the job succeeded, so a failed generation is never saved.
+    # Extend the extension list if a component ships extensionless headers.
+    - name: Cache clang-tidy idedata (write on dev)
+      if: (github.ref == 'refs/heads/dev' || contains(github.event.pull_request.labels.*.name, 'ci-cache-write')) && steps.key.outputs.skip != 'true'
+      uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
+      with:
+        path: ${{ steps.key.outputs.path }}
+        key: ${{ steps.key.outputs.key }}
+    - name: Cache clang-tidy idedata (restore-only off dev)
+      if: github.ref != 'refs/heads/dev' && !contains(github.event.pull_request.labels.*.name, 'ci-cache-write') && steps.key.outputs.skip != 'true'
+      uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
+      with:
+        path: ${{ steps.key.outputs.path }}
+        key: ${{ steps.key.outputs.key }}

.github/actions/cache-esp-idf/action.yml

@@ -26,6 +26,9 @@ runs:
       # The native-IDF version is pinned in code, not in any file that feeds the
       # other cache keys, so resolve it explicitly. Keying on it means the cache
       # invalidates on a version bump (actions/cache never overwrites a key).
+      # Also key on the Python version: the cached IDF venv links to the
+      # runner's toolcache interpreter and is reinstalled every run after a
+      # runner image bump.
       id: version
       shell: bash
       run: |
@@ -36,19 +39,22 @@ runs:
           version=$(python -c 'from esphome.components.esp32 import ESP_IDF_FRAMEWORK_VERSION_LOOKUP as L; print(L["recommended"])')
         fi
         echo "version=$version" >> "$GITHUB_OUTPUT"
+        echo "python-version=$(python -c 'import platform; print(platform.python_version())')" >> "$GITHUB_OUTPUT"
     # Mirror the adjacent PlatformIO cache: only dev-branch runs write the
     # shared cache (so it lives in the default-branch scope readable by all
     # PRs), and PRs are restore-only -- they never push multi-GB artifacts into
-    # their own scope / the repo quota (e.g. on a version-bump PR).
+    # their own scope / the repo quota (e.g. on a version-bump PR). The
+    # ci-cache-write label lets a PR write into its own scope to test the hit path;
+    # that costs about 1GB of the repo cache quota per run, so remove it when done.
     - name: Cache ESP-IDF install (write on dev)
-      if: github.ref == 'refs/heads/dev' && inputs.restore-only != 'true'
+      if: (github.ref == 'refs/heads/dev' || contains(github.event.pull_request.labels.*.name, 'ci-cache-write')) && inputs.restore-only != 'true'
       uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
       with:
         path: ~/.esphome-idf
-        key: ${{ runner.os }}-esphome-idf-${{ steps.version.outputs.version }}
+        key: ${{ runner.os }}-esphome-idf-${{ steps.version.outputs.version }}-py${{ steps.version.outputs.python-version }}
     - name: Cache ESP-IDF install (restore-only off dev)
-      if: github.ref != 'refs/heads/dev' || inputs.restore-only == 'true'
+      if: github.ref != 'refs/heads/dev' && !contains(github.event.pull_request.labels.*.name, 'ci-cache-write') || inputs.restore-only == 'true'
       uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
       with:
         path: ~/.esphome-idf
-        key: ${{ runner.os }}-esphome-idf-${{ steps.version.outputs.version }}
+        key: ${{ runner.os }}-esphome-idf-${{ steps.version.outputs.version }}-py${{ steps.version.outputs.python-version }}

.github/workflows/ci.yml

@@ -649,6 +649,12 @@ jobs:
         with:
           framework: arduino
 
+      - name: Cache clang-tidy idedata
+        if: matrix.cache_idf
+        uses: ./.github/actions/cache-clang-tidy-idedata
+        with:
+          environment: esp32-arduino-tidy
+
       - name: Cache nRF Connect SDK install
         if: matrix.cache_sdk_nrf
         uses: ./.github/actions/cache-sdk-nrf
@@ -730,6 +736,11 @@ jobs:
       - name: Cache ESP-IDF install
         uses: ./.github/actions/cache-esp-idf
 
+      - name: Cache clang-tidy idedata
+        uses: ./.github/actions/cache-clang-tidy-idedata
+        with:
+          environment: esp32-idf-tidy
+
       - name: Register problem matchers
         run: |
           echo "::add-matcher::.github/workflows/matchers/gcc.json"
@@ -809,6 +820,11 @@ jobs:
       - name: Cache ESP-IDF install
         uses: ./.github/actions/cache-esp-idf
 
+      - name: Cache clang-tidy idedata
+        uses: ./.github/actions/cache-clang-tidy-idedata
+        with:
+          environment: esp32-idf-tidy
+
       - name: Register problem matchers
         run: |
           echo "::add-matcher::.github/workflows/matchers/gcc.json"
@@ -866,16 +882,19 @@ jobs:
             name: Run script/clang-tidy for ESP32 S3
             # yamllint disable-line rule:line-length
             options: --environment esp32s3-idf-tidy --grep SOC_TEMP_SENSOR_SUPPORTED --grep USE_ESP32_VARIANT_ESP32S3 --grep USE_LOGGER_USB_CDC
+            tidy_environment: esp32s3-idf-tidy
           - id: clang-tidy
             name: Run script/clang-tidy for ESP32 P4
             # P4 has no native Wi-Fi/BLE; those run over the hosted co-processor,
             # so their code paths differ -- lint them under the P4 build too.
             # yamllint disable-line rule:line-length
             options: --environment esp32p4-idf-tidy --grep USE_ESP32_VARIANT_ESP32P4 --grep USE_ESP32_HOSTED --grep USE_WIFI --grep USE_BLE
+            tidy_environment: esp32p4-idf-tidy
           - id: clang-tidy
             name: Run script/clang-tidy for ESP32 C6
             # yamllint disable-line rule:line-length
             options: --environment esp32c6-idf-tidy --grep SOC_LP_I2C_SUPPORTED --grep USE_ESP32_VARIANT_ESP32C6 --grep USE_OPENTHREAD --grep USE_ZIGBEE
+            tidy_environment: esp32c6-idf-tidy
 
     steps:
       - name: Check out code from GitHub
@@ -893,6 +912,11 @@ jobs:
       - name: Cache ESP-IDF install
         uses: ./.github/actions/cache-esp-idf
 
+      - name: Cache clang-tidy idedata
+        uses: ./.github/actions/cache-clang-tidy-idedata
+        with:
+          environment: ${{ matrix.tidy_environment }}
+
       - name: Register problem matchers
         run: |
           echo "::add-matcher::.github/workflows/matchers/gcc.json"

script/clang_tidy_hash.py

@@ -1,13 +1,10 @@
-"""Files that affect clang-tidy results, and a content hash over them.
+"""Files that affect clang-tidy results and the idedata built from them.
 
-``CLANG_TIDY_GLOBAL_FILES`` (plus ``SDKCONFIG_DEFAULTS_PREFIX``) is the single
-source of truth for which files influence clang-tidy output. A change to any of
-them can surface warnings in source files a PR didn't touch, so:
-
-* ``script/determine-jobs.py`` runs a full clang-tidy scan when one changes, and
-* ``calculate_clang_tidy_hash()`` folds them into the idedata cache key used by
-  ``script/helpers.py`` (a content hash, unlike an mtime check, stays correct
-  across git checkouts).
+``CLANG_TIDY_GLOBAL_FILES`` (plus ``SDKCONFIG_DEFAULTS_PREFIX``) lists the files
+that influence clang-tidy output; ``script/determine-jobs.py`` runs a full scan
+when one changes. ``ESP_IDF_INFRA_TRIGGER_*`` lists the native ESP-IDF build
+code. ``idedata_cache_hash()`` folds the right set into the idedata cache key
+used by ``script/helpers.py`` and the CI cache action.
 """
 
 from __future__ import annotations
@@ -31,6 +28,18 @@ CLANG_TIDY_GLOBAL_FILES = (
 # this prefix at the repo root.
 SDKCONFIG_DEFAULTS_PREFIX = "sdkconfig.defaults"
 
+# Native ESP-IDF build infra: determine-jobs forces an esp32 compile when these
+# change, and they feed the clang-tidy idedata cache key.
+ESP_IDF_INFRA_TRIGGER_PATH_PREFIXES = ("esphome/espidf/", "esphome/build_helpers/")
+ESP_IDF_INFRA_TRIGGER_FILES = frozenset(
+    {
+        "esphome/build_gen/espidf.py",
+        "esphome/framework_helpers.py",
+        "esphome/platformio/library.py",
+        "esphome/platformio/extra_script.py",
+    }
+)
+
 
 def read_file_bytes(path: Path) -> bytes:
     """Read bytes from a file."""
@@ -66,3 +75,33 @@ def calculate_clang_tidy_hash(repo_root: Path | None = None) -> str:
         hasher.update(read_file_bytes(path))
 
     return hasher.hexdigest()
+
+
+def calculate_idedata_cache_hash(repo_root: Path | None = None) -> str:
+    """Clang-tidy hash plus the Python that generates the idedata."""
+    repo_root = _ensure_repo_root(repo_root)
+
+    hasher = hashlib.sha256()
+    hasher.update(calculate_clang_tidy_hash(repo_root).encode())
+
+    paths = {repo_root / name for name in ESP_IDF_INFRA_TRIGGER_FILES}
+    for prefix in ESP_IDF_INFRA_TRIGGER_PATH_PREFIXES:
+        # .pyc files appear between the CI key computation and load_idedata's.
+        paths.update(
+            path
+            for path in (repo_root / prefix).rglob("*")
+            if "__pycache__" not in path.parts
+        )
+    for path in sorted(paths):
+        if path.is_file():
+            hasher.update(str(path.relative_to(repo_root)).encode())
+            hasher.update(read_file_bytes(path))
+
+    return hasher.hexdigest()
+
+
+def idedata_cache_hash(environment: str, repo_root: Path | None = None) -> str:
+    """Hash gating the cached idedata of one clang-tidy environment."""
+    if "esp32" in environment:
+        return calculate_idedata_cache_hash(repo_root)
+    return calculate_clang_tidy_hash(repo_root)

script/determine-jobs.py

@@ -58,7 +58,12 @@ from pathlib import Path
 import sys
 from typing import Any
 
-from clang_tidy_hash import CLANG_TIDY_GLOBAL_FILES, SDKCONFIG_DEFAULTS_PREFIX
+from clang_tidy_hash import (
+    CLANG_TIDY_GLOBAL_FILES,
+    ESP_IDF_INFRA_TRIGGER_FILES,
+    ESP_IDF_INFRA_TRIGGER_PATH_PREFIXES,
+    SDKCONFIG_DEFAULTS_PREFIX,
+)
 from helpers import (
     CPP_FILE_EXTENSIONS,
     ESPHOME_TESTS_COMPONENTS_PATH,
@@ -524,23 +529,6 @@ def _esp32_platformio_path_or_file_trigger(files: list[str]) -> bool:
     return False
 
 
-# Native-build infra: changes under esphome/espidf/, the shared
-# esphome/build_helpers/ package, or the modules the native ESP-IDF build
-# imports affect every esp32 IDF build (now the default toolchain) but aren't
-# components, so the component matrix wouldn't otherwise force any esp32
-# compile. When they change we fold the `esp32` component into the matrix so
-# the default native-IDF build path is still compiled on an infra-only PR.
-ESP_IDF_INFRA_TRIGGER_PATH_PREFIXES = ("esphome/espidf/", "esphome/build_helpers/")
-ESP_IDF_INFRA_TRIGGER_FILES = frozenset(
-    {
-        "esphome/build_gen/espidf.py",
-        "esphome/framework_helpers.py",
-        "esphome/platformio/library.py",
-        "esphome/platformio/extra_script.py",
-    }
-)
-
-
 def _esp_idf_infra_changed(files: list[str]) -> bool:
     """Whether any changed file is ESP-IDF build/runner infrastructure."""
     for file in files:

script/helpers.py

@@ -809,17 +809,14 @@ def load_idedata(environment: str) -> dict[str, Any]:
     start_time = time.time()
     print(f"Loading IDE data for environment '{environment}'...")
 
-    # Reuse the clang-tidy input hash as the cache key: it already covers every
-    # file baked into the generated idedata (platformio.ini, sdkconfig.defaults,
-    # esphome/idf_component.yml), so this can't drift from that file list. A
-    # content hash -- unlike an mtime comparison -- stays correct across git
-    # checkouts, which don't preserve mtimes.
-    from clang_tidy_hash import calculate_clang_tidy_hash
+    # Content hash of the idedata inputs (data files and the generator code); a
+    # content hash, unlike mtimes, stays correct across git checkouts.
+    from clang_tidy_hash import idedata_cache_hash
 
     temp_idedata = Path(temp_folder) / f"idedata-{environment}.json"
     temp_hash = Path(temp_folder) / f"idedata-{environment}.hash"
 
-    cache_key = calculate_clang_tidy_hash()
+    cache_key = idedata_cache_hash(environment)
     changed = (
         not temp_idedata.is_file()
         or not temp_hash.is_file()

tests/script/test_clang_tidy_hash.py

@@ -81,3 +81,40 @@ def test_read_file_bytes(tmp_path: Path) -> None:
     result = clang_tidy_hash.read_file_bytes(test_file)
 
     assert result == test_content
+
+
+def test_calculate_idedata_cache_hash_changes_with_infra_code(tmp_path: Path) -> None:
+    _populate(tmp_path)
+    infra = tmp_path / "esphome" / "espidf" / "clang_tidy.py"
+    infra.parent.mkdir(parents=True)
+    infra.write_text("a")
+    before = clang_tidy_hash.calculate_idedata_cache_hash(repo_root=tmp_path)
+    assert before == clang_tidy_hash.calculate_idedata_cache_hash(repo_root=tmp_path)
+    infra.write_text("b")
+    assert clang_tidy_hash.calculate_idedata_cache_hash(repo_root=tmp_path) != before
+
+
+def test_calculate_idedata_cache_hash_includes_listed_files(tmp_path: Path) -> None:
+    _populate(tmp_path)
+    before = clang_tidy_hash.calculate_idedata_cache_hash(repo_root=tmp_path)
+    listed = tmp_path / "esphome" / "platformio" / "library.py"
+    listed.parent.mkdir(parents=True)
+    listed.write_text("x")
+    assert clang_tidy_hash.calculate_idedata_cache_hash(repo_root=tmp_path) != before
+
+
+def test_idedata_cache_hash_only_widens_for_esp32(tmp_path: Path) -> None:
+    _populate(tmp_path)
+    infra = tmp_path / "esphome" / "espidf" / "clang_tidy.py"
+    infra.parent.mkdir(parents=True)
+    infra.write_text("a")
+    esp32_before = clang_tidy_hash.idedata_cache_hash("esp32-idf-tidy", tmp_path)
+    other_before = clang_tidy_hash.idedata_cache_hash("esp8266-arduino-tidy", tmp_path)
+    infra.write_text("b")
+    assert (
+        clang_tidy_hash.idedata_cache_hash("esp32-idf-tidy", tmp_path) != esp32_before
+    )
+    assert (
+        clang_tidy_hash.idedata_cache_hash("esp8266-arduino-tidy", tmp_path)
+        == other_before
+    )