Merge remote-tracking branch 'origin/integration-debug-rerun' into integration-ccache-seed

An overlapping dev run may delete the same old slice first. A test now
fails when INTEGRATION_TESTS_SPLIT_BUCKETS and the restore steps differ.

.github/workflows/ci.yml

@@ -406,6 +406,11 @@ jobs:
       # Registry libraries (noise-c, libsodium, ArduinoJson, lvgl) the host builds
       # download, shared per xdist worker by tests/integration/conftest.py
       INTEGRATION_LIBRARY_CACHE_PATH: ~/.esphome-integration-tests/pio_components
+      # The host build's ccache (get_build_env() in esphome/host/toolchain.py)
+      INTEGRATION_CCACHE_PATH: ~/.cache/esphome/host/ccache
+      # Only a full matrix run saves the ccache seed: a partial run would
+      # trim it to the few tests it ran
+      INTEGRATION_CCACHE_SAVE: ${{ needs.determine-jobs.outputs.integration-run-all == 'true' && (github.ref == 'refs/heads/dev' || contains(github.event.pull_request.labels.*.name, 'ci-cache-write')) }}
     steps:
       - name: Check out code from GitHub
         uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
@@ -432,6 +437,54 @@ jobs:
         with:
           path: ${{ env.INTEGRATION_LIBRARY_CACHE_PATH }}
           key: integration-libraries-v1-${{ runner.os }}-${{ hashFiles('esphome/components/json/__init__.py', 'esphome/components/noise/__init__.py', 'esphome/components/lvgl/__init__.py', 'esphome/components/improv_base/__init__.py') }}
+      # A full run saves one ccache slice per bucket and restores only its own; a
+      # partial run groups tests differently, so it restores all of them. One step
+      # per slice of INTEGRATION_TESTS_SPLIT_BUCKETS in script/determine-jobs.py.
+      - name: Restore integration ccache (slice 1/5)
+        if: needs.determine-jobs.outputs.integration-run-all != 'true' || strategy.job-index == 0
+        uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
+        with:
+          path: ${{ env.INTEGRATION_CCACHE_PATH }}
+          key: integration-ccache-v1-0-${{ github.run_id }}
+          restore-keys: integration-ccache-v1-0-
+      - name: Restore integration ccache (slice 2/5)
+        if: needs.determine-jobs.outputs.integration-run-all != 'true' || strategy.job-index == 1
+        uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
+        with:
+          path: ${{ env.INTEGRATION_CCACHE_PATH }}
+          key: integration-ccache-v1-1-${{ github.run_id }}
+          restore-keys: integration-ccache-v1-1-
+      - name: Restore integration ccache (slice 3/5)
+        if: needs.determine-jobs.outputs.integration-run-all != 'true' || strategy.job-index == 2
+        uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
+        with:
+          path: ${{ env.INTEGRATION_CCACHE_PATH }}
+          key: integration-ccache-v1-2-${{ github.run_id }}
+          restore-keys: integration-ccache-v1-2-
+      - name: Restore integration ccache (slice 4/5)
+        if: needs.determine-jobs.outputs.integration-run-all != 'true' || strategy.job-index == 3
+        uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
+        with:
+          path: ${{ env.INTEGRATION_CCACHE_PATH }}
+          key: integration-ccache-v1-3-${{ github.run_id }}
+          restore-keys: integration-ccache-v1-3-
+      - name: Restore integration ccache (slice 5/5)
+        if: needs.determine-jobs.outputs.integration-run-all != 'true' || strategy.job-index == 4
+        uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
+        with:
+          path: ${{ env.INTEGRATION_CCACHE_PATH }}
+          key: integration-ccache-v1-4-${{ github.run_id }}
+          restore-keys: integration-ccache-v1-4-
+      - name: Mark the restored ccache as unused
+        # A hit refreshes an entry's mtime, so the trim before the save keeps
+        # exactly what this run used; the zeroed statistics cover this run only
+        run: |
+          dir="${INTEGRATION_CCACHE_PATH/#\~/$HOME}"
+          if [ -d "$dir" ]; then
+            find "$dir" -type f -exec touch -m -d 2000-01-01 {} +
+          fi
+          echo "CCACHE_DIR=$dir" >> "$GITHUB_ENV"
+          CCACHE_DIR="$dir" ccache -z
       - name: Restore Python virtual environment
         id: cache-venv
         uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
@@ -526,10 +579,19 @@ jobs:
           if-no-files-found: error
           # A full cron period of margin for the weekly refresh
           retention-days: 14
+      - name: Trim the integration ccache to this run
+        # Restored entries this run did not hit are stale; dropping them keeps
+        # the seed at one full run's working set instead of growing per save
+        if: success() && env.INTEGRATION_CCACHE_SAVE == 'true'
+        run: ccache --evict-older-than 1d
       - name: Print ccache statistics
-        # esphome stores the host build's ccache under the machine-global
-        # cache dir (see get_build_env() in esphome/host/toolchain.py).
-        run: CCACHE_DIR="$HOME/.cache/esphome/host/ccache" ccache -s
+        run: ccache -s
+      - name: Save integration ccache slice
+        if: success() && env.INTEGRATION_CCACHE_SAVE == 'true'
+        uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
+        with:
+          path: ${{ env.INTEGRATION_CCACHE_PATH }}
+          key: integration-ccache-v1-${{ strategy.job-index }}-${{ github.run_id }}
       - name: Save integration library cache
         # Bucket 0 only; the others would race the same immutable key.
         if: success() && (github.ref == 'refs/heads/dev' || contains(github.event.pull_request.labels.*.name, 'ci-cache-write')) && strategy.job-index == 0 && steps.library-cache.outputs.cache-hit != 'true'
@@ -538,6 +600,38 @@ jobs:
           path: ${{ env.INTEGRATION_LIBRARY_CACHE_PATH }}
           key: ${{ steps.library-cache.outputs.cache-primary-key }}
 
+  integration-ccache-prune:
+    name: Prune replaced integration ccache slices
+    runs-on: ubuntu-24.04
+    needs:
+      - determine-jobs
+      - integration-tests
+    # Dev only: a pull request's saves stay in its own scope and expire unused.
+    # A separate job so no test code runs with the cache write scope.
+    if: >-
+      !cancelled() &&
+      github.ref == 'refs/heads/dev' &&
+      needs.determine-jobs.outputs.integration-run-all == 'true'
+    permissions:
+      actions: write
+    steps:
+      - name: Delete the slices this run replaced
+        env:
+          GH_TOKEN: ${{ github.token }}
+          GH_REPO: ${{ github.repository }}
+        run: |
+          entries=$(gh cache list --ref refs/heads/dev --key integration-ccache-v1- --limit 100 --json id,key)
+          # A slice whose bucket failed did not save; its older entry stays
+          for key in $(jq -r --arg run "-$GITHUB_RUN_ID" '.[].key | select(endswith($run))' <<<"$entries"); do
+            prefix="${key%-"$GITHUB_RUN_ID"}-"
+            jq -r --arg p "$prefix" --argjson run "$GITHUB_RUN_ID" \
+              '.[] | select((.key | startswith($p)) and (.key | ltrimstr($p) | tonumber) < $run) | .id' <<<"$entries" |
+              while read -r id; do
+                # An overlapping dev run may have deleted it already
+                gh cache delete "$id" || echo "::notice::cache $id already gone"
+              done
+          done
+
   import-time:
     name: Check import esphome.__main__ time
     runs-on: ubuntu-24.04

script/determine-jobs.py

@@ -107,6 +107,7 @@ CLANG_TIDY_SPLIT_THRESHOLD = 65
 # bucket, not wall time (calibrated with the conftest compile cap); it
 # sizes the bucket count for small subsets.
 INTEGRATION_TESTS_SPLIT_THRESHOLD = 10
+# ci.yml restores one integration ccache slice per bucket; add a step there too
 INTEGRATION_TESTS_SPLIT_BUCKETS = 5
 INTEGRATION_TESTS_TARGET_BUCKET_WEIGHT = 360.0
 

tests/script/test_determine_jobs.py

@@ -9,6 +9,7 @@ import sys
 from unittest.mock import Mock, patch
 
 import pytest
+import yaml
 
 # Add the script directory to Python path so we can import the module
 script_dir = str((Path(__file__).parent / ".." / ".." / "script").resolve())
@@ -1308,6 +1309,20 @@ def test_esp_idf_infra_changed(changed_files: list[str], expected: bool) -> None
     assert determine_jobs._esp_idf_infra_changed(changed_files) is expected
 
 
+def test_integration_ccache_restores_every_bucket_slice() -> None:
+    """ci.yml restores one ccache slice per full-run bucket; a bucket count
+    change without a matching restore step would leave a slice unread."""
+    repo_root = Path(__file__).resolve().parents[2]
+    workflow = yaml.safe_load((repo_root / ".github/workflows/ci.yml").read_text())
+    keys = [
+        step["with"]["restore-keys"]
+        for step in workflow["jobs"]["integration-tests"]["steps"]
+        if step.get("name", "").startswith("Restore integration ccache (slice")
+    ]
+    buckets = determine_jobs.INTEGRATION_TESTS_SPLIT_BUCKETS
+    assert keys == [f"integration-ccache-v1-{i}-" for i in range(buckets)]
+
+
 def test_esp_idf_infra_trigger_paths_exist() -> None:
     """A renamed or moved trigger module must fail here, not silently stop
     forcing the esp32 IDF compile."""