diff --git a/.github/actions/cache-esp-idf/action.yml b/.github/actions/cache-esp-idf/action.yml index 7a17c222a3..f566ba4c43 100644 --- a/.github/actions/cache-esp-idf/action.yml +++ b/.github/actions/cache-esp-idf/action.yml @@ -2,8 +2,8 @@ name: Cache ESP-IDF description: > Resolve the pinned ESP-IDF version and cache the native ESP-IDF install (toolchains + source) at ~/.esphome-idf. Every job that installs ESP-IDF - natively (clang-tidy for IDF/Arduino and the native-IDF component build) - shares one cache, since the install is identical (ESPHOME_IDF_DEFAULT_TARGETS + natively (clang-tidy for IDF/Arduino and the component test batches) shares + one cache, since the install is identical (ESPHOME_IDF_DEFAULT_TARGETS defaults to "all", so all toolchains are present regardless of the chip). Callers must set env ESPHOME_ESP_IDF_PREFIX: ~/.esphome-idf and have the Python venv already restored. @@ -11,6 +11,12 @@ inputs: framework: description: 'Which pinned IDF version to key on: "espidf" (recommended) or "arduino".' default: espidf + restore-only: + description: > + When "true", only restore -- never save the cache, even on dev. Use from + jobs that may not produce an ESP-IDF install (e.g. a component batch with + no esp32 target), so a partial/empty install is never written to the key. + default: "false" runs: using: composite steps: @@ -33,13 +39,13 @@ runs: # PRs), and PRs are restore-only -- they never push multi-GB artifacts into # their own scope / the repo quota (e.g. on a version-bump PR). - name: Cache ESP-IDF install (write on dev) - if: github.ref == 'refs/heads/dev' + if: github.ref == 'refs/heads/dev' && inputs.restore-only != 'true' uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 with: path: ~/.esphome-idf key: ${{ runner.os }}-esphome-idf-${{ steps.version.outputs.version }} - name: Cache ESP-IDF install (restore-only off dev) - if: github.ref != 'refs/heads/dev' + if: github.ref != 'refs/heads/dev' || inputs.restore-only == 'true' uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 with: path: ~/.esphome-idf diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index aca6d9007a..29d42330cd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -270,8 +270,8 @@ jobs: python-linters: ${{ steps.determine.outputs.python-linters }} import-time: ${{ steps.determine.outputs.import-time }} device-builder: ${{ steps.determine.outputs.device-builder }} - native-idf: ${{ steps.determine.outputs.native-idf }} - native-idf-components: ${{ steps.determine.outputs.native-idf-components }} + esp32-platformio: ${{ steps.determine.outputs.esp32-platformio }} + esp32-platformio-components: ${{ steps.determine.outputs.esp32-platformio-components }} changed-components: ${{ steps.determine.outputs.changed-components }} changed-components-with-tests: ${{ steps.determine.outputs.changed-components-with-tests }} directly-changed-components-with-tests: ${{ steps.determine.outputs.directly-changed-components-with-tests }} @@ -324,8 +324,8 @@ jobs: echo "python-linters=$(echo "$output" | jq -r '.python_linters')" >> $GITHUB_OUTPUT echo "import-time=$(echo "$output" | jq -r '.import_time')" >> $GITHUB_OUTPUT echo "device-builder=$(echo "$output" | jq -r '.device_builder')" >> $GITHUB_OUTPUT - echo "native-idf=$(echo "$output" | jq -r '.native_idf')" >> $GITHUB_OUTPUT - echo "native-idf-components=$(echo "$output" | jq -r '.native_idf_components')" >> $GITHUB_OUTPUT + echo "esp32-platformio=$(echo "$output" | jq -r '.esp32_platformio')" >> $GITHUB_OUTPUT + echo "esp32-platformio-components=$(echo "$output" | jq -r '.esp32_platformio_components')" >> $GITHUB_OUTPUT echo "changed-components=$(echo "$output" | jq -c '.changed_components')" >> $GITHUB_OUTPUT echo "changed-components-with-tests=$(echo "$output" | jq -c '.changed_components_with_tests')" >> $GITHUB_OUTPUT echo "directly-changed-components-with-tests=$(echo "$output" | jq -c '.directly_changed_components_with_tests')" >> $GITHUB_OUTPUT @@ -522,7 +522,6 @@ jobs: key: platformio-${{ matrix.pio_cache_key }}-${{ hashFiles('platformio.ini') }} - name: Cache ESP-IDF install - # Shared with the IDF tidy + native-IDF build jobs (same install). if: matrix.cache_idf uses: ./.github/actions/cache-esp-idf with: @@ -592,7 +591,6 @@ jobs: cache-key: ${{ needs.common.outputs.cache-key }} - name: Cache ESP-IDF install - # Shared with the Arduino tidy + native-IDF build jobs (same install). uses: ./.github/actions/cache-esp-idf - name: Register problem matchers @@ -673,7 +671,6 @@ jobs: cache-key: ${{ needs.common.outputs.cache-key }} - name: Cache ESP-IDF install - # Shared with the Arduino tidy + native-IDF build jobs (same install). uses: ./.github/actions/cache-esp-idf - name: Register problem matchers @@ -758,7 +755,6 @@ jobs: cache-key: ${{ needs.common.outputs.cache-key }} - name: Cache ESP-IDF install - # Shared with the IDF/Arduino clang-tidy jobs + native-IDF build (same install). uses: ./.github/actions/cache-esp-idf - name: Register problem matchers @@ -805,6 +801,10 @@ jobs: - common - determine-jobs if: github.event_name == 'pull_request' && fromJSON(needs.determine-jobs.outputs.component-test-count) > 0 + env: + # esp32 component builds use the native ESP-IDF toolchain (default), so + # share the tidy jobs' install location -- the restore below lands here. + ESPHOME_ESP_IDF_PREFIX: ~/.esphome-idf strategy: fail-fast: false max-parallel: ${{ (startsWith(github.base_ref, 'beta') || startsWith(github.base_ref, 'release')) && 8 || 4 }} @@ -832,6 +832,12 @@ jobs: with: python-version: ${{ env.DEFAULT_PYTHON }} cache-key: ${{ needs.common.outputs.cache-key }} + - name: Cache ESP-IDF install (restore-only) + # A batch may contain no esp32 build, so never save -- just reuse the + # shared install the dev tidy jobs already cached when present. + uses: ./.github/actions/cache-esp-idf + with: + restore-only: true - name: Validate and compile components with intelligent grouping run: | . venv/bin/activate @@ -935,20 +941,19 @@ jobs: echo "All components in this batch are validate-only -- skipping compile stage." fi - test-native-idf: - name: Test components with native ESP-IDF + test-esp32-platformio: + name: Test esp32 components with PlatformIO runs-on: ubuntu-24.04 needs: - common - determine-jobs - if: github.event_name == 'pull_request' && needs.determine-jobs.outputs.native-idf == 'true' + if: github.event_name == 'pull_request' && needs.determine-jobs.outputs.esp32-platformio == 'true' env: - ESPHOME_ESP_IDF_PREFIX: ~/.esphome-idf - # Comma-joined subset of the native-IDF representative component list, - # computed by script/determine-jobs.py (native_idf_components_to_test). + # Comma-joined subset of the esp32 PlatformIO representative component list, + # computed by script/determine-jobs.py (esp32_platformio_components_to_test). # Single source of truth -- the full list lives in - # script/determine-jobs.py::NATIVE_IDF_TEST_COMPONENTS. - TEST_COMPONENTS: ${{ needs.determine-jobs.outputs.native-idf-components }} + # script/determine-jobs.py::ESP32_PLATFORMIO_TEST_COMPONENTS. + TEST_COMPONENTS: ${{ needs.determine-jobs.outputs.esp32-platformio-components }} steps: - name: Check out code from GitHub uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 @@ -959,66 +964,22 @@ jobs: python-version: ${{ env.DEFAULT_PYTHON }} cache-key: ${{ needs.common.outputs.cache-key }} - - name: Prepare build storage on /mnt - # Bind-mount the larger /mnt disk over the IDF install + build dirs BEFORE - # restoring the cache, so the ~4.5GB restore lands on the roomier volume - # instead of being shadowed by a mount set up later in the run step. - run: | - root_avail=$(df -k / | awk 'NR==2 {print $4}') - mnt_avail=$(df -k /mnt 2>/dev/null | awk 'NR==2 {print $4}') - echo "Available space: / has ${root_avail}KB, /mnt has ${mnt_avail}KB" - if [ -n "$mnt_avail" ] && [ "$mnt_avail" -gt "$root_avail" ]; then - echo "Using /mnt for build files (more space available)" - sudo mkdir -p /mnt/esphome-idf - sudo chown $USER:$USER /mnt/esphome-idf - mkdir -p ~/.esphome-idf - sudo mount --bind /mnt/esphome-idf ~/.esphome-idf - sudo mkdir -p /mnt/test_build_components_build - sudo chown $USER:$USER /mnt/test_build_components_build - mkdir -p tests/test_build_components/build - sudo mount --bind /mnt/test_build_components_build tests/test_build_components/build - else - echo "Using / for build files (more space available than /mnt or /mnt unavailable)" - fi - - - name: Cache ESP-IDF install - # Shared with the IDF/Arduino clang-tidy jobs (same install); restores - # into the /mnt bind-mount prepared above when present. - uses: ./.github/actions/cache-esp-idf - - - name: Run native ESP-IDF compile test + - name: Run PlatformIO compile test run: | . venv/bin/activate echo "Testing components: $TEST_COMPONENTS" echo "" - # Show disk space before validation - echo "Disk space before config validation:" - df -h - echo "" - # Run config validation (auto-grouped by test_build_components.py) - python3 script/test_build_components.py -e config -t esp32-idf -c "$TEST_COMPONENTS" -f --toolchain esp-idf + python3 script/test_build_components.py -e config -t esp32-idf -c "$TEST_COMPONENTS" -f --toolchain platformio echo "" echo "Config validation passed! Starting compilation..." echo "" - # Show disk space before compilation - echo "Disk space before compilation:" - df -h - echo "" - # Run compilation (auto-grouped by test_build_components.py) - python3 script/test_build_components.py -e compile -t esp32-idf -c "$TEST_COMPONENTS" -f --toolchain esp-idf - - - name: Save ESPHome cache - if: github.ref == 'refs/heads/dev' - uses: actions/cache/save@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 - with: - path: ~/.esphome-idf - key: ${{ runner.os }}-esphome-${{ needs.common.outputs.cache-key }} + python3 script/test_build_components.py -e compile -t esp32-idf -c "$TEST_COMPONENTS" -f --toolchain platformio pre-commit-ci-lite: name: pre-commit.ci lite @@ -1353,7 +1314,7 @@ jobs: - determine-jobs - device-builder - test-build-components-split - - test-native-idf + - test-esp32-platformio - pre-commit-ci-lite - memory-impact-target-branch - memory-impact-pr-branch diff --git a/script/determine-jobs.py b/script/determine-jobs.py index 4904883ca9..af3e83f96b 100755 --- a/script/determine-jobs.py +++ b/script/determine-jobs.py @@ -466,11 +466,11 @@ def should_run_device_builder(branch: str | None = None) -> bool: return False -# Components tested by the native ESP-IDF compile-test job. This is the +# Components tested by the PlatformIO compile-test job. This is the # single source of truth: the workflow reads the comma-joined list from the -# `native-idf-components` output of `determine-jobs` and uses it as the -# `TEST_COMPONENTS` env on the `test-native-idf` job. -NATIVE_IDF_TEST_COMPONENTS = frozenset( +# `esp32-platformio-components` output of `determine-jobs` and uses it as the +# `TEST_COMPONENTS` env on the `test-esp32-platformio` job. +ESP32_PLATFORMIO_TEST_COMPONENTS = frozenset( { "esp32", "api", @@ -490,53 +490,75 @@ NATIVE_IDF_TEST_COMPONENTS = frozenset( } ) -# Path prefixes whose changes always trigger the native ESP-IDF compile -# test: anything under esphome/espidf/ (the native IDF runner / API / -# framework / component generator). -NATIVE_IDF_TRIGGER_PATH_PREFIXES = ("esphome/espidf/",) +# Path prefixes whose changes always trigger the PlatformIO compile test: +# anything under esphome/platformio/ (the PlatformIO runner / toolchain that +# drives every PlatformIO build). The esp32 platform component is already in +# ESP32_PLATFORMIO_TEST_COMPONENTS, so its changes are covered by the normal +# component-narrowing path. +ESP32_PLATFORMIO_TRIGGER_PATH_PREFIXES = ("esphome/platformio/",) -# Standalone files that, when changed, also trigger the native ESP-IDF -# compile test: -# - esphome/build_gen/espidf.py -- the native IDF build generator -# (other files under build_gen/ target PlatformIO and don't affect -# the native IDF path) +# Standalone files that, when changed, trigger the PlatformIO compile test: +# - esphome/build_gen/platformio.py -- the PlatformIO build generator # - script/test_build_components.py -- the harness the job invokes # - .github/workflows/ci.yml -- the job's own definition -NATIVE_IDF_TRIGGER_FILES = frozenset( +ESP32_PLATFORMIO_TRIGGER_FILES = frozenset( { - "esphome/build_gen/espidf.py", + "esphome/build_gen/platformio.py", "script/test_build_components.py", ".github/workflows/ci.yml", } ) -def _native_idf_path_or_file_trigger(files: list[str]) -> bool: - """Whether any changed file is a native IDF infrastructure / harness trigger.""" +def _esp32_platformio_path_or_file_trigger(files: list[str]) -> bool: + """Whether any changed file is a PlatformIO infrastructure / harness trigger.""" for file in files: - if file in NATIVE_IDF_TRIGGER_FILES: + if file in ESP32_PLATFORMIO_TRIGGER_FILES: return True - if any(file.startswith(prefix) for prefix in NATIVE_IDF_TRIGGER_PATH_PREFIXES): + if any( + file.startswith(prefix) for prefix in ESP32_PLATFORMIO_TRIGGER_PATH_PREFIXES + ): return True return False -def native_idf_components_to_test(branch: str | None = None) -> list[str]: - """Subset of ``NATIVE_IDF_TEST_COMPONENTS`` the job needs to compile. +# ESP-IDF infra: changes under esphome/espidf/ or to the IDF build generator +# affect every esp32 IDF build (now the default toolchain) but aren't +# components, so the component matrix wouldn't otherwise force any esp32 +# compile. When they change we fold the `esp32` component into the matrix so +# the default native-IDF build path is still compiled on an infra-only PR. +ESP_IDF_INFRA_TRIGGER_PATH_PREFIXES = ("esphome/espidf/",) +ESP_IDF_INFRA_TRIGGER_FILES = frozenset({"esphome/build_gen/espidf.py"}) - The job builds components with the native ESP-IDF toolchain (no - PlatformIO). When only a specific component (or something it depends - on) changed, there's no value in re-building every other unrelated - component in the test list -- the regular ``component-test`` matrix - already covers them via PlatformIO. So we narrow to the intersection - of ``NATIVE_IDF_TEST_COMPONENTS`` and the changed-component dependency + +def _esp_idf_infra_changed(files: list[str]) -> bool: + """Whether any changed file is ESP-IDF build/runner infrastructure.""" + for file in files: + if file in ESP_IDF_INFRA_TRIGGER_FILES: + return True + if any( + file.startswith(prefix) for prefix in ESP_IDF_INFRA_TRIGGER_PATH_PREFIXES + ): + return True + return False + + +def esp32_platformio_components_to_test(branch: str | None = None) -> list[str]: + """Subset of ``ESP32_PLATFORMIO_TEST_COMPONENTS`` the job needs to compile. + + The job builds components with the PlatformIO toolchain. When only a + specific component (or something it depends on) changed, there's no + value in re-building every other unrelated component in the test list -- + the regular ``component-test`` matrix already covers them via the + default toolchain. So we narrow to the intersection of + ``ESP32_PLATFORMIO_TEST_COMPONENTS`` and the changed-component dependency closure. Returns the full list (sorted) when we can't safely narrow: 1. Core C++/Python files changed (``esphome/core/*``). - 2. Native IDF infrastructure changed (``esphome/espidf/*`` or - ``esphome/build_gen/espidf.py``). + 2. PlatformIO infrastructure changed (``esphome/platformio/*`` or + ``esphome/build_gen/platformio.py``). 3. The test harness or workflow itself changed (``script/test_build_components.py``, ``.github/workflows/ci.yml``). @@ -558,31 +580,31 @@ def native_idf_components_to_test(branch: str | None = None) -> list[str]: """ files = changed_files(branch) - if core_changed(files) or _native_idf_path_or_file_trigger(files): - return sorted(NATIVE_IDF_TEST_COMPONENTS) + if core_changed(files) or _esp32_platformio_path_or_file_trigger(files): + return sorted(ESP32_PLATFORMIO_TEST_COMPONENTS) component_files = [f for f in files if filter_component_and_test_files(f)] changed = get_components_with_dependencies(component_files, True) - return sorted(NATIVE_IDF_TEST_COMPONENTS & set(changed)) + return sorted(ESP32_PLATFORMIO_TEST_COMPONENTS & set(changed)) -def should_run_native_idf(branch: str | None = None) -> bool: - """Determine if the `test-native-idf` compile-test job should run. +def should_run_esp32_platformio(branch: str | None = None) -> bool: + """Determine if the `test-esp32-platformio` compile-test job should run. - Runs whenever ``native_idf_components_to_test()`` returns a non-empty + Runs whenever ``esp32_platformio_components_to_test()`` returns a non-empty list. Skipping the job on unrelated Python-only PRs avoids ~5 min of CI per PR (worse on cold caches). The regular ``component-test`` - matrix still exercises the same components through PlatformIO when - those components change. + matrix still exercises the same components through the default + toolchain when those components change. Args: branch: Branch to compare against. If None, uses default. Returns: - True if the native ESP-IDF compile test should run, False otherwise. + True if the PlatformIO compile test should run, False otherwise. """ - return bool(native_idf_components_to_test(branch)) + return bool(esp32_platformio_components_to_test(branch)) def determine_cpp_unit_tests( @@ -1162,8 +1184,8 @@ def main() -> None: run_python_linters = True run_import_time = True run_device_builder = True - native_idf_components = sorted(NATIVE_IDF_TEST_COMPONENTS) - run_native_idf = True + esp32_platformio_components = sorted(ESP32_PLATFORMIO_TEST_COMPONENTS) + run_esp32_platformio = True else: integration_run_all, integration_test_files = determine_integration_tests( args.branch @@ -1173,8 +1195,8 @@ def main() -> None: run_python_linters = should_run_python_linters(args.branch) run_import_time = should_run_import_time(args.branch) run_device_builder = should_run_device_builder(args.branch) - native_idf_components = native_idf_components_to_test(args.branch) - run_native_idf = bool(native_idf_components) + esp32_platformio_components = esp32_platformio_components_to_test(args.branch) + run_esp32_platformio = bool(esp32_platformio_components) run_integration, integration_test_buckets = _compute_integration_test_buckets( integration_run_all, integration_test_files ) @@ -1228,6 +1250,18 @@ def main() -> None: if _component_has_tests(component) ] + # ESP-IDF build-gen/runner changed but no component pulled esp32 in: fold the + # `esp32` component into the matrix so the default native-IDF build path is + # still compiled on an infra-only PR. force_all/core already test everything, + # so skip there. Runs grouped (not added to directly-changed). + if ( + not is_core_change + and _esp_idf_infra_changed(changed) + and "esp32" not in changed_components_with_tests + and _component_has_tests("esp32") + ): + changed_components_with_tests.append("esp32") + # Get directly changed components with tests (for isolated testing) # These will be tested WITHOUT --testing-mode in CI to enable full validation # (pin conflicts, etc.) since they contain the actual changes being reviewed @@ -1345,8 +1379,8 @@ def main() -> None: "python_linters": run_python_linters, "import_time": run_import_time, "device_builder": run_device_builder, - "native_idf": run_native_idf, - "native_idf_components": ",".join(native_idf_components), + "esp32_platformio": run_esp32_platformio, + "esp32_platformio_components": ",".join(esp32_platformio_components), "changed_components": changed_components, "changed_components_with_tests": changed_components_with_tests, "directly_changed_components_with_tests": list(directly_changed_with_tests), diff --git a/tests/script/test_determine_jobs.py b/tests/script/test_determine_jobs.py index f8f359ee22..a9876632bd 100644 --- a/tests/script/test_determine_jobs.py +++ b/tests/script/test_determine_jobs.py @@ -68,13 +68,13 @@ def mock_should_run_device_builder() -> Generator[Mock, None, None]: @pytest.fixture -def mock_native_idf_components_to_test() -> Generator[Mock, None, None]: - """Mock native_idf_components_to_test from determine_jobs. +def mock_esp32_platformio_components_to_test() -> Generator[Mock, None, None]: + """Mock esp32_platformio_components_to_test from determine_jobs. - main() drives both the ``native_idf`` boolean output and the - ``native_idf_components`` CSV from this one function. + main() drives both the ``esp32_platformio`` boolean output and the + ``esp32_platformio_components`` CSV from this one function. """ - with patch.object(determine_jobs, "native_idf_components_to_test") as mock: + with patch.object(determine_jobs, "esp32_platformio_components_to_test") as mock: yield mock @@ -115,7 +115,7 @@ def test_main_all_tests_should_run( mock_should_run_python_linters: Mock, mock_should_run_import_time: Mock, mock_should_run_device_builder: Mock, - mock_native_idf_components_to_test: Mock, + mock_esp32_platformio_components_to_test: Mock, mock_changed_files: Mock, mock_determine_cpp_unit_tests: Mock, capsys: pytest.CaptureFixture[str], @@ -131,7 +131,7 @@ def test_main_all_tests_should_run( mock_should_run_python_linters.return_value = True mock_should_run_import_time.return_value = True mock_should_run_device_builder.return_value = True - mock_native_idf_components_to_test.return_value = ["api", "esp32"] + mock_esp32_platformio_components_to_test.return_value = ["api", "esp32"] mock_determine_cpp_unit_tests.return_value = (False, ["wifi", "api", "sensor"]) # Mock changed_files to return non-component files (to avoid memory impact) @@ -213,8 +213,8 @@ def test_main_all_tests_should_run( assert output["python_linters"] is True assert output["import_time"] is True assert output["device_builder"] is True - assert output["native_idf"] is True - assert output["native_idf_components"] == "api,esp32" + assert output["esp32_platformio"] is True + assert output["esp32_platformio_components"] == "api,esp32" assert output["changed_components"] == ["wifi", "api", "sensor"] # changed_components_with_tests will only include components that actually have test files assert "changed_components_with_tests" in output @@ -248,7 +248,7 @@ def test_main_no_tests_should_run( mock_should_run_python_linters: Mock, mock_should_run_import_time: Mock, mock_should_run_device_builder: Mock, - mock_native_idf_components_to_test: Mock, + mock_esp32_platformio_components_to_test: Mock, mock_changed_files: Mock, mock_determine_cpp_unit_tests: Mock, capsys: pytest.CaptureFixture[str], @@ -264,7 +264,7 @@ def test_main_no_tests_should_run( mock_should_run_python_linters.return_value = False mock_should_run_import_time.return_value = False mock_should_run_device_builder.return_value = False - mock_native_idf_components_to_test.return_value = [] + mock_esp32_platformio_components_to_test.return_value = [] mock_determine_cpp_unit_tests.return_value = (False, []) # Mock changed_files to return no component files @@ -305,8 +305,8 @@ def test_main_no_tests_should_run( assert output["python_linters"] is False assert output["import_time"] is False assert output["device_builder"] is False - assert output["native_idf"] is False - assert output["native_idf_components"] == "" + assert output["esp32_platformio"] is False + assert output["esp32_platformio_components"] == "" assert output["changed_components"] == [] assert output["changed_components_with_tests"] == [] assert output["component_test_count"] == 0 @@ -322,6 +322,65 @@ def test_main_no_tests_should_run( assert output["component_test_batches"] == [] +def test_main_esp_idf_infra_change_folds_esp32( + mock_determine_integration_tests: Mock, + mock_should_run_clang_tidy: Mock, + mock_should_run_clang_format: Mock, + mock_should_run_python_linters: Mock, + mock_should_run_import_time: Mock, + mock_should_run_device_builder: Mock, + mock_esp32_platformio_components_to_test: Mock, + mock_changed_files: Mock, + mock_determine_cpp_unit_tests: Mock, + capsys: pytest.CaptureFixture[str], + monkeypatch: pytest.MonkeyPatch, +) -> None: + """An ESP-IDF infra-only change folds the `esp32` component into the matrix, + so the default native-IDF build path is still compiled.""" + monkeypatch.delenv("GITHUB_ACTIONS", raising=False) + + mock_determine_integration_tests.return_value = (False, []) + mock_should_run_clang_tidy.return_value = False + mock_should_run_clang_format.return_value = False + mock_should_run_python_linters.return_value = False + mock_should_run_import_time.return_value = False + mock_should_run_device_builder.return_value = False + mock_esp32_platformio_components_to_test.return_value = [] + mock_determine_cpp_unit_tests.return_value = (False, []) + + # IDF build generator changed; no component changed. + mock_changed_files.return_value = ["esphome/build_gen/espidf.py"] + + with ( + patch("sys.argv", ["determine-jobs.py"]), + patch.object(determine_jobs, "get_changed_components", return_value=[]), + patch.object( + determine_jobs, "filter_component_and_test_files", return_value=False + ), + patch.object( + determine_jobs, "get_components_with_dependencies", return_value=[] + ), + # esp32 has tests on disk, but pin it so the fold-in isn't coupled to layout. + patch.object(determine_jobs, "_component_has_tests", return_value=True), + patch.object( + determine_jobs, + "detect_memory_impact_config", + return_value={"should_run": "false"}, + ), + patch.object( + determine_jobs, "create_intelligent_batches", return_value=([], {}) + ), + ): + determine_jobs.main() + + output = json.loads(capsys.readouterr().out) + # Only `esp32` is folded in (not the whole representative set), and it's + # grouped, not isolated (infra changed, not the component). + assert output["changed_components_with_tests"] == ["esp32"] + assert output["directly_changed_components_with_tests"] == [] + assert output["component_test_count"] == 1 + + def test_main_with_branch_argument( mock_determine_integration_tests: Mock, mock_should_run_clang_tidy: Mock, @@ -329,7 +388,7 @@ def test_main_with_branch_argument( mock_should_run_python_linters: Mock, mock_should_run_import_time: Mock, mock_should_run_device_builder: Mock, - mock_native_idf_components_to_test: Mock, + mock_esp32_platformio_components_to_test: Mock, mock_changed_files: Mock, mock_determine_cpp_unit_tests: Mock, capsys: pytest.CaptureFixture[str], @@ -345,7 +404,7 @@ def test_main_with_branch_argument( mock_should_run_python_linters.return_value = True mock_should_run_import_time.return_value = True mock_should_run_device_builder.return_value = True - mock_native_idf_components_to_test.return_value = ["esp32"] + mock_esp32_platformio_components_to_test.return_value = ["esp32"] mock_determine_cpp_unit_tests.return_value = (False, ["mqtt"]) # Mock changed_files to return non-component files (to avoid memory impact) @@ -384,7 +443,7 @@ def test_main_with_branch_argument( mock_should_run_python_linters.assert_called_once_with("main") mock_should_run_import_time.assert_called_once_with("main") mock_should_run_device_builder.assert_called_once_with("main") - mock_native_idf_components_to_test.assert_called_once_with("main") + mock_esp32_platformio_components_to_test.assert_called_once_with("main") # Check output captured = capsys.readouterr() @@ -398,8 +457,8 @@ def test_main_with_branch_argument( assert output["python_linters"] is True assert output["import_time"] is True assert output["device_builder"] is True - assert output["native_idf"] is True - assert output["native_idf_components"] == "esp32" + assert output["esp32_platformio"] is True + assert output["esp32_platformio_components"] == "esp32" assert output["changed_components"] == ["mqtt"] # changed_components_with_tests will only include components that actually have test files assert "changed_components_with_tests" in output @@ -916,23 +975,22 @@ def test_should_run_device_builder_skips_beta_release(target_branch: str) -> Non mock_changed.assert_not_called() -_NATIVE_IDF_FULL_LIST_FILES = [ +_ESP32_PLATFORMIO_FULL_LIST_FILES = [ # Core C++/Python changes -- caught by core_changed() ["esphome/core/component.cpp"], ["esphome/core/config.py"], - # Native IDF infrastructure paths - ["esphome/espidf/framework.py"], - ["esphome/espidf/component.py"], - ["esphome/espidf/api.py"], - ["esphome/build_gen/espidf.py"], + # PlatformIO subsystem (path-prefix trigger) + build generator + ["esphome/platformio/runner.py"], + ["esphome/platformio/toolchain.py"], + ["esphome/build_gen/platformio.py"], # Workflow / harness files ["script/test_build_components.py"], [".github/workflows/ci.yml"], ] -@pytest.mark.parametrize("changed_files", _NATIVE_IDF_FULL_LIST_FILES) -def test_native_idf_components_to_test_returns_full_list_on_infrastructure( +@pytest.mark.parametrize("changed_files", _ESP32_PLATFORMIO_FULL_LIST_FILES) +def test_esp32_platformio_components_to_test_returns_full_list_on_infrastructure( changed_files: list[str], ) -> None: """Infrastructure / core / harness changes fall back to the full component list.""" @@ -944,8 +1002,8 @@ def test_native_idf_components_to_test_returns_full_list_on_infrastructure( determine_jobs, "get_components_with_dependencies", return_value=["wifi"] ), ): - result = determine_jobs.native_idf_components_to_test() - assert result == sorted(determine_jobs.NATIVE_IDF_TEST_COMPONENTS) + result = determine_jobs.esp32_platformio_components_to_test() + assert result == sorted(determine_jobs.ESP32_PLATFORMIO_TEST_COMPONENTS) @pytest.mark.parametrize( @@ -965,7 +1023,7 @@ def test_native_idf_components_to_test_returns_full_list_on_infrastructure( ["ble_scanner", "esp32_ble", "esp32_ble_tracker"], ), # api in the test set -- narrow to [api] even though the closure - # has other (unrelated to native-IDF coverage) entries. + # has other (unrelated to PlatformIO coverage) entries. ( ["esphome/components/api/api_connection.cpp"], ["api", "logger"], @@ -979,15 +1037,15 @@ def test_native_idf_components_to_test_returns_full_list_on_infrastructure( ), # Pure Python-only change outside trigger paths -> empty. (["esphome/yaml_util.py"], [], []), - # Non-IDF files in esphome/build_gen/ do NOT trigger the full - # list -- only esphome/build_gen/espidf.py is a trigger. - (["esphome/build_gen/platformio.py"], [], []), + # Non-PlatformIO files in esphome/build_gen/ do NOT trigger the + # full list -- only esphome/build_gen/platformio.py is a trigger. + (["esphome/build_gen/espidf.py"], [], []), # Docs / unrelated files -> empty. (["README.md"], [], []), ([], [], []), ], ) -def test_native_idf_components_to_test_narrowing( +def test_esp32_platformio_components_to_test_narrowing( changed_files: list[str], dependency_closure: list[str], expected: list[str], @@ -1001,12 +1059,12 @@ def test_native_idf_components_to_test_narrowing( return_value=dependency_closure, ), ): - result = determine_jobs.native_idf_components_to_test() + result = determine_jobs.esp32_platformio_components_to_test() assert result == expected -def test_native_idf_components_to_test_with_branch() -> None: - """native_idf_components_to_test passes branch argument through. +def test_esp32_platformio_components_to_test_with_branch() -> None: + """esp32_platformio_components_to_test passes branch argument through. Regression test: an earlier version called ``get_changed_components()``, which silently ignored the branch argument because that helper re-runs @@ -1021,7 +1079,7 @@ def test_native_idf_components_to_test_with_branch() -> None: ), ): mock_changed.return_value = [] - determine_jobs.native_idf_components_to_test("release") + determine_jobs.esp32_platformio_components_to_test("release") mock_changed.assert_called_once_with("release") @@ -1033,25 +1091,46 @@ def test_native_idf_components_to_test_with_branch() -> None: (["esp32", "api"], True), ], ) -def test_should_run_native_idf(components_to_test: list[str], expected: bool) -> None: - """should_run_native_idf is a thin wrapper around the component list.""" +def test_should_run_esp32_platformio( + components_to_test: list[str], expected: bool +) -> None: + """should_run_esp32_platformio is a thin wrapper around the component list.""" with patch.object( determine_jobs, - "native_idf_components_to_test", + "esp32_platformio_components_to_test", return_value=components_to_test, ): - assert determine_jobs.should_run_native_idf() is expected + assert determine_jobs.should_run_esp32_platformio() is expected -def test_should_run_native_idf_with_branch() -> None: - """Test should_run_native_idf passes branch argument through.""" +def test_should_run_esp32_platformio_with_branch() -> None: + """Test should_run_esp32_platformio passes branch argument through.""" with patch.object( - determine_jobs, "native_idf_components_to_test", return_value=[] + determine_jobs, "esp32_platformio_components_to_test", return_value=[] ) as mock_inner: - determine_jobs.should_run_native_idf("release") + determine_jobs.should_run_esp32_platformio("release") mock_inner.assert_called_once_with("release") +@pytest.mark.parametrize( + ("changed_files", "expected"), + [ + # ESP-IDF runner / framework / build generator -> trigger + (["esphome/espidf/runner.py"], True), + (["esphome/espidf/framework.py"], True), + (["esphome/build_gen/espidf.py"], True), + # PlatformIO build gen and esp32 component are NOT IDF-infra triggers + (["esphome/build_gen/platformio.py"], False), + (["esphome/components/esp32/__init__.py"], False), + (["README.md"], False), + ([], False), + ], +) +def test_esp_idf_infra_changed(changed_files: list[str], expected: bool) -> None: + """ESP-IDF build/runner infra paths are detected; other paths are not.""" + assert determine_jobs._esp_idf_infra_changed(changed_files) is expected + + @pytest.mark.parametrize( ("changed_files", "expected_result"), [ @@ -2751,7 +2830,7 @@ def test_main_force_all_overrides_detection( mock_should_run_python_linters: Mock, mock_should_run_import_time: Mock, mock_should_run_device_builder: Mock, - mock_native_idf_components_to_test: Mock, + mock_esp32_platformio_components_to_test: Mock, mock_determine_cpp_unit_tests: Mock, mock_changed_files: Mock, capsys: pytest.CaptureFixture[str], @@ -2772,7 +2851,7 @@ def test_main_force_all_overrides_detection( mock_should_run_python_linters.return_value = False mock_should_run_import_time.return_value = False mock_should_run_device_builder.return_value = False - mock_native_idf_components_to_test.return_value = [] + mock_esp32_platformio_components_to_test.return_value = [] mock_determine_cpp_unit_tests.return_value = (False, []) mock_changed_files.return_value = [] @@ -2813,9 +2892,9 @@ def test_main_force_all_overrides_detection( assert output["python_linters"] is True assert output["import_time"] is True assert output["device_builder"] is True - assert output["native_idf"] is True - # native_idf_components is a CSV of NATIVE_IDF_TEST_COMPONENTS - assert "esp32" in output["native_idf_components"].split(",") + assert output["esp32_platformio"] is True + # esp32_platformio_components is a CSV of ESP32_PLATFORMIO_TEST_COMPONENTS + assert "esp32" in output["esp32_platformio_components"].split(",") assert output["cpp_unit_tests_run_all"] is True assert output["cpp_unit_tests_components"] == [] assert output["benchmarks"] is True @@ -2826,7 +2905,7 @@ def test_main_force_all_overrides_detection( mock_should_run_python_linters.assert_not_called() mock_should_run_import_time.assert_not_called() mock_should_run_device_builder.assert_not_called() - mock_native_idf_components_to_test.assert_not_called() + mock_esp32_platformio_components_to_test.assert_not_called() mock_determine_cpp_unit_tests.assert_not_called() # Component matrix is populated from disk (tests/components/ in the repo) assert output["component_test_count"] > 0 @@ -2840,7 +2919,7 @@ def test_main_force_all_off_uses_detection( mock_should_run_python_linters: Mock, mock_should_run_import_time: Mock, mock_should_run_device_builder: Mock, - mock_native_idf_components_to_test: Mock, + mock_esp32_platformio_components_to_test: Mock, mock_determine_cpp_unit_tests: Mock, mock_changed_files: Mock, capsys: pytest.CaptureFixture[str], @@ -2855,7 +2934,7 @@ def test_main_force_all_off_uses_detection( mock_should_run_python_linters.return_value = False mock_should_run_import_time.return_value = False mock_should_run_device_builder.return_value = False - mock_native_idf_components_to_test.return_value = [] + mock_esp32_platformio_components_to_test.return_value = [] mock_determine_cpp_unit_tests.return_value = (False, []) mock_changed_files.return_value = [] @@ -2886,7 +2965,7 @@ def test_main_force_all_off_uses_detection( assert output["clang_tidy"] is False assert output["clang_format"] is False assert output["python_linters"] is False - assert output["native_idf"] is False + assert output["esp32_platformio"] is False assert output["component_test_count"] == 0 mock_determine_integration_tests.assert_called_once() mock_should_run_clang_tidy.assert_called_once()