From aae5538fd56665c01c839418470a02ed5d216a23 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 21 Aug 2026 00:05:23 -0500 Subject: [PATCH] Gate uploads on a built firmware, refuse unclaimed native builds, seed the CI toolchain cache from dev The upload path now checks CORE.firmware_bin exists before invoking esptool and names the fix (compile first) instead of failing inside esptool. compile_program raises when a native toolchain resolved but no platform backend claimed the build rather than falling through to a mis-configured PlatformIO project. The addr2line failure warning now includes the exception so the cause is visible without debug logging. CI: the native toolchain cache is restore-only on PRs and seeded by a new dev-push job, mirroring seed-apt-cache; the key resolves the pinned core and toolchain versions from code instead of hashing framework.py. determine-jobs main() output tests now pin the esp8266_native pair. --- .github/workflows/ci.yml | 55 +++++++++++++++++++++++--- esphome/__main__.py | 13 +++++- esphome/components/esp8266/__init__.py | 7 ++-- tests/script/test_determine_jobs.py | 19 +++++++++ tests/unit_tests/test_main.py | 25 ++++++++++++ 5 files changed, 110 insertions(+), 9 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ffcbb59ecc..83d3692b76 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -177,6 +177,41 @@ jobs: path: .temp/components_graph.json key: components-graph-${{ hashFiles('esphome/components/**/*.py') }} + seed-esp8266-native-cache: + name: Seed the esp8266 native toolchain cache + runs-on: ubuntu-24.04 + needs: + - common + # PR-branch cache saves are invisible to other PRs, so pushes seed the + # shared entry test-esp8266-native restores (same pattern as + # seed-apt-cache / cache-esp-idf). + if: github.event_name == 'push' + timeout-minutes: 15 + steps: + - name: Check out code from GitHub + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: Restore Python + uses: ./.github/actions/restore-python + with: + python-version: ${{ env.DEFAULT_PYTHON }} + cache-key: ${{ needs.common.outputs.cache-key }} + - name: Resolve the native toolchain cache key + id: esp8266-native-cache-key + run: | + . venv/bin/activate + echo "key=esp8266-native-$(python -c 'from esphome.components.esp8266 import RECOMMENDED_ARDUINO_FRAMEWORK_VERSION as f; from esphome.arduino8266.framework import TOOLCHAIN_VERSION as t; print(f"{f}-{t}")')" >> $GITHUB_OUTPUT + - name: Cache the native toolchain + id: esp8266-native-cache + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: ~/.cache/esphome/arduino8266 + key: ${{ steps.esp8266-native-cache-key.outputs.key }} + - name: Install the native toolchain + if: steps.esp8266-native-cache.outputs.cache-hit != 'true' + run: | + . venv/bin/activate + python -c "from esphome.arduino8266.framework import check_and_install; from esphome.components.esp8266 import RECOMMENDED_ARDUINO_FRAMEWORK_VERSION; check_and_install(RECOMMENDED_ARDUINO_FRAMEWORK_VERSION)" + ci-custom: name: Run script/ci-custom runs-on: ubuntu-24.04 @@ -1152,13 +1187,22 @@ jobs: python-version: ${{ env.DEFAULT_PYTHON }} cache-key: ${{ needs.common.outputs.cache-key }} - # ~110 MB of framework + toolchain plus the ccache store; keyed on the - # pinned versions in arduino8266/framework.py so a bump re-downloads - - name: Cache the native toolchain and ccache - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + # ~110 MB of framework + toolchain plus the ccache store. The versions + # are pinned in code, not in a hashable file, so resolve them for the + # key (actions/cache never overwrites a key, so a bump must change it). + # PRs are restore-only; the shared entry is seeded on pushes to dev by + # seed-esp8266-native-cache, mirroring the seed-apt-cache scoping. + - name: Resolve the native toolchain cache key + id: esp8266-native-cache-key + run: | + . venv/bin/activate + echo "key=esp8266-native-$(python -c 'from esphome.components.esp8266 import RECOMMENDED_ARDUINO_FRAMEWORK_VERSION as f; from esphome.arduino8266.framework import TOOLCHAIN_VERSION as t; print(f"{f}-{t}")')" >> $GITHUB_OUTPUT + + - name: Restore the native toolchain and ccache + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: ~/.cache/esphome/arduino8266 - key: esp8266-native-${{ hashFiles('esphome/arduino8266/framework.py') }} + key: ${{ steps.esp8266-native-cache-key.outputs.key }} - name: Run native toolchain compile test run: | @@ -1533,6 +1577,7 @@ jobs: needs: - common - seed-apt-cache + - seed-esp8266-native-cache - determine-jobs - ci-custom - pylint diff --git a/esphome/__main__.py b/esphome/__main__.py index 15e7ddc6c5..acb2a9ef46 100644 --- a/esphome/__main__.py +++ b/esphome/__main__.py @@ -849,6 +849,13 @@ def compile_program(args: ArgsProtocol, config: ConfigType) -> int: platform_run_compile = getattr(module, "run_compile", None) if platform_run_compile is not None and platform_run_compile(args, config): pass + elif CORE.using_native_toolchain and not CORE.using_toolchain_esp_idf: + # A resolved native toolchain must be claimed by its platform hook; + # falling through would build a mis-configured PlatformIO project + raise EsphomeError( + f"Toolchain '{CORE.toolchain.value}' resolved but no platform " + "backend claimed the build" + ) elif CORE.using_toolchain_esp_idf: from esphome.espidf import toolchain @@ -984,9 +991,13 @@ def upload_using_esptool( flash_images = [ FlashImage(path=toolchain.get_factory_firmware_path(), offset="0x0") ] - elif CORE.using_toolchain_arduino: + elif CORE.using_native_toolchain: # The native backend writes PlatformIO-compatible output paths, so the # shared property already points at the right file. + if not CORE.firmware_bin.is_file(): + raise EsphomeError( + f"{CORE.firmware_bin} does not exist; compile the configuration first" + ) flash_images = [FlashImage(path=CORE.firmware_bin, offset="0x0")] else: from esphome.platformio import toolchain diff --git a/esphome/components/esp8266/__init__.py b/esphome/components/esp8266/__init__.py index fe6474ca18..86c98db2db 100644 --- a/esphome/components/esp8266/__init__.py +++ b/esphome/components/esp8266/__init__.py @@ -650,11 +650,12 @@ def _decode_pc(config, addr): command = [addr2line, "-pfiaC", "-e", elf, addr] try: translation = subprocess.check_output(command, close_fds=False).decode().strip() - except Exception: # noqa: BLE001 # pylint: disable=broad-except + except Exception as err: # noqa: BLE001 # pylint: disable=broad-except # A present-but-failing addr2line (stale ELF, bad install) must be - # visible on either toolchain, matching the missing-tool warning above + # visible on either toolchain, matching the missing-tool warning + # above, and the cause must not need debug logging to see _warn_decode_problem( - "addr2line-failed", "Could not decode crash address %s", addr + "addr2line-failed", "Could not decode crash address %s (%s)", addr, err ) _LOGGER.debug("Caught exception for command %s", command, exc_info=1) return diff --git a/tests/script/test_determine_jobs.py b/tests/script/test_determine_jobs.py index 9b44283d76..f1f2ead81a 100644 --- a/tests/script/test_determine_jobs.py +++ b/tests/script/test_determine_jobs.py @@ -78,6 +78,17 @@ def mock_esp32_platformio_components_to_test() -> Generator[Mock, None, None]: yield mock +@pytest.fixture +def mock_esp8266_native_components_to_test() -> Generator[Mock, None, None]: + """Mock esp8266_native_components_to_test from determine_jobs. + + main() drives both the ``esp8266_native`` boolean output and the + ``esp8266_native_components`` CSV from this one function. + """ + with patch.object(determine_jobs, "esp8266_native_components_to_test") as mock: + yield mock + + @pytest.fixture def mock_determine_cpp_unit_tests() -> Generator[Mock, None, None]: """Mock determine_cpp_unit_tests from helpers.""" @@ -116,6 +127,7 @@ def test_main_all_tests_should_run( mock_should_run_import_time: Mock, mock_should_run_device_builder: Mock, mock_esp32_platformio_components_to_test: Mock, + mock_esp8266_native_components_to_test: Mock, mock_changed_files: Mock, mock_determine_cpp_unit_tests: Mock, capsys: pytest.CaptureFixture[str], @@ -132,6 +144,7 @@ def test_main_all_tests_should_run( mock_should_run_import_time.return_value = True mock_should_run_device_builder.return_value = True mock_esp32_platformio_components_to_test.return_value = ["api", "esp32"] + mock_esp8266_native_components_to_test.return_value = ["api", "logger"] mock_determine_cpp_unit_tests.return_value = (False, ["wifi", "api", "sensor"]) # Mock changed_files to return non-component files (to avoid memory impact) @@ -215,6 +228,8 @@ def test_main_all_tests_should_run( assert output["device_builder"] is True assert output["esp32_platformio"] is True assert output["esp32_platformio_components"] == "api,esp32" + assert output["esp8266_native"] is True + assert output["esp8266_native_components"] == "api,logger" assert output["changed_components"] == ["wifi", "api", "sensor"] # changed_components_with_tests will only include components that actually have test files assert "changed_components_with_tests" in output @@ -251,6 +266,7 @@ def test_main_no_tests_should_run( mock_should_run_import_time: Mock, mock_should_run_device_builder: Mock, mock_esp32_platformio_components_to_test: Mock, + mock_esp8266_native_components_to_test: Mock, mock_changed_files: Mock, mock_determine_cpp_unit_tests: Mock, capsys: pytest.CaptureFixture[str], @@ -267,6 +283,7 @@ def test_main_no_tests_should_run( mock_should_run_import_time.return_value = False mock_should_run_device_builder.return_value = False mock_esp32_platformio_components_to_test.return_value = [] + mock_esp8266_native_components_to_test.return_value = [] mock_determine_cpp_unit_tests.return_value = (False, []) # Mock changed_files to return no component files @@ -309,6 +326,8 @@ def test_main_no_tests_should_run( assert output["device_builder"] is False assert output["esp32_platformio"] is False assert output["esp32_platformio_components"] == "" + assert output["esp8266_native"] is False + assert output["esp8266_native_components"] == "" assert output["changed_components"] == [] assert output["changed_components_with_tests"] == [] assert output["component_test_count"] == 0 diff --git a/tests/unit_tests/test_main.py b/tests/unit_tests/test_main.py index 501e0545e7..a809d3b4c8 100644 --- a/tests/unit_tests/test_main.py +++ b/tests/unit_tests/test_main.py @@ -7139,6 +7139,8 @@ def test_upload_using_esptool_arduino_toolchain( """The native ESP8266 Arduino toolchain flashes CORE.firmware_bin at 0x0.""" setup_core(platform=PLATFORM_ESP8266, tmp_path=tmp_path, name="test") CORE.toolchain = Toolchain.ARDUINO + CORE.firmware_bin.parent.mkdir(parents=True, exist_ok=True) + CORE.firmware_bin.touch() config = {CONF_ESPHOME: {"platformio_options": {}}} result = upload_using_esptool(config, "/dev/ttyUSB0", None, None) @@ -7404,3 +7406,26 @@ def test_cli_toolchain_skips_the_validated_config_cache(tmp_path: Path) -> None: assert run_esphome(argv) == 2 mock_cache.assert_not_called() mock_read.assert_called_once() + + +def test_upload_using_esptool_native_missing_firmware_raises( + tmp_path: Path, +) -> None: + """A stale or absent firmware.bin fails by name instead of flashing air.""" + setup_core(platform=PLATFORM_ESP8266, tmp_path=tmp_path, name="test") + CORE.toolchain = Toolchain.ARDUINO + with pytest.raises(EsphomeError, match="compile the configuration first"): + upload_using_esptool( + {CONF_ESPHOME: {"platformio_options": {}}}, "/dev/ttyUSB0", None, None + ) + + +def test_compile_program_unclaimed_native_toolchain_raises( + tmp_path: Path, +) -> None: + """A resolved native toolchain no platform backend claims must fail, + never fall through to the PlatformIO project path.""" + setup_core(platform=PLATFORM_ESP32, tmp_path=tmp_path, name="test_device") + CORE.toolchain = Toolchain.ARDUINO # esp32 has no arduino-native backend + with pytest.raises(EsphomeError, match="no platform backend claimed"): + compile_program(MockArgs(), {})