Merge branch 'dev' into esp8266-arduino-toolchain

tests/unit_tests/build_gen/test_espidf.py

@@ -11,6 +11,7 @@ import pytest
 from esphome.components.esp32 import (
     KEY_COMPONENTS,
     KEY_ESP32,
+    KEY_EXCLUDE_COMPONENTS,
     KEY_IDF_VERSION,
     KEY_PATH,
     KEY_REF,
@@ -28,6 +29,7 @@ def _reset_core(tmp_path: Path) -> None:
     CORE.data.setdefault(KEY_CORE, {})
     CORE.data[KEY_ESP32] = {
         KEY_COMPONENTS: {},
+        KEY_EXCLUDE_COMPONENTS: set(),
         KEY_IDF_VERSION: cv.Version(5, 5, 4),
     }
 
@@ -47,6 +49,17 @@ def _write_project_description(tmp_path: Path, components: dict[str, str]) -> No
     )
 
 
+def _render(minimal: bool = False) -> str:
+    """Render the top-level CMakeLists with the standard variant/name patches."""
+    with (
+        patch("esphome.build_gen.espidf.get_esp32_variant", return_value="ESP32"),
+        patch.object(CORE, "name", "test"),
+    ):
+        from esphome.build_gen.espidf import get_project_cmakelists
+
+        return get_project_cmakelists(minimal=minimal)
+
+
 def test_get_available_components_returns_none_without_build_path() -> None:
     """No build_path set yet: must not raise on Path(None)."""
     CORE.build_path = None
@@ -88,13 +101,7 @@ def test_get_project_cmakelists_minimal_omits_builtin_components_property(
     first write before the discovery pass refreshes it)."""
     _write_project_description(tmp_path, {"esp_lcd": "/idf/components/esp_lcd"})
 
-    with (
-        patch("esphome.build_gen.espidf.get_esp32_variant", return_value="ESP32"),
-        patch.object(CORE, "name", "test"),
-    ):
-        from esphome.build_gen.espidf import get_project_cmakelists
-
-        content = get_project_cmakelists(minimal=True)
+    content = _render(minimal=True)
 
     assert "ESPHOME_PROJECT_BUILTIN_COMPONENTS" not in content
 
@@ -115,13 +122,7 @@ def test_get_project_cmakelists_full_emits_builtin_components_property(
         },
     )
 
-    with (
-        patch("esphome.build_gen.espidf.get_esp32_variant", return_value="ESP32"),
-        patch.object(CORE, "name", "test"),
-    ):
-        from esphome.build_gen.espidf import get_project_cmakelists
-
-        content = get_project_cmakelists(minimal=False)
+    content = _render()
 
     assert (
         "idf_build_set_property(ESPHOME_PROJECT_BUILTIN_COMPONENTS esp_lcd APPEND)"
@@ -136,6 +137,92 @@ def test_get_project_cmakelists_full_emits_builtin_components_property(
     assert "JPEGDEC APPEND" not in content
 
 
+def test_get_project_cmakelists_emits_exclude_components(tmp_path: Path) -> None:
+    """Excluded components are passed to IDF via EXCLUDE_COMPONENTS and are
+    dropped from ESPHOME_PROJECT_BUILTIN_COMPONENTS even when a stale
+    project_description.json still lists them (requiring an excluded
+    component would pull it back into the build)."""
+    _write_project_description(
+        tmp_path,
+        {
+            "esp_lcd": "/idf/components/esp_lcd",
+            "freertos": "/idf/components/freertos",
+            "unity": "/idf/components/unity",
+        },
+    )
+    CORE.data[KEY_ESP32][KEY_EXCLUDE_COMPONENTS] = {"unity", "esp_lcd"}
+
+    content = _render()
+
+    assert 'set(EXCLUDE_COMPONENTS "esp_lcd;unity")' in content
+    # Must be set before project() so project.cmake sees it.
+    assert content.index("set(EXCLUDE_COMPONENTS") < content.index("project(test)")
+    assert (
+        "idf_build_set_property(ESPHOME_PROJECT_BUILTIN_COMPONENTS freertos APPEND)"
+        in content
+    )
+    assert "ESPHOME_PROJECT_BUILTIN_COMPONENTS unity" not in content
+    assert "ESPHOME_PROJECT_BUILTIN_COMPONENTS esp_lcd" not in content
+
+
+def test_get_project_cmakelists_minimal_emits_exclude_components() -> None:
+    """The discovery (minimal) write also excludes components so they never
+    register in project_description.json."""
+    CORE.data[KEY_ESP32][KEY_EXCLUDE_COMPONENTS] = {"unity"}
+
+    content = _render(minimal=True)
+
+    assert 'set(EXCLUDE_COMPONENTS "unity")' in content
+
+
+def test_get_project_cmakelists_no_exclude_components_line_when_empty() -> None:
+    """No EXCLUDE_COMPONENTS line at all when nothing is excluded."""
+    content = _render()
+
+    assert "EXCLUDE_COMPONENTS" not in content
+
+
+def test_include_builtin_idf_component_removes_exclusion() -> None:
+    """include_builtin_idf_component() drops a name from the exclusion set so
+    a component a config actually uses is not passed to EXCLUDE_COMPONENTS."""
+    from esphome.components.esp32 import (
+        exclude_builtin_idf_component,
+        get_excluded_builtin_components,
+        include_builtin_idf_component,
+    )
+
+    exclude_builtin_idf_component("esp_eth")
+    exclude_builtin_idf_component("unity")
+    include_builtin_idf_component("esp_eth")
+
+    assert get_excluded_builtin_components() == ["unity"]
+
+    content = _render()
+
+    assert 'set(EXCLUDE_COMPONENTS "unity")' in content
+    assert "esp_eth" not in content
+
+
+def test_write_project_writes_exclude_components_stamp(tmp_path: Path) -> None:
+    """write_project() snapshots the exclusion set; the toolchain watches the
+    stamp to trigger a discovery reconfigure when the set changes (excluded
+    components never register in project_description.json)."""
+    CORE.build_flags = set()
+    CORE.build_path = tmp_path
+    CORE.data[KEY_ESP32][KEY_EXCLUDE_COMPONENTS] = {"unity", "esp_lcd"}
+
+    with (
+        patch("esphome.build_gen.espidf.get_esp32_variant", return_value="ESP32"),
+        patch.object(CORE, "name", "test"),
+    ):
+        from esphome.build_gen.espidf import write_project
+
+        write_project()
+
+    stamp = tmp_path / "exclude_components.esphomeinternal"
+    assert stamp.read_text() == "esp_lcd;unity"
+
+
 def test_get_component_cmakelists_no_link_flags() -> None:
     """With no -Wl, flags the target_link_options block is emitted with an empty body."""
     CORE.build_flags = set()

tests/unit_tests/test_espidf_toolchain.py

@@ -100,6 +100,33 @@ def _setup_build(setup_core: Path) -> tuple[Path, Path]:
     return compile_commands, cache
 
 
+def test_has_outdated_files_detects_exclusion_change(setup_core: Path) -> None:
+    """A newer exclude_components.esphomeinternal stamp forces a reconfigure
+    so components that leave the exclusion set get rediscovered."""
+    CORE.build_path = setup_core
+    build = setup_core / "build"
+    (build / "config").mkdir(parents=True)
+    (build / "config" / "sdkconfig.h").write_text("")
+    cmakecache = build / "CMakeCache.txt"
+    cmakecache.write_text("")
+    (build / "build.ninja").write_text("")
+
+    with patch.object(CORE, "name", "test"):
+        assert not toolchain.has_outdated_files()
+
+        stamp = setup_core / "exclude_components.esphomeinternal"
+        stamp.write_text("unity")
+        os.utime(stamp, (cmakecache.stat().st_mtime + 10,) * 2)
+
+        assert toolchain.has_outdated_files()
+
+        # The flag must clear once the reference file is restamped (as
+        # run_compile does after a successful discovery reconfigure);
+        # otherwise every later build would repeat the discovery pass.
+        os.utime(cmakecache, (stamp.stat().st_mtime + 10,) * 2)
+        assert not toolchain.has_outdated_files()
+
+
 def test_get_idedata_returns_none_without_compile_commands(setup_core: Path) -> None:
     """No compile DB yet -> None (rather than an error)."""
     _setup_build(setup_core)
@@ -373,6 +400,48 @@ def test_run_idf_py_jobs_sets_build_jobs_env(setup_core: Path) -> None:
         assert "IDF_PY_BUILD_JOBS" not in env
 
 
+def test_run_compile_restamps_cmakecache_after_discovery(setup_core: Path) -> None:
+    """After a successful discovery reconfigure the reference CMakeCache.txt
+    is restamped; cmake does not rewrite it when only properties or plain
+    variables change, so the staleness flag would otherwise never clear."""
+    _setup_build(setup_core)
+    config = {CONF_ESPHOME: {}}
+    cmakecache = CORE.relative_build_path("build/CMakeCache.txt")
+    cmakecache.parent.mkdir(parents=True, exist_ok=True)
+    cmakecache.write_text("")
+    old = cmakecache.stat().st_mtime - 100
+    os.utime(cmakecache, (old, old))
+
+    with (
+        patch.object(toolchain, "need_reconfigure", return_value=True),
+        patch("esphome.build_gen.espidf.write_project"),
+        patch.object(toolchain, "run_reconfigure", return_value=0),
+        patch.object(toolchain, "run_idf_py", return_value=0),
+        patch.object(toolchain, "print_summary"),
+    ):
+        assert toolchain.run_compile(config, verbose=False) == 0
+
+    assert cmakecache.stat().st_mtime > old
+
+
+def test_run_compile_discovery_without_cmakecache(setup_core: Path) -> None:
+    """A discovery pass that produced no CMakeCache.txt (nothing to restamp)
+    still completes normally."""
+    _setup_build(setup_core)
+    config = {CONF_ESPHOME: {}}
+
+    with (
+        patch.object(toolchain, "need_reconfigure", return_value=True),
+        patch("esphome.build_gen.espidf.write_project"),
+        patch.object(toolchain, "run_reconfigure", return_value=0),
+        patch.object(toolchain, "run_idf_py", return_value=0),
+        patch.object(toolchain, "print_summary"),
+    ):
+        assert toolchain.run_compile(config, verbose=False) == 0
+
+    assert not CORE.relative_build_path("build/CMakeCache.txt").exists()
+
+
 def test_run_compile_passes_compile_process_limit(setup_core: Path) -> None:
     """compile_process_limit is forwarded to run_idf_py as the job limit."""
     _setup_build(setup_core)

tests/unit_tests/test_nrf52_framework.py

@@ -16,6 +16,7 @@ from esphome.components.nrf52.framework import (
     _get_penv_site_packages,
     _get_platformio_penv_path,
     _get_toolchain_platform_info,
+    _needs_venv_rebuild,
     check_and_install,
     get_build_env,
     get_sdk_nrf_tools_path,
@@ -123,10 +124,19 @@ def mock_nrf52_ops():
 # ---------------------------------------------------------------------------
 
 
+def _touch_penv_python(penv: Path) -> None:
+    """Create the interpreter file so the rebuild gate sees a live venv."""
+    python = get_python_env_executable_path(penv, "python")
+    python.parent.mkdir(parents=True, exist_ok=True)
+    python.touch()
+
+
 def _mark_venv_ready(python_env: Path) -> None:
-    """Write the venv sentinel with the current requirements hash."""
+    """Write the venv sentinel with the current requirements hash and a
+    present interpreter so the rebuild gate passes."""
     requirements_hash = hashlib.sha256(_REQUIREMENTS.read_bytes()).hexdigest()
     (python_env / ".ready").write_text(requirements_hash, encoding="utf-8")
+    _touch_penv_python(python_env)
 
 
 class TestCheckAndInstall:
@@ -148,6 +158,23 @@ class TestCheckAndInstall:
         mock_nrf52_ops.download_from_mirrors.assert_not_called()
         mock_nrf52_ops.archive_extract_all.assert_not_called()
 
+    def test_missing_interpreter_rebuilds_venv(
+        self,
+        nrf52_dirs: SimpleNamespace,
+        mock_nrf52_ops: SimpleNamespace,
+    ) -> None:
+        """A valid sentinel must not mask a missing interpreter (a cached venv
+        restored after a host interpreter upgrade)."""
+        requirements_hash = hashlib.sha256(_REQUIREMENTS.read_bytes()).hexdigest()
+        (nrf52_dirs.python_env / ".ready").write_text(
+            requirements_hash, encoding="utf-8"
+        )
+        # no interpreter on disk
+
+        check_and_install()
+
+        mock_nrf52_ops.create_venv.assert_called_once()
+
     def test_fresh_install_runs_all_steps(
         self,
         nrf52_dirs: SimpleNamespace,
@@ -348,6 +375,7 @@ class TestSetupPlatformioPythonEnv:
         (platformio_penv_dir / ".ready").write_text(
             _platformio_requirements_hash(), encoding="utf-8"
         )
+        _touch_penv_python(platformio_penv_dir)
 
         with patch.dict(os.environ):
             setup_platformio_python_env()
@@ -392,6 +420,22 @@ class TestSetupPlatformioPythonEnv:
 
         assert not (platformio_penv_dir / ".ready").exists()
 
+    def test_missing_interpreter_reinstalls(
+        self,
+        platformio_penv_dir: Path,
+        mock_nrf52_ops: SimpleNamespace,
+    ) -> None:
+        """A valid sentinel must not mask a missing interpreter."""
+        (platformio_penv_dir / ".ready").write_text(
+            _platformio_requirements_hash(), encoding="utf-8"
+        )
+        # no interpreter on disk
+
+        with patch.dict(os.environ):
+            setup_platformio_python_env()
+
+        mock_nrf52_ops.create_venv.assert_called_once()
+
     def test_repeated_calls_do_not_duplicate_env_entries(
         self,
         platformio_penv_dir: Path,
@@ -401,6 +445,7 @@ class TestSetupPlatformioPythonEnv:
         (platformio_penv_dir / ".ready").write_text(
             _platformio_requirements_hash(), encoding="utf-8"
         )
+        _touch_penv_python(platformio_penv_dir)
         site_packages = str(_get_penv_site_packages(platformio_penv_dir))
         bin_dir = str(
             get_python_env_executable_path(platformio_penv_dir, "python").parent
@@ -422,6 +467,7 @@ class TestSetupPlatformioPythonEnv:
         (platformio_penv_dir / ".ready").write_text(
             _platformio_requirements_hash(), encoding="utf-8"
         )
+        _touch_penv_python(platformio_penv_dir)
         site_packages = str(_get_penv_site_packages(platformio_penv_dir))
 
         with patch.dict(os.environ, {"PYTHONPATH": "/existing/path"}):
@@ -533,6 +579,48 @@ def testget_tools_path_default_is_global_cache(
     assert get_sdk_nrf_tools_path() == expected
 
 
+def test_needs_venv_rebuild_gates(tmp_path: Path) -> None:
+    """The shared penv gate rebuilds on any missing or stale piece."""
+    penv = tmp_path / "penv"
+    penv.mkdir()
+    python = penv / "python"
+    sentinel = penv / ".ready"
+    good_hash = "abc123"
+
+    # Nothing in place yet
+    assert _needs_venv_rebuild(python, sentinel, good_hash)
+
+    python.write_text("")
+    # Interpreter present but no sentinel
+    assert _needs_venv_rebuild(python, sentinel, good_hash)
+
+    sentinel.write_text(good_hash, encoding="utf-8")
+    # Everything in place
+    assert not _needs_venv_rebuild(python, sentinel, good_hash)
+
+    # Stale requirements hash
+    assert _needs_venv_rebuild(python, sentinel, "otherhash")
+
+
+@pytest.mark.skipif(
+    sys.platform == "win32", reason="symlink creation needs privileges on Windows"
+)
+def test_needs_venv_rebuild_on_dangling_interpreter_symlink(tmp_path: Path) -> None:
+    """A cached venv restored after a host interpreter upgrade has a
+    bin/python symlink whose target is gone; the valid sentinel must not
+    mask it."""
+    penv = tmp_path / "penv"
+    penv.mkdir()
+    python = penv / "python"
+    sentinel = penv / ".ready"
+    sentinel.write_text("abc123", encoding="utf-8")
+    python.symlink_to(tmp_path / "hostedtoolcache" / "3.12.14" / "python3")
+    assert python.is_symlink()
+    assert not python.exists()
+
+    assert _needs_venv_rebuild(python, sentinel, "abc123")
+
+
 def test_resolve_toolchain_rejects_unsupported() -> None:
     """A --toolchain nRF52 cannot serve fails instead of degrading silently."""
     from esphome.components.nrf52 import _resolve_toolchain