Merge remote-tracking branch 'upstream/dev' into integration

tests/unit_tests/build_gen/test_espidf.py

@@ -0,0 +1,159 @@
+"""Tests for esphome.build_gen.espidf module."""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from unittest.mock import patch
+
+import pytest
+
+from esphome.components.esp32 import (
+    KEY_COMPONENTS,
+    KEY_ESP32,
+    KEY_PATH,
+    KEY_REF,
+    KEY_REPO,
+)
+from esphome.const import KEY_CORE
+from esphome.core import CORE
+
+
+@pytest.fixture(autouse=True)
+def _reset_core(tmp_path: Path) -> None:
+    """Give each test its own CORE.build_path and a clean esp32 data slot."""
+    CORE.build_path = str(tmp_path)
+    CORE.data.setdefault(KEY_CORE, {})
+    CORE.data[KEY_ESP32] = {KEY_COMPONENTS: {}}
+
+
+def _write_project_description(tmp_path: Path, components: dict[str, str]) -> None:
+    """Stub a project_description.json with the given component_name -> dir map."""
+    build_dir = tmp_path / "build"
+    build_dir.mkdir(exist_ok=True)
+    (build_dir / "project_description.json").write_text(
+        json.dumps(
+            {
+                "build_component_info": {
+                    name: {"dir": dir_} for name, dir_ in components.items()
+                }
+            }
+        )
+    )
+
+
+def test_get_available_components_returns_none_without_build_path() -> None:
+    """No build_path set yet: must not raise on Path(None)."""
+    CORE.build_path = None
+    from esphome.build_gen.espidf import get_available_components
+
+    assert get_available_components() is None
+
+
+def test_get_available_components_returns_none_without_project_description(
+    tmp_path: Path,
+) -> None:
+    from esphome.build_gen.espidf import get_available_components
+
+    assert get_available_components() is None
+
+
+def test_get_available_components_filters_src_managed_and_pio(tmp_path: Path) -> None:
+    """Built-ins are returned; src/, managed_components/, pio_components/ skipped."""
+    _write_project_description(
+        tmp_path,
+        {
+            "src": f"{tmp_path}/src",
+            "esp_lcd": "/idf/components/esp_lcd",
+            "espressif__arduino-esp32": f"{tmp_path}/managed_components/arduino",
+            "JPEGDEC": f"{tmp_path}/pio_components/arduino/abc/bitbank2/JPEGDEC",
+            "freertos": "/idf/components/freertos",
+        },
+    )
+    from esphome.build_gen.espidf import get_available_components
+
+    assert sorted(get_available_components()) == ["esp_lcd", "freertos"]
+
+
+def test_get_project_cmakelists_minimal_omits_builtin_components_property(
+    tmp_path: Path,
+) -> None:
+    """Minimal write must not emit ESPHOME_PROJECT_BUILTIN_COMPONENTS even
+    when project_description.json exists (the data may be stale on the
+    first write before the discovery pass refreshes it)."""
+    _write_project_description(tmp_path, {"esp_lcd": "/idf/components/esp_lcd"})
+
+    with (
+        patch("esphome.build_gen.espidf.get_esp32_variant", return_value="ESP32"),
+        patch.object(CORE, "name", "test"),
+    ):
+        from esphome.build_gen.espidf import get_project_cmakelists
+
+        content = get_project_cmakelists(minimal=True)
+
+    assert "ESPHOME_PROJECT_BUILTIN_COMPONENTS" not in content
+
+
+def test_get_project_cmakelists_full_emits_builtin_components_property(
+    tmp_path: Path,
+) -> None:
+    """Non-minimal write emits one idf_build_set_property line per built-in,
+    sorted, and excludes src/managed/pio components."""
+    _write_project_description(
+        tmp_path,
+        {
+            "src": f"{tmp_path}/src",
+            "esp_lcd": "/idf/components/esp_lcd",
+            "freertos": "/idf/components/freertos",
+            "espressif__esp-dsp": f"{tmp_path}/managed_components/esp-dsp",
+            "JPEGDEC": f"{tmp_path}/pio_components/arduino/abc/bitbank2/JPEGDEC",
+        },
+    )
+
+    with (
+        patch("esphome.build_gen.espidf.get_esp32_variant", return_value="ESP32"),
+        patch.object(CORE, "name", "test"),
+    ):
+        from esphome.build_gen.espidf import get_project_cmakelists
+
+        content = get_project_cmakelists(minimal=False)
+
+    assert (
+        "idf_build_set_property(ESPHOME_PROJECT_BUILTIN_COMPONENTS esp_lcd APPEND)"
+        in content
+    )
+    assert (
+        "idf_build_set_property(ESPHOME_PROJECT_BUILTIN_COMPONENTS freertos APPEND)"
+        in content
+    )
+    # Excluded by get_available_components filtering.
+    assert "espressif__esp-dsp APPEND" not in content
+    assert "JPEGDEC APPEND" not in content
+
+
+def test_get_project_cmakelists_emits_managed_components_property(
+    tmp_path: Path,
+) -> None:
+    """ESPHOME_PROJECT_MANAGED_COMPONENTS is always emitted (both modes)
+    from the esp32 add_idf_component registry."""
+    CORE.data[KEY_ESP32][KEY_COMPONENTS] = {
+        "espressif/esp-dsp": {KEY_REPO: None, KEY_REF: "1.7.1", KEY_PATH: None},
+        "espressif/arduino-esp32": {KEY_REPO: None, KEY_REF: "3.3.8", KEY_PATH: None},
+    }
+
+    with (
+        patch("esphome.build_gen.espidf.get_esp32_variant", return_value="ESP32"),
+        patch.object(CORE, "name", "test"),
+    ):
+        from esphome.build_gen.espidf import get_project_cmakelists
+
+        for minimal in (True, False):
+            content = get_project_cmakelists(minimal=minimal)
+            assert (
+                "idf_build_set_property(ESPHOME_PROJECT_MANAGED_COMPONENTS"
+                " espressif__arduino-esp32 APPEND)"
+            ) in content
+            assert (
+                "idf_build_set_property(ESPHOME_PROJECT_MANAGED_COMPONENTS"
+                " espressif__esp-dsp APPEND)"
+            ) in content

tests/unit_tests/test_espidf_component.py

@@ -1,5 +1,6 @@
 import json
 import os
+from pathlib import Path
 from unittest.mock import MagicMock
 
 import pytest
@@ -21,7 +22,6 @@ from esphome.espidf.component import (
     _check_library_data,
     _collect_filtered_files,
     _convert_library_to_component,
-    _detect_requires,
     _parse_library_json,
     _parse_library_properties,
     _process_dependencies,
@@ -83,19 +83,6 @@ def test_collect_filtered_files_exclude(tmp_path):
     assert str(f2) not in result
 
 
-def test_detect_requires(tmp_path):
-    f = tmp_path / "main.c"
-    f.write_text('#include "mbedtls/foo.h"')
-
-    result = _detect_requires([str(f)])
-    assert "mbedtls" in result
-
-
-def test_detect_requires_ignores_invalid_file(tmp_path):
-    result = _detect_requires([str(tmp_path / "missing.c")])
-    assert result == set()
-
-
 def test_split_list_by_condition():
     items = ["-Iinclude", "-Llib", "-Wall"]
 
@@ -142,7 +129,7 @@ def test_generate_cmakelists_txt_with_flags(tmp_component, tmp_path):
         == f"""idf_component_register(
   SRCS "src{sep}main.c"
   INCLUDE_DIRS "src"
-  REQUIRES dep
+  REQUIRES dep ${{ESPHOME_PROJECT_MANAGED_COMPONENTS}} ${{ESPHOME_PROJECT_BUILTIN_COMPONENTS}}
 )
 target_compile_options(${{COMPONENT_LIB}} PUBLIC
   "-DTEST"
@@ -160,6 +147,58 @@ target_link_libraries(${{COMPONENT_LIB}} INTERFACE
     )
 
 
+def test_generate_cmakelists_txt_references_project_managed_components_variable(
+    tmp_component: IDFComponent,
+) -> None:
+    # The CMakeLists is cached under pio_components/<hash>/ and shared
+    # across projects, so the project-managed REQUIRES list is exposed via
+    # a CMake variable expanded at configure time rather than baked here.
+    src_dir = tmp_component.path / "src"
+    src_dir.mkdir()
+    (src_dir / "main.c").write_text("int main() {}")
+    tmp_component.data = {}
+
+    content = generate_cmakelists_txt(tmp_component)
+    assert "${ESPHOME_PROJECT_MANAGED_COMPONENTS}" in content
+
+
+def test_generate_idf_component_overwrites_bundled_files(
+    tmp_path: Path,
+    monkeypatch: pytest.MonkeyPatch,
+    esp32_idf_core: None,
+) -> None:
+    # A library that ships its own CMakeLists.txt + idf_component.yml must
+    # have both replaced by ESPHome's generated content. Library authors'
+    # bundled IDF metadata is frequently broken (bogus REQUIRES, hard-coded
+    # frameworks), so we always regenerate from library.json.
+    from esphome.espidf.component import _generate_idf_component
+
+    (tmp_path / "src").mkdir()
+    (tmp_path / "src" / "main.cpp").write_text("// dummy\n")
+    (tmp_path / "library.json").write_text(json.dumps({"name": "tripwire-lib"}))
+    (tmp_path / "CMakeLists.txt").write_text("# TRIPWIRE_BUNDLED_CMAKELISTS\n")
+    (tmp_path / "idf_component.yml").write_text("# TRIPWIRE_BUNDLED_MANIFEST\n")
+
+    fake_component = IDFComponent(
+        "owner/tripwire-lib", "1.0.0", source=URLSource("http://dummy")
+    )
+    fake_component.path = tmp_path
+    monkeypatch.setattr(
+        esphome.espidf.component,
+        "_convert_library_to_component",
+        lambda _lib: fake_component,
+    )
+    monkeypatch.setattr(fake_component, "download", lambda force=False: None)
+
+    _generate_idf_component(Library("owner/tripwire-lib", "1.0.0", None))
+
+    cml = (tmp_path / "CMakeLists.txt").read_text()
+    manifest = (tmp_path / "idf_component.yml").read_text()
+    assert "TRIPWIRE_BUNDLED_CMAKELISTS" not in cml
+    assert "TRIPWIRE_BUNDLED_MANIFEST" not in manifest
+    assert "idf_component_register" in cml
+
+
 def test_generate_idf_component_yml_basic(tmp_component):
     tmp_component.data = {"description": "test", "repository": {"url": "http://aaa"}}
     result = generate_idf_component_yml(tmp_component)
@@ -187,27 +226,6 @@ dependencies:
     )
 
 
-def test_generate_idf_component_yml_arduino_registry_dep(tmp_component):
-    # Synthetic arduino-esp32 dep with no source / no path: should emit a
-    # version-only entry so the IDF component manager resolves it from the
-    # registry instead of via git.
-    dep = IDFComponent("espressif/arduino-esp32", "3.3.8", source=None)
-
-    tmp_component.dependencies = [dep]
-    tmp_component.data = {}
-
-    result = generate_idf_component_yml(tmp_component)
-
-    assert (
-        result
-        == """version: 1.0.0
-dependencies:
-  espressif/arduino-esp32:
-    version: 3.3.8
-"""
-    )
-
-
 def test_generate_idf_component_yml_missing_path_reraises(tmp_component):
     # A dep without a path and without a recognised source should re-raise
     # the underlying RuntimeError instead of silently producing a bad manifest.
@@ -250,14 +268,126 @@ def test_check_library_data_invalid_framework(esp32_idf_core):
         _check_library_data({"platforms": "*", "frameworks": ["other"]})
 
 
-def test_extra_script_logs_warning(caplog, esp32_idf_core):
-    extra_script = "myscript.sh"
+def test_extra_script_captures_libpath_libs_and_defines(tmp_path):
+    from esphome.espidf.extra_script import captured_as_build_flags, run_extra_script
+
+    (tmp_path / "src" / "esp32").mkdir(parents=True)
+    script = tmp_path / "extra_script.py"
+    script.write_text(
+        "Import('env')\n"
+        "mcu = env.get('BOARD_MCU')\n"
+        "env.Append(\n"
+        "    LIBPATH=[join('src', mcu)],\n"
+        "    LIBS=['algobsec'],\n"
+        "    CPPDEFINES=['FOO', ('BAR', '1')],\n"
+        "    LINKFLAGS=['-Wl,--gc-sections'],\n"
+        ")\n"
+    )
+    # The script uses bare ``join`` (PIO's extra-scripts run inside SCons
+    # where this is in scope). Inject it via the script header so the
+    # shim's exec namespace can resolve it.
+    script.write_text("from os.path import join\n" + script.read_text())
+
+    result = run_extra_script(script, library_dir=tmp_path, idf_target="esp32")
+
+    assert result.libpath == [os.path.join("src", "esp32")]
+    assert result.libs == ["algobsec"]
+    assert ("BAR", "1") in result.cppdefines
+    assert "FOO" in result.cppdefines
+    assert result.linkflags == ["-Wl,--gc-sections"]
+
+    flags = captured_as_build_flags(result, library_dir=tmp_path)
+    sep = os.sep
+    assert f"-Lsrc{sep}esp32" in flags
+    assert "-lalgobsec" in flags
+    assert "-DFOO" in flags
+    assert "-DBAR=1" in flags
+    assert "-Wl,--gc-sections" in flags
+
+
+def test_extra_script_libpath_relative_resolves_against_library_dir(
+    tmp_path, monkeypatch
+):
+    """Relative LIBPATH entries must resolve against ``library_dir``, not the
+    caller's CWD (the shim restores CWD before ``captured_as_build_flags``
+    runs)."""
+    from esphome.espidf.extra_script import ExtraScriptResult, captured_as_build_flags
+
+    (tmp_path / "lib" / "esp32").mkdir(parents=True)
+    elsewhere = tmp_path.parent / "not_the_library_dir"
+    elsewhere.mkdir(exist_ok=True)
+    monkeypatch.chdir(elsewhere)
+
+    result = ExtraScriptResult(libpath=["lib/esp32"])
+    flags = captured_as_build_flags(result, library_dir=tmp_path)
+
+    sep = os.sep
+    assert flags == [f"-Llib{sep}esp32"]
+
+
+def test_extra_script_libpath_absolute_outside_library_dir(tmp_path):
+    from esphome.espidf.extra_script import ExtraScriptResult, captured_as_build_flags
+
+    outside = tmp_path.parent / "system_lib"
+    outside.mkdir(exist_ok=True)
+    result = ExtraScriptResult(libpath=[str(outside)])
+
+    flags = captured_as_build_flags(result, library_dir=tmp_path)
+    assert flags == [f"-L{outside.resolve()}"]
+
+
+def test_extra_script_failure_returns_empty_result(tmp_path, caplog):
+    from esphome.espidf.extra_script import run_extra_script
+
+    script = tmp_path / "broken.py"
+    script.write_text("raise RuntimeError('boom')\n")
 
     with caplog.at_level("WARNING"):
-        _check_library_data({"build": {"extraScript": extra_script}})
+        result = run_extra_script(script, library_dir=tmp_path, idf_target="esp32")
 
-    assert "not supported" in caplog.text
-    assert "myscript.sh" in caplog.text
+    assert result.libpath == []
+    assert result.libs == []
+    assert "broken.py" in caplog.text
+
+
+def test_apply_extra_script_path_traversal_is_rejected(tmp_path):
+    from esphome.espidf.component import _apply_extra_script
+
+    library_dir = tmp_path / "lib"
+    library_dir.mkdir()
+    outside = tmp_path / "evil.py"
+    outside.write_text("env.Append(LIBS=['pwned'])\n")
+
+    c = IDFComponent("owner/name", "1.0", source=URLSource("http://dummy"))
+    c.path = library_dir
+    c.data = {"build": {"extraScript": "../evil.py"}}
+
+    _apply_extra_script(c)
+
+    # Nothing was folded into flags: the traversal was rejected before
+    # the script could run.
+    assert "flags" not in c.data["build"]
+
+
+def test_apply_extra_script_merges_into_existing_flags(tmp_path, monkeypatch):
+    from esphome.components import esp32 as esp32_module
+
+    monkeypatch.setattr(esp32_module, "get_esp32_variant", lambda: "ESP32")
+
+    from esphome.espidf.component import _apply_extra_script
+
+    (tmp_path / "src").mkdir()
+    script = tmp_path / "extra.py"
+    script.write_text("env.Append(LIBS=['algobsec'])\n")
+
+    c = IDFComponent("owner/name", "1.0", source=URLSource("http://dummy"))
+    c.path = tmp_path
+    c.data = {"build": {"extraScript": "extra.py", "flags": ["-DEXISTING"]}}
+
+    _apply_extra_script(c)
+
+    assert "-DEXISTING" in c.data["build"]["flags"]
+    assert "-lalgobsec" in c.data["build"]["flags"]
 
 
 def test_parse_library_json(tmp_path):

tests/unit_tests/test_writer.py

@@ -443,6 +443,14 @@ def test_clean_build(
     dependencies_lock = tmp_path / "dependencies.lock"
     dependencies_lock.write_text("lock file")
 
+    # Native ESP-IDF toolchain artifacts.
+    idf_build_dir = tmp_path / "build"
+    idf_build_dir.mkdir()
+    (idf_build_dir / "CMakeCache.txt").write_text("cache")
+    managed_components_dir = tmp_path / "managed_components"
+    managed_components_dir.mkdir()
+    (managed_components_dir / "espressif__arduino-esp32").mkdir()
+
     # Create PlatformIO cache directory
     platformio_cache_dir = tmp_path / ".platformio" / ".cache"
     platformio_cache_dir.mkdir(parents=True)
@@ -456,11 +464,14 @@ def test_clean_build(
     mock_core.relative_piolibdeps_path.return_value = piolibdeps_dir
     mock_core.relative_build_path.return_value = dependencies_lock
     mock_core.platformio_cache_dir = str(platformio_cache_dir)
+    mock_core.relative_build_path.side_effect = lambda name: tmp_path / name
 
     # Verify all exist before
     assert pioenvs_dir.exists()
     assert piolibdeps_dir.exists()
     assert dependencies_lock.exists()
+    assert idf_build_dir.exists()
+    assert managed_components_dir.exists()
     assert platformio_cache_dir.exists()
 
     # Mock PlatformIO's ProjectConfig cache_dir
@@ -483,6 +494,8 @@ def test_clean_build(
     assert not pioenvs_dir.exists()
     assert not piolibdeps_dir.exists()
     assert not dependencies_lock.exists()
+    assert not idf_build_dir.exists()
+    assert not managed_components_dir.exists()
     assert not platformio_cache_dir.exists()
 
     # Verify logging
@@ -490,6 +503,8 @@ def test_clean_build(
     assert ".pioenvs" in caplog.text
     assert ".piolibdeps" in caplog.text
     assert "dependencies.lock" in caplog.text
+    assert str(idf_build_dir) in caplog.text
+    assert str(managed_components_dir) in caplog.text
     assert "PlatformIO cache" in caplog.text
 
 
@@ -511,7 +526,7 @@ def test_clean_build_partial_exists(
     # Setup mocks
     mock_core.relative_pioenvs_path.return_value = pioenvs_dir
     mock_core.relative_piolibdeps_path.return_value = piolibdeps_dir
-    mock_core.relative_build_path.return_value = dependencies_lock
+    mock_core.relative_build_path.side_effect = lambda name: tmp_path / name
 
     # Verify only pioenvs exists
     assert pioenvs_dir.exists()
@@ -548,7 +563,7 @@ def test_clean_build_nothing_exists(
     # Setup mocks
     mock_core.relative_pioenvs_path.return_value = pioenvs_dir
     mock_core.relative_piolibdeps_path.return_value = piolibdeps_dir
-    mock_core.relative_build_path.return_value = dependencies_lock
+    mock_core.relative_build_path.side_effect = lambda name: tmp_path / name
 
     # Verify nothing exists
     assert not pioenvs_dir.exists()
@@ -584,7 +599,7 @@ def test_clean_build_platformio_not_available(
     # Setup mocks
     mock_core.relative_pioenvs_path.return_value = pioenvs_dir
     mock_core.relative_piolibdeps_path.return_value = piolibdeps_dir
-    mock_core.relative_build_path.return_value = dependencies_lock
+    mock_core.relative_build_path.side_effect = lambda name: tmp_path / name
 
     # Verify all exist before
     assert pioenvs_dir.exists()
@@ -622,7 +637,7 @@ def test_clean_build_empty_cache_dir(
     # Setup mocks
     mock_core.relative_pioenvs_path.return_value = pioenvs_dir
     mock_core.relative_piolibdeps_path.return_value = tmp_path / ".piolibdeps"
-    mock_core.relative_build_path.return_value = tmp_path / "dependencies.lock"
+    mock_core.relative_build_path.side_effect = lambda name: tmp_path / name
 
     # Verify pioenvs exists before
     assert pioenvs_dir.exists()
@@ -1351,7 +1366,7 @@ def test_clean_build_handles_readonly_files(
     # Setup mocks
     mock_core.relative_pioenvs_path.return_value = pioenvs_dir
     mock_core.relative_piolibdeps_path.return_value = tmp_path / ".piolibdeps"
-    mock_core.relative_build_path.return_value = tmp_path / "dependencies.lock"
+    mock_core.relative_build_path.side_effect = lambda name: tmp_path / name
 
     # Verify file is read-only
     assert not os.access(readonly_file, os.W_OK)
@@ -1415,7 +1430,7 @@ def test_clean_build_reraises_for_other_errors(
     # Setup mocks
     mock_core.relative_pioenvs_path.return_value = pioenvs_dir
     mock_core.relative_piolibdeps_path.return_value = tmp_path / ".piolibdeps"
-    mock_core.relative_build_path.return_value = tmp_path / "dependencies.lock"
+    mock_core.relative_build_path.side_effect = lambda name: tmp_path / name
 
     try:
         # Mock os.access in writer module to return True (writable)

tests/unit_tests/test_yaml_util.py

@@ -390,6 +390,21 @@ def test_track_yaml_loads_cleanup_on_exception(tmp_path: Path) -> None:
     assert len(yaml_util._load_listeners) == before
 
 
+def test_track_yaml_loads_no_duplicate_load_on_top_level_include_failure(
+    tmp_path: Path,
+) -> None:
+    """A failed top-level !include must not record any file twice in track_yaml_loads."""
+    main = tmp_path / "main.yaml"
+    main.write_text("!include missing.yaml\n")
+
+    with yaml_util.track_yaml_loads() as loaded, pytest.raises(EsphomeError):
+        yaml_util.load_yaml(main)
+
+    assert len(loaded) == len(set(loaded)), (
+        f"Files loaded more than once during a failed top-level include: {loaded}"
+    )
+
+
 @pytest.mark.parametrize(
     "data",
     [