Merge remote-tracking branch 'upstream/api-break-circular-include' into integration

tests/components/i2s_audio/common-spdif_mode.yaml

@@ -1,13 +1,3 @@
-substitutions:
-  i2s_bclk_pin: GPIO27
-  i2s_lrclk_pin: GPIO26
-  i2s_mclk_pin: GPIO25
-  i2s_dout_pin: GPIO12
-  spdif_data_pin: GPIO4
-
-packages:
-  i2c: !include ../../test_build_components/common/i2c/esp32-idf.yaml
-
 i2s_audio:
   - id: i2s_output
 
@@ -20,6 +10,5 @@ speaker:
     use_apll: true
     timeout: 2s
     sample_rate: 48000
-    bits_per_sample: 16bit
     channel: stereo
     i2s_mode: primary

tests/components/i2s_audio/test-spdif_speaker.esp32-idf.yaml

@@ -0,0 +1,8 @@
+substitutions:
+  i2s_bclk_pin: GPIO27
+  i2s_lrclk_pin: GPIO26
+  i2s_mclk_pin: GPIO25
+  i2s_dout_pin: GPIO12
+  spdif_data_pin: GPIO4
+
+<<: !include common-spdif_mode.yaml

tests/unit_tests/test_espidf_component.py

@@ -203,7 +203,7 @@ def test_generate_idf_component_yml_basic(tmp_component):
     tmp_component.data = {"description": "test", "repository": {"url": "http://aaa"}}
     result = generate_idf_component_yml(tmp_component)
 
-    assert result == "description: test\nversion: 1.0.0\nrepository: http://aaa\n"
+    assert result == "description: test\nrepository: http://aaa\n"
 
 
 def test_generate_idf_component_yml_with_dependencies(tmp_component, tmp_path):
@@ -217,18 +217,16 @@ def test_generate_idf_component_yml_with_dependencies(tmp_component, tmp_path):
 
     assert (
         result
-        == f"""version: 1.0.0
-dependencies:
+        == f"""dependencies:
   dep:
-    version: '1.0'
     override_path: {dep.path}
 """
     )
 
 
-def test_generate_idf_component_yml_missing_path_reraises(tmp_component):
-    # A dep without a path and without a recognised source should re-raise
-    # the underlying RuntimeError instead of silently producing a bad manifest.
+def test_generate_idf_component_yml_missing_path_raises(tmp_component):
+    # A dep without a path is a contract violation — every dep is expected
+    # to have been downloaded before YAML generation. Raise loudly.
     dep = IDFComponent("foo/bar", "1.0", source=None)
 
     tmp_component.dependencies = [dep]
@@ -422,15 +420,37 @@ def test_convert_library_with_repository():
     result = _convert_library_to_component(lib)
 
     assert result.name == "foo/bar"
-    assert result.version == "1.2.3"
+    assert result.version == "*"
     assert isinstance(result.source, GitSource)
+    assert result.source.ref == "v1.2.3"
 
 
-def test_convert_library_missing_ref():
+def test_convert_library_with_branch_ref():
+    lib = Library("name", None, "https://github.com/foo/bar.git#some-branch")
+
+    result = _convert_library_to_component(lib)
+
+    assert result.name == "foo/bar"
+    assert result.version == "*"
+    assert isinstance(result.source, GitSource)
+    assert result.source.ref == "some-branch"
+
+
+def test_convert_library_missing_ref_uses_default_branch():
+    """A bare URL with no #ref clones the remote's default branch.
+
+    Matches PIO's lib_deps behavior and external_components handling --
+    git.clone_or_update with ref=None leaves the depth-1 clone on
+    whatever branch the remote HEAD points at.
+    """
     lib = Library("name", None, "https://github.com/foo/bar.git")
 
-    with pytest.raises(ValueError):
-        _convert_library_to_component(lib)
+    result = _convert_library_to_component(lib)
+
+    assert result.name == "foo/bar"
+    assert result.version == "*"
+    assert isinstance(result.source, GitSource)
+    assert result.source.ref is None
 
 
 def test_convert_library_registry(monkeypatch):
@@ -485,3 +505,113 @@ def test_process_dependencies_skips_invalid(tmp_component):
     _process_dependencies(tmp_component)
 
     assert tmp_component.dependencies == []
+
+
+def test_process_dependencies_dict_form(tmp_component, monkeypatch):
+    """PIO library.json shorthand ``{"owner/Name": "version"}`` is honored.
+
+    Iterating a dict gives string keys, which would silently fail the
+    ``"name" in dependency`` substring check. Normalize to list-of-dicts
+    first so the dict form (used by e.g. tesla-ble for its nanopb dep)
+    is treated the same as the verbose list form.
+    """
+    captured: list[Library] = []
+
+    def fake_generate(library):
+        captured.append(library)
+        return IDFComponent(
+            library.name, library.version, source=URLSource("http://dummy.com")
+        )
+
+    tmp_component.data = {
+        "dependencies": {
+            "nanopb/Nanopb": "^0.4.91",
+            "BareName": "1.2.3",
+        }
+    }
+    monkeypatch.setattr(
+        esphome.espidf.component, "_generate_idf_component", fake_generate
+    )
+    monkeypatch.setattr(esphome.espidf.component, "_check_library_data", lambda x: None)
+
+    _process_dependencies(tmp_component)
+
+    assert len(tmp_component.dependencies) == 2
+    names = sorted(lib.name for lib in captured)
+    versions = sorted(lib.version for lib in captured)
+    assert names == ["BareName", "nanopb/Nanopb"]
+    assert versions == ["1.2.3", "^0.4.91"]
+
+
+def test_process_dependencies_dict_form_with_url_value(tmp_component, monkeypatch):
+    """A dict-value that's a URL gets routed to ``repository`` like the list form."""
+    captured: list[Library] = []
+
+    def fake_generate(library):
+        captured.append(library)
+        return IDFComponent(library.name, "*", source=URLSource("http://dummy.com"))
+
+    tmp_component.data = {
+        "dependencies": {
+            "foo/Bar": "https://github.com/foo/bar.git#main",
+        }
+    }
+    monkeypatch.setattr(
+        esphome.espidf.component, "_generate_idf_component", fake_generate
+    )
+    monkeypatch.setattr(esphome.espidf.component, "_check_library_data", lambda x: None)
+
+    _process_dependencies(tmp_component)
+
+    assert len(captured) == 1
+    assert captured[0].name == "foo/Bar"
+    assert captured[0].version is None
+    assert captured[0].repository == "https://github.com/foo/bar.git#main"
+
+
+def test_process_dependencies_dict_form_with_nested_spec(tmp_component, monkeypatch):
+    """A dict-value that's itself a dict is merged into the entry.
+
+    PIO's library.json allows ``{"owner/Name": {"version": "...", ...}}``
+    for entries that need fields beyond just a version (platforms,
+    frameworks, etc.). The extra fields flow into _check_library_data
+    via the entry merge.
+    """
+    captured: list[Library] = []
+    checked: list[dict] = []
+
+    def fake_generate(library):
+        captured.append(library)
+        return IDFComponent(
+            library.name, library.version, source=URLSource("http://dummy.com")
+        )
+
+    tmp_component.data = {
+        "dependencies": {
+            "nanopb/Nanopb": {"version": "^0.4.91", "platforms": "espidf"},
+        }
+    }
+    monkeypatch.setattr(
+        esphome.espidf.component, "_generate_idf_component", fake_generate
+    )
+    monkeypatch.setattr(
+        esphome.espidf.component,
+        "_check_library_data",
+        checked.append,
+    )
+
+    _process_dependencies(tmp_component)
+
+    assert len(captured) == 1
+    assert captured[0].name == "nanopb/Nanopb"
+    assert captured[0].version == "^0.4.91"
+    # Extra spec fields reach _check_library_data so platform/framework
+    # gating still applies.
+    assert checked == [
+        {
+            "name": "Nanopb",
+            "owner": "nanopb",
+            "version": "^0.4.91",
+            "platforms": "espidf",
+        }
+    ]

tests/unit_tests/test_espidf_toolchain.py

@@ -0,0 +1,58 @@
+"""Tests for esphome.espidf.toolchain helpers."""
+
+# pylint: disable=protected-access
+
+from unittest.mock import patch
+
+from esphome.const import CONF_FRAMEWORK, CONF_SOURCE
+from esphome.core import CORE
+from esphome.espidf import toolchain
+
+
+def test_get_framework_source_override_no_config():
+    """When CORE.config hasn't been set, no override is returned."""
+    CORE.config = None
+    assert toolchain._get_framework_source_override() is None
+
+
+def test_get_framework_source_override_no_esp32_section():
+    """A config without an esp32 section yields no override."""
+    CORE.config = {}
+    assert toolchain._get_framework_source_override() is None
+
+
+def test_get_framework_source_override_no_framework_source():
+    """An esp32 section without framework.source yields no override."""
+    CORE.config = {"esp32": {CONF_FRAMEWORK: {}}}
+    assert toolchain._get_framework_source_override() is None
+
+
+def test_get_framework_source_override_returns_value():
+    """A user-supplied framework source is returned verbatim."""
+    url = "https://example.com/esp-idf-v{VERSION}.tar.xz"
+    CORE.config = {"esp32": {CONF_FRAMEWORK: {CONF_SOURCE: url}}}
+    assert toolchain._get_framework_source_override() == url
+
+
+def test_get_esphome_esp_idf_paths_forwards_source_override():
+    """_get_esphome_esp_idf_paths threads the override into check_esp_idf_install."""
+    url = "https://my-mirror/esp-idf-v{VERSION}.tar.xz"
+    CORE.config = {"esp32": {CONF_FRAMEWORK: {CONF_SOURCE: url}}}
+    # Hit a fresh cache key so check_esp_idf_install is actually called.
+    toolchain._cache().paths.clear()
+    with patch.object(
+        toolchain, "check_esp_idf_install", return_value=("/fw", "/penv")
+    ) as mock_install:
+        toolchain._get_esphome_esp_idf_paths("5.5.4")
+    mock_install.assert_called_once_with("5.5.4", source_url=url)
+
+
+def test_get_esphome_esp_idf_paths_no_override():
+    """When no source override is configured, source_url=None is passed."""
+    CORE.config = {}
+    toolchain._cache().paths.clear()
+    with patch.object(
+        toolchain, "check_esp_idf_install", return_value=("/fw", "/penv")
+    ) as mock_install:
+        toolchain._get_esphome_esp_idf_paths("5.5.4")
+    mock_install.assert_called_once_with("5.5.4", source_url=None)

tests/unit_tests/test_size_summary.py

@@ -0,0 +1,128 @@
+"""Tests for esphome.espidf.size_summary.print_summary."""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+import pytest
+
+from esphome.espidf.size_summary import print_summary
+
+
+def _write_size_json(tmp_path: Path, data: dict) -> Path:
+    """Drop a fake esp_idf_size.json under ``tmp_path`` and return the path."""
+    out = tmp_path / "esp_idf_size.json"
+    out.write_text(json.dumps(data))
+    return out
+
+
+def _esp32_size_data() -> dict:
+    """Synthetic esp_idf_size.json for the original ESP32 (split IRAM/DRAM)."""
+    return {
+        "image_size": 827455,
+        "memory_types": {
+            "DRAM": {
+                "size": 180736,
+                "used": 47332,
+                "sections": {
+                    ".dram0.bss": {"abbrev_name": ".bss", "size": 30616},
+                    ".dram0.data": {"abbrev_name": ".data", "size": 16716},
+                },
+            },
+            "IRAM": {
+                "size": 131072,
+                "used": 80351,
+                "sections": {
+                    ".iram0.text": {"abbrev_name": ".text", "size": 79323},
+                    ".iram0.vectors": {"abbrev_name": ".vectors", "size": 1028},
+                },
+            },
+        },
+    }
+
+
+def _s3_size_data() -> dict:
+    """Synthetic esp_idf_size.json for ESP32-S3 (unified DIRAM)."""
+    return {
+        "image_size": 724215,
+        "memory_types": {
+            "DIRAM": {
+                "size": 341760,
+                "used": 104999,
+                "sections": {
+                    ".iram0.text": {"abbrev_name": ".text", "size": 58051},
+                    ".dram0.bss": {"abbrev_name": ".bss", "size": 27088},
+                    ".dram0.data": {"abbrev_name": ".data", "size": 19708},
+                    ".noinit": {"abbrev_name": ".noinit", "size": 152},
+                },
+            },
+            "IRAM": {
+                "size": 16384,
+                "used": 16384,
+                "sections": {
+                    ".iram0.text": {"abbrev_name": ".text", "size": 15356},
+                    ".iram0.vectors": {"abbrev_name": ".vectors", "size": 1028},
+                },
+            },
+        },
+    }
+
+
+def test_print_summary_esp32_uses_dram(
+    tmp_path: Path, capsys: pytest.CaptureFixture[str]
+) -> None:
+    """Original ESP32: DRAM has no ``.text``, so RAM = DRAM.used / DRAM.size unchanged."""
+    size_json = _write_size_json(tmp_path, _esp32_size_data())
+    print_summary(size_json, partitions_csv=None)
+    out = capsys.readouterr().out
+    assert "RAM:" in out
+    assert "used 47332 bytes from 180736 bytes" in out
+
+
+def test_print_summary_s3_falls_back_to_diram(
+    tmp_path: Path, capsys: pytest.CaptureFixture[str]
+) -> None:
+    """ESP32-S3 with no DRAM key falls back to DIRAM and reports raw region usage."""
+    size_json = _write_size_json(tmp_path, _s3_size_data())
+    print_summary(size_json, partitions_csv=None)
+    out = capsys.readouterr().out
+    assert "used 104999 bytes from 341760 bytes" in out
+
+
+def test_print_summary_skips_when_diram_total_collapses(
+    tmp_path: Path, capsys: pytest.CaptureFixture[str]
+) -> None:
+    """A zero-size region drops the RAM line rather than divide by zero."""
+    size_json = _write_size_json(
+        tmp_path,
+        {
+            "memory_types": {
+                "DIRAM": {
+                    "size": 0,
+                    "used": 0,
+                    "sections": {},
+                },
+            },
+        },
+    )
+    print_summary(size_json, partitions_csv=None)
+    out = capsys.readouterr().out
+    assert "RAM:" not in out
+
+
+def test_print_summary_handles_missing_json(
+    tmp_path: Path, capsys: pytest.CaptureFixture[str]
+) -> None:
+    """Missing size json is non-fatal and prints nothing."""
+    print_summary(tmp_path / "does_not_exist.json", partitions_csv=None)
+    assert capsys.readouterr().out == ""
+
+
+def test_print_summary_handles_no_memory_types(
+    tmp_path: Path, capsys: pytest.CaptureFixture[str]
+) -> None:
+    """A size json without ``memory_types`` still doesn't crash."""
+    size_json = _write_size_json(tmp_path, {"image_size": 0})
+    print_summary(size_json, partitions_csv=None)
+    assert capsys.readouterr().out == ""

tests/unit_tests/test_storage_json.py

@@ -9,7 +9,7 @@ from unittest.mock import MagicMock, Mock, patch
 import pytest
 
 from esphome import storage_json
-from esphome.const import CONF_DISABLED, CONF_MDNS
+from esphome.const import CONF_DISABLED, CONF_MDNS, Toolchain
 from esphome.core import CORE
 
 
@@ -308,6 +308,7 @@ def test_storage_json_from_esphome_core(setup_core: Path) -> None:
     mock_core.loaded_platforms = {"sensor"}
     mock_core.config = {CONF_MDNS: {CONF_DISABLED: True}}
     mock_core.target_framework = "esp-idf"
+    mock_core.toolchain = Toolchain.ESP_IDF
 
     with patch("esphome.components.esp32.get_esp32_variant") as mock_variant:
         mock_variant.return_value = "ESP32-C3"
@@ -327,6 +328,7 @@ def test_storage_json_from_esphome_core(setup_core: Path) -> None:
     assert result.no_mdns is True
     assert result.framework == "esp-idf"
     assert result.core_platform == "esp32"
+    assert result.toolchain == "esp-idf"
 
 
 def test_storage_json_from_esphome_core_mdns_enabled(setup_core: Path) -> None:
@@ -345,10 +347,12 @@ def test_storage_json_from_esphome_core_mdns_enabled(setup_core: Path) -> None:
     mock_core.loaded_platforms = set()
     mock_core.config = {}  # No MDNS config means enabled
     mock_core.target_framework = "arduino"
+    mock_core.toolchain = None
 
     result = storage_json.StorageJSON.from_esphome_core(mock_core, old=None)
 
     assert result.no_mdns is False
+    assert result.toolchain is None
 
 
 def test_storage_json_load_valid_file(tmp_path: Path) -> None:
@@ -470,6 +474,73 @@ def test_storage_json_equality() -> None:
     assert storage1 != "not a storage object"
 
 
+def _make_storage_with_toolchain(
+    toolchain: str | None,
+) -> storage_json.StorageJSON:
+    return storage_json.StorageJSON(
+        storage_version=1,
+        name="dev",
+        friendly_name=None,
+        comment=None,
+        esphome_version="2024.1.0",
+        src_version=1,
+        address="dev.local",
+        web_port=None,
+        target_platform="ESP32",
+        build_path=Path("/build"),
+        firmware_bin_path=Path("/build/firmware.bin"),
+        loaded_integrations=set(),
+        loaded_platforms=set(),
+        no_mdns=False,
+        framework="esp-idf",
+        core_platform="esp32",
+        toolchain=toolchain,
+    )
+
+
+def test_storage_json_toolchain_round_trip(setup_core: Path) -> None:
+    """Sidecar toolchain survives save -> load -> apply_to_core."""
+    storage = _make_storage_with_toolchain("esp-idf")
+    path = setup_core / "storage.json"
+    path.write_text(storage.to_json())
+
+    # Serialization key is stable -- device-builder relies on it.
+    assert json.loads(path.read_text())["toolchain"] == "esp-idf"
+
+    loaded = storage_json.StorageJSON.load(path)
+    assert loaded is not None
+    assert loaded.toolchain == "esp-idf"
+
+    CORE.toolchain = None
+    with patch("esphome.components.esp32.get_esp32_variant"):
+        loaded.apply_to_core()
+    assert CORE.toolchain == Toolchain.ESP_IDF
+
+
+def test_storage_json_apply_to_core_preserves_cli_toolchain(
+    setup_core: Path,
+) -> None:
+    """A CLI-set CORE.toolchain wins over the sidecar value."""
+    loaded = _make_storage_with_toolchain("esp-idf")
+
+    CORE.toolchain = Toolchain.PLATFORMIO
+    with patch("esphome.components.esp32.get_esp32_variant"):
+        loaded.apply_to_core()
+    assert CORE.toolchain == Toolchain.PLATFORMIO
+
+
+def test_storage_json_apply_to_core_ignores_unknown_toolchain(
+    setup_core: Path,
+) -> None:
+    """Unknown enum values (corrupt sidecar / newer ESPHome) fall through to None."""
+    loaded = _make_storage_with_toolchain("gcc")
+
+    CORE.toolchain = None
+    with patch("esphome.components.esp32.get_esp32_variant"):
+        loaded.apply_to_core()
+    assert CORE.toolchain is None
+
+
 def test_esphome_storage_json_as_dict() -> None:
     """Test EsphomeStorageJSON.as_dict returns correct dictionary."""
     storage = storage_json.EsphomeStorageJSON(