diff --git a/Doxyfile b/Doxyfile index 30ae42ea2c..3dfe6c5ed4 100644 --- a/Doxyfile +++ b/Doxyfile @@ -48,7 +48,7 @@ PROJECT_NAME = ESPHome # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = 2026.5.1 +PROJECT_NUMBER = 2026.5.2 # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a diff --git a/esphome/components/api/api_connection.cpp b/esphome/components/api/api_connection.cpp index f2bf3752fa..cd5b3fd694 100644 --- a/esphome/components/api/api_connection.cpp +++ b/esphome/components/api/api_connection.cpp @@ -1306,6 +1306,9 @@ void APIConnection::on_voice_assistant_announce_request(const VoiceAssistantAnno bool APIConnection::send_voice_assistant_get_configuration_response_(const VoiceAssistantConfigurationRequest &msg) { VoiceAssistantConfigurationResponse resp; if (!this->check_voice_assistant_api_connection_()) { + // send_message encodes synchronously, so this stack local outlives the encode + const std::vector empty_wake_words; + resp.active_wake_words = &empty_wake_words; return this->send_message(resp); } diff --git a/esphome/components/esp32/__init__.py b/esphome/components/esp32/__init__.py index 4f77258b2c..1a95f77437 100644 --- a/esphome/components/esp32/__init__.py +++ b/esphome/components/esp32/__init__.py @@ -46,7 +46,7 @@ from esphome.const import ( Toolchain, __version__, ) -from esphome.core import CORE, HexInt, Library +from esphome.core import CORE, EsphomeError, HexInt, Library from esphome.core.config import BOARD_MAX_LENGTH from esphome.coroutine import CoroPriority, coroutine_with_priority from esphome.espidf.component import generate_idf_component @@ -2657,13 +2657,29 @@ def copy_files(): def _decode_pc(config, addr): - from esphome.platformio import toolchain + # _decode_pc runs from the api log processor's asyncio callback, which + # only catches EsphomeError. Any other exception escaping here tears down + # the protocol and triggers an infinite reconnect/replay loop. Convert + # toolchain-resolution errors (e.g. missing build dir / cmake cache) into + # EsphomeError so the caller can disable decoding cleanly. + if CORE.using_toolchain_esp_idf: + from esphome.espidf import toolchain as idf_toolchain - idedata = toolchain.get_idedata(config) - if not idedata.addr2line_path or not idedata.firmware_elf_path: + try: + addr2line_path = idf_toolchain.get_addr2line_path() + firmware_elf_path = idf_toolchain.get_elf_path() + except RuntimeError as err: + raise EsphomeError(f"ESP-IDF toolchain not available: {err}") from err + else: + from esphome.platformio import toolchain + + idedata = toolchain.get_idedata(config) + addr2line_path = idedata.addr2line_path + firmware_elf_path = idedata.firmware_elf_path + if not addr2line_path or not firmware_elf_path: _LOGGER.debug("decode_pc no addr2line") return - command = [idedata.addr2line_path, "-pfiaC", "-e", idedata.firmware_elf_path, addr] + command = [str(addr2line_path), "-pfiaC", "-e", str(firmware_elf_path), addr] try: translation = subprocess.check_output(command, close_fds=False).decode().strip() except Exception: # pylint: disable=broad-except diff --git a/esphome/components/libretiny/hal.h b/esphome/components/libretiny/hal.h index 9c512504b7..01a7b5450b 100644 --- a/esphome/components/libretiny/hal.h +++ b/esphome/components/libretiny/hal.h @@ -11,11 +11,19 @@ #include "esphome/core/time_64.h" // IRAM_ATTR places a function in executable RAM so it is callable from an -// ISR even while flash is busy (XIP stall, OTA, logger flash write). -// Each family uses a section its stock linker already routes to RAM: -// RTL8710B → .image2.ram.text, RTL8720C → .sram.text. LN882H is the -// exception: its stock linker has no matching glob, so patch_linker.py -// injects KEEP(*(.sram.text*)) into .flash_copysection at pre-link. +// ISR even while flash is busy (XIP stall, OTA, logger flash write). All +// LibreTiny families that need it share the same .sram.text input section +// name; how that section is routed into RAM differs per family: +// RTL8720C: stock linker consumes *(.sram.text*) into .ram.code_text. +// RTL8710B: patch_linker.py.script injects KEEP(*(.sram.text*)) at the +// top of .ram_image2.data (which IS in ltchiptool's +// sections_ram). The stock linker has KEEP(*(.image2.ram.text*)) +// in .ram_image2.text but that output section is NOT in +// ltchiptool's AmebaZ elf2bin sections_ram list, so code routed +// there is dropped from the flashed binary. +// LN882H: patch_linker.py.script injects KEEP(*(.sram.text*)) into +// .flash_copysection (> RAM0 AT> FLASH), after KEEP(*(.vectors)) +// so the Cortex-M4 vector table stays 512-byte-aligned for VTOR. // // BK72xx (all variants) are left as a no-op: their SDK wraps flash // operations in GLOBAL_INT_DISABLE() which masks FIQ + IRQ at the CPU for @@ -26,13 +34,7 @@ // layer. #if defined(USE_BK72XX) #define IRAM_ATTR -#elif defined(USE_LIBRETINY_VARIANT_RTL8710B) -// Stock linker consumes *(.image2.ram.text*) into .ram_image2.text (> BD_RAM). -#define IRAM_ATTR __attribute__((noinline, section(".image2.ram.text"))) #else -// RTL8720C: stock linker consumes *(.sram.text*) into .ram.code_text. -// LN882H: patch_linker.py.script injects *(.sram.text*) into -// .flash_copysection (> RAM0 AT> FLASH). #define IRAM_ATTR __attribute__((noinline, section(".sram.text"))) #endif #define PROGMEM diff --git a/esphome/components/libretiny/patch_linker.py.script b/esphome/components/libretiny/patch_linker.py.script index 3a8a4787ed..dfeaaa57d1 100644 --- a/esphome/components/libretiny/patch_linker.py.script +++ b/esphome/components/libretiny/patch_linker.py.script @@ -6,12 +6,18 @@ import re import subprocess # ESPHome marks ISR code IRAM_ATTR, which on LibreTiny maps to a per-family -# section routed into RAM-executable memory (see esphome/core/hal.h). +# section routed into RAM-executable memory (see esphome/core/hal.h). The +# input section name is always .sram.text; only the output section it lands +# in differs per family. # # This script is NOT loaded on BK72xx (IRAM_ATTR is a no-op there; the SDK # masks FIQ+IRQ around flash writes). On the remaining families: -# - RTL8710B: hal.h uses section(".image2.ram.text"); stock linker consumes it. -# - RTL8720C: hal.h uses section(".sram.text"); stock linker consumes it. +# - RTL8720C: stock linker consumes *(.sram.text*) into .ram.code_text. +# - RTL8710B: stock linker has KEEP(*(.image2.ram.text*)) in .ram_image2.text, +# but ltchiptool's AmebaZ elf2bin (soc/ambz/binary.py) does NOT list +# .ram_image2.text in sections_ram, so code there is silently dropped from +# the flashed image. Inject KEEP(*(.sram.text*)) at the top of +# .ram_image2.data (which IS extracted) instead. # - LN882H: stock linker has no glob for ".sram.text", so we inject # KEEP(*(.sram.text*)) into ".flash_copysection" (> RAM0 AT> FLASH) # immediately after KEEP(*(.vectors)), so the vector table stays at @@ -34,6 +40,20 @@ _KEEP_LINE = ( # aligned address; injecting before the vectors would push them to an # unaligned offset and mis-route every IRQ handler. _LN_COPY = re.compile(r"(KEEP\(\*\(\.vectors\)\)[^\n]*\n)") +# Inject at the top of .ram_image2.data, before __data_start__ so our code +# does not fall inside the data range markers. .ram_image2.data is one of the +# sections ltchiptool's AmebaZ elf2bin extracts; BD_RAM is rwx so the code is +# executable. AmbZ has no C runtime .data copy loop (the bootloader loads +# image2 into BD_RAM whole) so the inline code is not clobbered after boot. +# +# The regex is intentionally strict (no attribute / ALIGN between the section +# name and the opening brace, brace on its own line). If a future AmbZ SDK +# linker template changes this format, _pre_link raises RuntimeError on the +# unpatched .ld file(s), and the RTL8710B CI compile job in +# tests/test_build_components fails on the PR, surfacing the mismatch loudly +# rather than silently shipping a binary with IRAM_ATTR code dropped from +# one or both OTA slots. +_AMBZ_DATA = re.compile(r"(\.ram_image2\.data\s*:\s*\n?\s*\{\s*\n)") def _detect(env): @@ -71,12 +91,11 @@ def _inject_keep(host_section): # Variants not listed here intentionally have no .ld patcher: -# - RTL8710B: hal.h uses section(".image2.ram.text") which the stock linker -# already routes into .ram_image2.text (> BD_RAM). -# - RTL8720C: stock linker already consumes *(.sram.text*). +# - RTL8720C: stock linker already consumes *(.sram.text*) into .ram.code_text. # - BK72xx (all): SDK masks FIQ+IRQ around flash writes, IRAM_ATTR is no-op. _PATCHERS_BY_VARIANT = { "LN882H": (_inject_keep(_LN_COPY),), + "RTL8710B": (_inject_keep(_AMBZ_DATA),), } @@ -87,13 +106,14 @@ def _patchers_for(variant): def _pre_link(target, source, env): build_dir = env.subst("$BUILD_DIR") ld_files = [f for f in os.listdir(build_dir) if f.endswith(".ld")] - patched = 0 + patched = [] + unpatched = [] for name in ld_files: path = os.path.join(build_dir, name) with open(path, "r", encoding="utf-8") as fh: original = fh.read() if _MARKER in original: - patched += 1 + patched.append(name) continue content = original for fn in _patchers: @@ -102,7 +122,9 @@ def _pre_link(target, source, env): with open(path, "w", encoding="utf-8") as fh: fh.write(content) print("ESPHome: patched {} for IRAM_ATTR placement".format(name)) - patched += 1 + patched.append(name) + else: + unpatched.append(name) if not patched: raise RuntimeError( "ESPHome: no .ld in {} was patched for IRAM_ATTR. Update the " @@ -110,6 +132,20 @@ def _pre_link(target, source, env): build_dir ) ) + # Every .ld in the build must be patched. RTL8710B generates one .ld per + # OTA slot (xip1, xip2); if only one matches, the unpatched slot would + # ship with IRAM_ATTR code dropped to zeros and brick the device on the + # boot after an OTA into that slot. + if unpatched: + raise RuntimeError( + "ESPHome: {} of {} .ld file(s) in {} were not patched for " + "IRAM_ATTR: {}. The regex in patch_linker.py.script " + "(_PATCHERS_BY_VARIANT[{!r}]) matched the others but not " + "these. Update the regex to cover all linker scripts.".format( + len(unpatched), len(ld_files), build_dir, + ", ".join(unpatched), _variant, + ) + ) # Substrings matched against demangled names as a fallback on RTL8720C, diff --git a/esphome/components/packages/__init__.py b/esphome/components/packages/__init__.py index 06a64208b6..f3e0e0db8f 100644 --- a/esphome/components/packages/__init__.py +++ b/esphome/components/packages/__init__.py @@ -215,7 +215,7 @@ def _process_remote_package(config: dict[str, Any]) -> dict[str, Any]: If loading fails after cloning, attempts a revert and retry in case a prior cached checkout is stale. """ - repo_dir, revert = git.clone_or_update( + repo_root, revert = git.clone_or_update( url=config[CONF_URL], ref=config.get(CONF_REF), refresh=config[CONF_REFRESH], @@ -225,6 +225,10 @@ def _process_remote_package(config: dict[str, Any]) -> dict[str, Any]: ) files: list[dict[str, Any]] = [] + # ``repo_root`` is the directory containing ``.git`` and must be passed + # to git for symlink-stub resolution. ``repo_dir`` may be narrowed to a + # subdirectory via the user's CONF_PATH and is used for file lookups. + repo_dir = repo_root if base_path := config.get(CONF_PATH): repo_dir = repo_dir / base_path @@ -236,13 +240,37 @@ def _process_remote_package(config: dict[str, Any]) -> dict[str, Any]: def _load_package_yaml(yaml_file: Path, filename: str) -> dict: """Load a YAML file from a remote package, validating min_version.""" - try: - new_yaml = yaml_util.load_yaml(yaml_file) - except EsphomeError as e: + + def _load(path: Path) -> dict | str | None: + try: + return yaml_util.load_yaml(path) + except EsphomeError as e: + raise cv.Invalid( + f"{filename} is not a valid YAML file." + f" Please check the file contents.\n{e}" + ) from e + + new_yaml = _load(yaml_file) + if not isinstance(new_yaml, dict): + # On Windows, git defaults to core.symlinks=false unless the user + # has Developer Mode enabled or is running elevated. Files stored + # in the repo as symlinks (tree mode 120000) are then checked out + # as plain text files containing the symlink target path, so + # parsing them as YAML yields a bare scalar instead of a mapping. + # Best-effort: follow the symlink target ourselves and re-load. + target = git.resolve_symlink_stub(repo_root, yaml_file) + if target is not None: + new_yaml = _load(target) + if not isinstance(new_yaml, dict): raise cv.Invalid( - f"{filename} is not a valid YAML file." - f" Please check the file contents.\n{e}" - ) from e + f"{filename} does not contain a YAML mapping at the top level " + f"(got {type(new_yaml).__name__}). " + f"If this file is a git symlink in the source repository, it " + f"may not have been materialized correctly on your platform " + f"(this is a known issue with git on Windows without Developer " + f"Mode enabled). Try pointing your package at the real file " + f"path instead." + ) esphome_config = new_yaml.get(CONF_ESPHOME) or {} min_version = esphome_config.get(CONF_MIN_VERSION) if min_version is not None and cv.Version.parse(min_version) > cv.Version.parse( diff --git a/esphome/components/sx126x/sx126x.cpp b/esphome/components/sx126x/sx126x.cpp index 83afeac50a..aed0105e1f 100644 --- a/esphome/components/sx126x/sx126x.cpp +++ b/esphome/components/sx126x/sx126x.cpp @@ -394,7 +394,7 @@ void SX126x::run_image_cal() { buf[1] = 0xE9; } else if (this->frequency_ > 850000000) { buf[0] = 0xD7; - buf[1] = 0xD8; + buf[1] = 0xDB; } else if (this->frequency_ > 770000000) { buf[0] = 0xC1; buf[1] = 0xC5; diff --git a/esphome/const.py b/esphome/const.py index 39c5c6b60e..fdbbbe5eab 100644 --- a/esphome/const.py +++ b/esphome/const.py @@ -4,7 +4,7 @@ from enum import Enum from esphome.enum import StrEnum -__version__ = "2026.5.1" +__version__ = "2026.5.2" ALLOWED_NAME_CHARS = "abcdefghijklmnopqrstuvwxyz0123456789-_" VALID_SUBSTITUTIONS_CHARACTERS = ( diff --git a/esphome/core/config.py b/esphome/core/config.py index 5a98b94781..e4298b0865 100644 --- a/esphome/core/config.py +++ b/esphome/core/config.py @@ -711,6 +711,7 @@ async def to_code(config: ConfigType) -> None: # Process areas all_areas: list[dict[str, str | core.ID]] = [] if CONF_AREA in config: + CORE.area = config[CONF_AREA][CONF_NAME] all_areas.append(config[CONF_AREA]) all_areas.extend(config[CONF_AREAS]) diff --git a/esphome/git.py b/esphome/git.py index 0106f24845..c724ea2875 100644 --- a/esphome/git.py +++ b/esphome/git.py @@ -6,6 +6,7 @@ import logging from pathlib import Path import re import subprocess +import sys import urllib.parse import esphome.config_validation as cv @@ -93,6 +94,92 @@ def _compute_destination_path(key: str, domain: str) -> Path: return base_dir / h.hexdigest()[:8] +def resolve_symlink_stub(repo_dir: Path, file_path: Path) -> Path | None: + """Return the symlink target if ``file_path`` is a Windows-checked-out symlink stub. + + On Windows, when ``core.symlinks=false`` (the default unless the user has + SeCreateSymbolicLinkPrivilege — i.e. Developer Mode or running elevated), + git materializes files with tree mode ``120000`` as plain text files + whose content is the literal symlink target path. Opening such a file + yields the target path string instead of the target's content. + + If ``file_path`` is one of those stubs, return the resolved target Path + inside ``repo_dir``. Otherwise return ``None`` and the caller should use + ``file_path`` as-is. + + Designed to be called *only* when normal access has already produced an + unexpected result (e.g. YAML parsed as a top-level scalar), so the + per-file ``git ls-files`` subprocess cost is paid only on the failure + path. Returns ``None`` on any error or check failure — it's purely a + best-effort recovery, never raises. + """ + # On non-Windows, git creates real symlinks; ordinary file access already + # transparently follows them. + if sys.platform != "win32": + return None + if file_path.is_symlink(): + return None + if not file_path.is_file(): + return None + + try: + rel = file_path.relative_to(repo_dir) + except ValueError: + return None + + try: + # ``git ls-files -s `` prints " \t" + # for that single entry, or empty if untracked. + out = run_git_command( + ["git", "ls-files", "-s", "--", rel.as_posix()], + git_dir=repo_dir, + ) + except GitException: + return None + + parts = out.split() + if not parts or parts[0] != "120000": + return None + + # Stubs are short ASCII relative paths. Decode defensively, and only + # strip the trailing newline git's checkout may append — preserving any + # whitespace that could be part of a valid target name. + try: + raw = file_path.read_bytes() + except OSError: + return None + try: + target_str = raw.decode("utf-8").rstrip("\r\n") + except UnicodeDecodeError: + return None + + # ``Path()`` and ``Path.resolve()`` can raise on malformed inputs (e.g. + # embedded NUL bytes from a hostile symlink blob, paths too long for the + # OS, or temporary I/O errors). Catch broadly — this helper is purely a + # best-effort recovery and must never raise. + try: + target_path = (file_path.parent / target_str).resolve() + repo_root_resolved = repo_dir.resolve() + except (OSError, ValueError, RuntimeError): + return None + + # ``Path.resolve()`` follows ``..``; re-verify containment afterwards. + try: + target_path.relative_to(repo_root_resolved) + except ValueError: + _LOGGER.warning( + "Refusing to follow symlink %s -> %s (escapes repository)", + file_path, + target_str, + ) + return None + + if not target_path.is_file(): + return None + + return target_path + + def clone_or_update( *, url: str, diff --git a/esphome/storage_json.py b/esphome/storage_json.py index 7f8885ba5f..dc1576ab18 100644 --- a/esphome/storage_json.py +++ b/esphome/storage_json.py @@ -100,6 +100,7 @@ class StorageJSON: framework: str | None = None, core_platform: str | None = None, toolchain: str | None = None, + area: str | None = None, ) -> None: # Version of the storage JSON schema assert storage_version is None or isinstance(storage_version, int) @@ -138,6 +139,8 @@ class StorageJSON: self.core_platform = core_platform # The toolchain used for the build ("platformio" / "esp-idf") self.toolchain = toolchain + # The area of the node + self.area = area def as_dict(self): return { @@ -158,6 +161,7 @@ class StorageJSON: "framework": self.framework, "core_platform": self.core_platform, "toolchain": self.toolchain, + "area": self.area, } def to_json(self): @@ -195,6 +199,7 @@ class StorageJSON: framework=esph.target_framework, core_platform=esph.target_platform, toolchain=esph.toolchain.value if esph.toolchain is not None else None, + area=esph.area, ) @staticmethod @@ -243,6 +248,7 @@ class StorageJSON: framework = storage.get("framework") core_platform = storage.get("core_platform") toolchain = storage.get("toolchain") + area = storage.get("area") return StorageJSON( storage_version, name, @@ -261,6 +267,7 @@ class StorageJSON: framework, core_platform, toolchain, + area, ) @staticmethod diff --git a/esphome/writer.py b/esphome/writer.py index 72c2c355dc..192c9d68e8 100644 --- a/esphome/writer.py +++ b/esphome/writer.py @@ -87,6 +87,23 @@ def replace_file_content(text, pattern, repl): def storage_should_clean(old: StorageJSON | None, new: StorageJSON) -> bool: + """Return True when the build tree must be wiped before reuse. + + Predicate is True when *old* is missing (first build), + ``src_version`` differs, ``build_path`` differs, the build + ``toolchain`` differs (e.g. switching between the PlatformIO and + native ESP-IDF toolchains, which produce incompatible build trees), + or a previously loaded integration was removed in *new*. Adding + integrations or changing unrelated fields (friendly name, esphome + version, etc.) does not trigger a clean. + + Used by esphome-device-builder (esphome/device-builder) to gate + its remote-build artifact materialiser so a local → remote → local + cycle preserves PlatformIO's local object cache instead of wiping + it on every cycle. The signature, semantics, and ``None`` handling + for *old* are part of the public contract; keep them stable so the + offloader's wipe decision tracks core's. + """ if old is None: return True @@ -94,6 +111,8 @@ def storage_should_clean(old: StorageJSON | None, new: StorageJSON) -> bool: return True if old.build_path != new.build_path: return True + if old.toolchain != new.toolchain: + return True # Check if any components have been removed return bool(old.loaded_integrations - new.loaded_integrations) @@ -490,6 +509,10 @@ def clean_build(clear_pio_cache: bool = True): if dependencies_lock.is_file(): _LOGGER.info("Deleting %s", dependencies_lock) dependencies_lock.unlink() + idedata_cache = CORE.relative_internal_path("idedata", f"{CORE.name}.json") + if idedata_cache.is_file(): + _LOGGER.info("Deleting %s", idedata_cache) + idedata_cache.unlink() # Native ESP-IDF toolchain artifacts: the IDF CMake/ninja build dir # and the Component Manager's fetched managed components live under # the project's build path, not under .pioenvs / .piolibdeps. diff --git a/tests/unit_tests/core/test_config.py b/tests/unit_tests/core/test_config.py index 4ce862315d..39cd042a96 100644 --- a/tests/unit_tests/core/test_config.py +++ b/tests/unit_tests/core/test_config.py @@ -140,6 +140,33 @@ def test_multiple_areas_and_devices(yaml_file: Callable[[str], str]) -> None: } +@pytest.mark.asyncio +@pytest.mark.filterwarnings("ignore::RuntimeWarning") +@pytest.mark.parametrize( + ("fixture", "expected_area"), + [ + ("legacy_string_area.yaml", "Living Room"), + ("multiple_areas_devices.yaml", "Main Area"), + ], +) +async def test_to_code_records_core_area( + yaml_file: Callable[[str], Path], + fixture: str, + expected_area: str, +) -> None: + """``to_code`` records the node's area name on CORE for StorageJSON.""" + result = load_config_from_fixture(yaml_file, fixture, FIXTURES_DIR) + assert result is not None + assert CORE.area is None + + with patch("esphome.core.config.cg") as mock_cg: + mock_cg.RawStatement.side_effect = lambda *args, **kwargs: MagicMock() + mock_cg.RawExpression.side_effect = lambda *args, **kwargs: MagicMock() + await config.to_code(result[CONF_ESPHOME]) + + assert CORE.area == expected_area + + def test_legacy_string_area( yaml_file: Callable[[str], str], caplog: pytest.LogCaptureFixture ) -> None: diff --git a/tests/unit_tests/test_git.py b/tests/unit_tests/test_git.py index eab6bfc2cb..690c47c183 100644 --- a/tests/unit_tests/test_git.py +++ b/tests/unit_tests/test_git.py @@ -4,7 +4,7 @@ from datetime import datetime, timedelta import os from pathlib import Path from typing import Any -from unittest.mock import Mock +from unittest.mock import Mock, patch import pytest @@ -1001,3 +1001,304 @@ def test_refresh_picks_up_new_remote_commits( "--hard", "old_sha", ] + + +def test_resolve_symlink_stub_returns_none_on_non_windows( + tmp_path: Path, mock_run_git_command: Mock +) -> None: + """On non-Windows, resolve_symlink_stub returns None without calling git.""" + repo_dir = tmp_path / "repo" + repo_dir.mkdir() + stub = repo_dir / "file.yaml" + stub.write_text("static/file.yaml") + + with patch("esphome.git.sys.platform", "linux"): + result = git.resolve_symlink_stub(repo_dir, stub) + + assert result is None + mock_run_git_command.assert_not_called() + + +def test_resolve_symlink_stub_returns_target_for_mode_120000( + tmp_path: Path, mock_run_git_command: Mock +) -> None: + """A mode-120000 file is recognised as a stub; its target Path is returned.""" + repo_dir = tmp_path / "repo" + repo_dir.mkdir() + (repo_dir / "static").mkdir() + + target = repo_dir / "static" / "real.yaml" + target.write_text("esphome:\n name: real\n") + + stub = repo_dir / "real.yaml" + stub.write_text("static/real.yaml") + + mock_run_git_command.return_value = "120000 abc123 0\treal.yaml" + + with patch("esphome.git.sys.platform", "win32"): + result = git.resolve_symlink_stub(repo_dir, stub) + + assert result == target.resolve() + # Stub file itself was not modified — only inspected. + assert stub.read_text() == "static/real.yaml" + + +def test_resolve_symlink_stub_resolves_relative_parent_paths( + tmp_path: Path, mock_run_git_command: Mock +) -> None: + """Symlink targets with ``..`` segments resolve correctly within the repo.""" + repo_dir = tmp_path / "repo" + (repo_dir / "subdir").mkdir(parents=True) + (repo_dir / "static").mkdir() + + target = repo_dir / "static" / "shared.yaml" + target.write_text("shared content") + + stub = repo_dir / "subdir" / "shared.yaml" + stub.write_text("../static/shared.yaml") + + mock_run_git_command.return_value = "120000 abc123 0\tsubdir/shared.yaml" + + with patch("esphome.git.sys.platform", "win32"): + result = git.resolve_symlink_stub(repo_dir, stub) + + assert result == target.resolve() + + +def test_resolve_symlink_stub_refuses_escape_outside_repo( + tmp_path: Path, mock_run_git_command: Mock +) -> None: + """A symlink pointing outside the repository is not followed.""" + outside = tmp_path / "outside.yaml" + outside.write_text("sensitive") + + repo_dir = tmp_path / "repo" + repo_dir.mkdir() + + stub = repo_dir / "escape.yaml" + stub.write_text("../outside.yaml") + + mock_run_git_command.return_value = "120000 abc123 0\tescape.yaml" + + with patch("esphome.git.sys.platform", "win32"): + result = git.resolve_symlink_stub(repo_dir, stub) + + assert result is None + + +def test_resolve_symlink_stub_returns_none_for_real_symlink( + tmp_path: Path, mock_run_git_command: Mock +) -> None: + """A real symlink already opens transparently, so the helper short-circuits. + + Skipped on Windows where symlink creation requires + SeCreateSymbolicLinkPrivilege. + """ + if os.name == "nt": + pytest.skip("Requires symlink-creation privilege on Windows") + + repo_dir = tmp_path / "repo" + repo_dir.mkdir() + target = repo_dir / "real.yaml" + target.write_text("real content") + + real_link = repo_dir / "link.yaml" + real_link.symlink_to("real.yaml") + + with patch("esphome.git.sys.platform", "win32"): + result = git.resolve_symlink_stub(repo_dir, real_link) + + assert result is None + # No git call needed for real symlinks. + mock_run_git_command.assert_not_called() + + +def test_resolve_symlink_stub_returns_none_for_regular_file( + tmp_path: Path, mock_run_git_command: Mock +) -> None: + """A regular file (mode 100644) whose content looks path-shaped is not + followed.""" + repo_dir = tmp_path / "repo" + repo_dir.mkdir() + + regular = repo_dir / "looks_like_path.txt" + regular.write_text("static/something.yaml") + + mock_run_git_command.return_value = "100644 abc123 0\tlooks_like_path.txt" + + with patch("esphome.git.sys.platform", "win32"): + result = git.resolve_symlink_stub(repo_dir, regular) + + assert result is None + + +def test_resolve_symlink_stub_returns_none_when_git_fails( + tmp_path: Path, mock_run_git_command: Mock +) -> None: + """If ``git ls-files`` fails (e.g. not a repo), the helper returns None.""" + repo_dir = tmp_path / "repo" + repo_dir.mkdir() + + stub = repo_dir / "real.yaml" + stub.write_text("static/real.yaml") + + mock_run_git_command.side_effect = GitCommandError("ls-files exploded") + + with patch("esphome.git.sys.platform", "win32"): + result = git.resolve_symlink_stub(repo_dir, stub) + + assert result is None + + +def test_resolve_symlink_stub_returns_none_for_non_utf8_content( + tmp_path: Path, mock_run_git_command: Mock +) -> None: + """A file whose bytes are not valid UTF-8 must not raise — return None.""" + repo_dir = tmp_path / "repo" + repo_dir.mkdir() + + stub = repo_dir / "binary.bin" + stub.write_bytes(b"\xff\xfe\x00\xff") + + mock_run_git_command.return_value = "120000 abc123 0\tbinary.bin" + + with patch("esphome.git.sys.platform", "win32"): + result = git.resolve_symlink_stub(repo_dir, stub) + + assert result is None + + +def test_resolve_symlink_stub_preserves_whitespace_in_target( + tmp_path: Path, mock_run_git_command: Mock +) -> None: + """Only trailing CR/LF is stripped — internal whitespace is preserved.""" + repo_dir = tmp_path / "repo" + repo_dir.mkdir() + target_dir = repo_dir / "dir with spaces" + target_dir.mkdir() + target = target_dir / "real.yaml" + target.write_text("hello") + + stub = repo_dir / "link.yaml" + # Trailing newline (as git's checkout may append) is stripped, but + # whitespace inside the target path itself must survive. + stub.write_bytes(b"dir with spaces/real.yaml\n") + + mock_run_git_command.return_value = "120000 abc123 0\tlink.yaml" + + with patch("esphome.git.sys.platform", "win32"): + result = git.resolve_symlink_stub(repo_dir, stub) + + assert result == target.resolve() + + +def test_resolve_symlink_stub_returns_none_for_directory_target( + tmp_path: Path, mock_run_git_command: Mock +) -> None: + """A symlink pointing at a directory has no file content to load.""" + repo_dir = tmp_path / "repo" + repo_dir.mkdir() + (repo_dir / "dir_target").mkdir() + + stub = repo_dir / "link_to_dir" + stub.write_text("dir_target") + + mock_run_git_command.return_value = "120000 abc123 0\tlink_to_dir" + + with patch("esphome.git.sys.platform", "win32"): + result = git.resolve_symlink_stub(repo_dir, stub) + + assert result is None + + +def test_resolve_symlink_stub_returns_none_when_resolve_raises( + tmp_path: Path, mock_run_git_command: Mock +) -> None: + """Path.resolve() raising (e.g. on a malformed target) must not propagate.""" + repo_dir = tmp_path / "repo" + repo_dir.mkdir() + + stub = repo_dir / "broken.yaml" + stub.write_text("ignored") + + mock_run_git_command.return_value = "120000 abc123 0\tbroken.yaml" + + with ( + patch("esphome.git.sys.platform", "win32"), + patch.object(Path, "resolve", side_effect=OSError("bad path")), + ): + result = git.resolve_symlink_stub(repo_dir, stub) + + assert result is None + + +def test_resolve_symlink_stub_returns_none_when_file_missing( + tmp_path: Path, mock_run_git_command: Mock +) -> None: + """A file path that doesn't exist is rejected before git is consulted.""" + repo_dir = tmp_path / "repo" + repo_dir.mkdir() + + missing = repo_dir / "ghost.yaml" # not created + + with patch("esphome.git.sys.platform", "win32"): + result = git.resolve_symlink_stub(repo_dir, missing) + + assert result is None + mock_run_git_command.assert_not_called() + + +def test_resolve_symlink_stub_returns_none_when_path_outside_repo( + tmp_path: Path, mock_run_git_command: Mock +) -> None: + """A file path that isn't under repo_dir is rejected (ValueError from relative_to).""" + repo_dir = tmp_path / "repo" + repo_dir.mkdir() + + outside = tmp_path / "stray.yaml" + outside.write_text("something") + + with patch("esphome.git.sys.platform", "win32"): + result = git.resolve_symlink_stub(repo_dir, outside) + + assert result is None + mock_run_git_command.assert_not_called() + + +def test_resolve_symlink_stub_returns_none_when_untracked( + tmp_path: Path, mock_run_git_command: Mock +) -> None: + """Empty `git ls-files` output (untracked file) makes the helper return None.""" + repo_dir = tmp_path / "repo" + repo_dir.mkdir() + + stub = repo_dir / "untracked.yaml" + stub.write_text("static/foo.yaml") + + mock_run_git_command.return_value = "" + + with patch("esphome.git.sys.platform", "win32"): + result = git.resolve_symlink_stub(repo_dir, stub) + + assert result is None + + +def test_resolve_symlink_stub_returns_none_when_read_bytes_raises( + tmp_path: Path, mock_run_git_command: Mock +) -> None: + """An OSError from read_bytes() (e.g. file vanished mid-call) must not propagate.""" + repo_dir = tmp_path / "repo" + repo_dir.mkdir() + + stub = repo_dir / "racy.yaml" + stub.write_text("static/racy.yaml") + + mock_run_git_command.return_value = "120000 abc123 0\tracy.yaml" + + with ( + patch("esphome.git.sys.platform", "win32"), + patch.object(Path, "read_bytes", side_effect=OSError("vanished")), + ): + result = git.resolve_symlink_stub(repo_dir, stub) + + assert result is None diff --git a/tests/unit_tests/test_storage_json.py b/tests/unit_tests/test_storage_json.py index ea37492cf4..2a6f22abb1 100644 --- a/tests/unit_tests/test_storage_json.py +++ b/tests/unit_tests/test_storage_json.py @@ -205,6 +205,7 @@ def test_storage_json_as_dict() -> None: no_mdns=True, framework="arduino", core_platform="esp32", + area="Living Room", ) result = storage.as_dict() @@ -233,6 +234,7 @@ def test_storage_json_as_dict() -> None: assert result["no_mdns"] is True assert result["framework"] == "arduino" assert result["core_platform"] == "esp32" + assert result["area"] == "Living Room" def test_storage_json_to_json() -> None: @@ -309,6 +311,7 @@ def test_storage_json_from_esphome_core(setup_core: Path) -> None: mock_core.config = {CONF_MDNS: {CONF_DISABLED: True}} mock_core.target_framework = "esp-idf" mock_core.toolchain = Toolchain.ESP_IDF + mock_core.area = "Living Room" with patch("esphome.components.esp32.get_esp32_variant") as mock_variant: mock_variant.return_value = "ESP32-C3" @@ -329,6 +332,7 @@ def test_storage_json_from_esphome_core(setup_core: Path) -> None: assert result.framework == "esp-idf" assert result.core_platform == "esp32" assert result.toolchain == "esp-idf" + assert result.area == "Living Room" def test_storage_json_from_esphome_core_mdns_enabled(setup_core: Path) -> None: @@ -729,3 +733,37 @@ def test_storage_json_load_legacy_esphomeyaml_version(tmp_path: Path) -> None: assert result is not None assert result.esphome_version == "1.14.0" # Should map to esphome_version + + +def test_storage_json_load_area(tmp_path: Path) -> None: + """``area`` round-trips through load; absence loads as None.""" + file_path = tmp_path / "with_area.json" + file_path.write_text( + json.dumps( + { + "storage_version": 1, + "name": "lamp", + "friendly_name": "Lamp", + "esp_platform": "ESP32", + "area": "Living Room", + } + ) + ) + result = storage_json.StorageJSON.load(file_path) + assert result is not None + assert result.area == "Living Room" + + legacy_path = tmp_path / "no_area.json" + legacy_path.write_text( + json.dumps( + { + "storage_version": 1, + "name": "lamp", + "friendly_name": "Lamp", + "esp_platform": "ESP32", + } + ) + ) + legacy = storage_json.StorageJSON.load(legacy_path) + assert legacy is not None + assert legacy.area is None diff --git a/tests/unit_tests/test_substitutions.py b/tests/unit_tests/test_substitutions.py index 4783112578..4ff857951f 100644 --- a/tests/unit_tests/test_substitutions.py +++ b/tests/unit_tests/test_substitutions.py @@ -838,3 +838,86 @@ def test_include_vars_applied_to_lambda_value(tmp_path: Path) -> None: assert isinstance(result["value"], Lambda) assert result["value"].value == 'return "bar";' + + +@patch("esphome.git.resolve_symlink_stub") +@patch("esphome.git.clone_or_update") +def test_remote_package_symlink_stub_is_followed( + mock_clone_or_update: MagicMock, + mock_resolve_symlink_stub: MagicMock, + tmp_path: Path, +) -> None: + """When a package YAML is a scalar (symlink stub) and resolve_symlink_stub + returns a target, the loader follows the target and uses its content.""" + CORE.config_path = tmp_path / "test.yaml" + + repo_dir = tmp_path / "repo" + repo_dir.mkdir() + (repo_dir / "static").mkdir() + + # Stub file: content is the target path string (simulating Windows behavior). + stub = repo_dir / "file1.yaml" + stub.write_text("static/file1.yaml") + + # Real target with valid YAML mapping. + target = repo_dir / "static" / "file1.yaml" + target.write_text("substitutions:\n hello: world\n") + + mock_clone_or_update.return_value = (repo_dir, None) + mock_resolve_symlink_stub.return_value = target + + config: dict[str, Any] = { + "packages": { + "test_package": { + "url": "https://github.com/esphome/repo1", + "ref": "main", + "files": ["file1.yaml"], + } + } + } + + # Must succeed (does not raise the helpful cv.Invalid) because the stub + # was followed and a valid mapping was loaded from the target. + do_packages_pass(config) + assert mock_resolve_symlink_stub.called + + +@patch("esphome.git.clone_or_update") +def test_remote_package_scalar_yaml_raises_helpful_error( + mock_clone_or_update: MagicMock, tmp_path: Path +) -> None: + """A remote package YAML that is a top-level scalar (e.g. an unmaterialized + git symlink on Windows) raises a clear cv.Invalid, not AttributeError. + + Regression test for the case where a repo containing a YAML symlink, + checked out on Windows without symlink privilege, lands as a short text + file containing the symlink target path. PyYAML parses that as a bare + string scalar; the package loader must reject it with a human-readable + error instead of dying inside ``.get()``. + """ + CORE.config_path = tmp_path / "test.yaml" + + repo_dir = tmp_path / "repo" + repo_dir.mkdir() + # Simulate the broken-symlink state: a YAML file whose entire content is + # the symlink target string. PyYAML parses this as a top-level scalar. + (repo_dir / "file1.yaml").write_text("static/file1.yaml") + + mock_clone_or_update.return_value = (repo_dir, None) + + config: dict[str, Any] = { + "packages": { + "test_package": { + "url": "https://github.com/esphome/repo1", + "ref": "main", + "files": ["file1.yaml"], + } + } + } + + with pytest.raises(cv.Invalid) as exc_info: + do_packages_pass(config) + + msg = str(exc_info.value) + assert "mapping at the top level" in msg + assert "file1.yaml" in msg diff --git a/tests/unit_tests/test_writer.py b/tests/unit_tests/test_writer.py index 91b4bd8e87..be37dd5d58 100644 --- a/tests/unit_tests/test_writer.py +++ b/tests/unit_tests/test_writer.py @@ -75,6 +75,7 @@ def create_storage() -> Callable[..., StorageJSON]: no_mdns=kwargs.get("no_mdns", False), framework=kwargs.get("framework", "arduino"), core_platform=kwargs.get("core_platform", "esp32"), + toolchain=kwargs.get("toolchain", "platformio"), ) return _create @@ -106,6 +107,20 @@ def test_storage_should_clean_when_build_path_changes( assert storage_should_clean(old, new) is True +def test_storage_should_clean_when_toolchain_changes( + create_storage: Callable[..., StorageJSON], +) -> None: + """Test that clean is triggered when the build toolchain changes. + + Switching between the PlatformIO and native ESP-IDF toolchains produces + incompatible build trees (and toolchain-specific idedata), so the build + must be wiped. + """ + old = create_storage(loaded_integrations=["api", "wifi"], toolchain="platformio") + new = create_storage(loaded_integrations=["api", "wifi"], toolchain="esp-idf") + assert storage_should_clean(old, new) is True + + def test_storage_should_clean_when_component_removed( create_storage: Callable[..., StorageJSON], ) -> None: @@ -443,6 +458,11 @@ def test_clean_build( dependencies_lock = tmp_path / "dependencies.lock" dependencies_lock.write_text("lock file") + # idedata cache lives under the data dir, not the build path. + idedata_cache = tmp_path / "idedata" / "test.json" + idedata_cache.parent.mkdir() + idedata_cache.write_text("{}") + # Native ESP-IDF toolchain artifacts. idf_build_dir = tmp_path / "build" idf_build_dir.mkdir() @@ -463,11 +483,14 @@ def test_clean_build( mock_core.relative_pioenvs_path.return_value = pioenvs_dir mock_core.relative_piolibdeps_path.return_value = piolibdeps_dir mock_core.relative_build_path.side_effect = lambda name: tmp_path / name + mock_core.name = "test" + mock_core.relative_internal_path.side_effect = tmp_path.joinpath # Verify all exist before assert pioenvs_dir.exists() assert piolibdeps_dir.exists() assert dependencies_lock.exists() + assert idedata_cache.exists() assert idf_build_dir.exists() assert managed_components_dir.exists() assert platformio_cache_dir.exists() @@ -492,6 +515,7 @@ def test_clean_build( assert not pioenvs_dir.exists() assert not piolibdeps_dir.exists() assert not dependencies_lock.exists() + assert not idedata_cache.exists() assert not idf_build_dir.exists() assert not managed_components_dir.exists() assert not platformio_cache_dir.exists() @@ -501,6 +525,7 @@ def test_clean_build( assert ".pioenvs" in caplog.text assert ".piolibdeps" in caplog.text assert "dependencies.lock" in caplog.text + assert str(idedata_cache) in caplog.text assert str(idf_build_dir) in caplog.text assert str(managed_components_dir) in caplog.text assert "PlatformIO cache" in caplog.text