diff --git a/esphome/arduino/library.py b/esphome/arduino/library.py index 75ce906b77..443438a725 100644 --- a/esphome/arduino/library.py +++ b/esphome/arduino/library.py @@ -28,7 +28,6 @@ from esphome.platformio.library import ( ConvertedLibrary, InvalidLibrary, LibraryBackend, - _parse_library_json, check_library_data, collect_filtered_files, convert_libraries, @@ -37,6 +36,7 @@ from esphome.platformio.library import ( lex_build_flags, lib_ignore_set, normalize_dependencies, + parse_library_json, parse_library_properties, ) @@ -138,7 +138,7 @@ def _bundled_library(framework_path: Path, name: str) -> ArduinoLibrary: lib_dir = framework_path / "libraries" / name manifest_json = lib_dir / "library.json" if manifest_json.is_file(): - data = _parse_library_json(manifest_json) + data = parse_library_json(manifest_json) else: manifest = lib_dir / "library.properties" data = parse_library_properties(manifest) if manifest.is_file() else {} @@ -174,6 +174,9 @@ def resolve_libraries( and "/" not in library.name and (framework_path / "libraries" / library.name).is_dir() ): + # A bundled library's own manifest dependencies are deliberately + # not walked (PlatformIO's lib_ldf_mode=off does not either); + # core add_library() calls list what they need explicitly. bundled.append(_bundled_library(framework_path, library.name)) else: external.append(library) diff --git a/esphome/arduino8266/framework.py b/esphome/arduino8266/framework.py index 475e61b6c0..da10502bc2 100644 --- a/esphome/arduino8266/framework.py +++ b/esphome/arduino8266/framework.py @@ -24,14 +24,11 @@ import os from pathlib import Path from typing import NamedTuple +from esphome.build_helpers.ccache import ccache_defaults_env, resolve_ccache_path from esphome.build_helpers.ninja import find_ninja +from esphome.build_helpers.tools_cache import tools_cache_path from esphome.core import EsphomeError, Version -from esphome.framework_helpers import ( - ccache_defaults_env, - resolve_ccache_path, - str_to_lst_of_str, - tools_cache_path, -) +from esphome.framework_helpers import str_to_lst_of_str from esphome.platformio.registry import install_package _LOGGER = logging.getLogger(__name__) diff --git a/esphome/build_gen/arduino8266.py b/esphome/build_gen/arduino8266.py index 1d72971434..645df0131c 100644 --- a/esphome/build_gen/arduino8266.py +++ b/esphome/build_gen/arduino8266.py @@ -153,6 +153,8 @@ _CCFLAGS = [ "-free", "-fipa-pta", ] +# Upstream's -u _scanf_float is deliberately absent: it is re-added from +# KEY_SCANF_FLOAT at emission (the remove_float_scanf extra script's job). _LINKFLAGS = [ "-Os", "-nostdlib", @@ -207,14 +209,10 @@ def _flag_defines() -> dict[str, str]: """Map define name -> full ``NAME[=VALUE]`` for every -D build flag.""" defines: dict[str, str] = {} for flag in CORE.build_flags: - # Shell-lex multi-token entries the way PlatformIO does, so a knob - # in "-DKNOB -DOTHER" or a spaced "-D KNOB" is still detected; - # single tokens pass verbatim to keep quoting in their bodies intact. - tokens = ( - join_flag_args(split_flag_entry(flag, "esphome"), "esphome") - if " " in flag - else (flag,) - ) + # Shell-lex every entry the way PlatformIO's ParseFlags does, so a + # knob in "-DKNOB -DOTHER", a spaced "-D KNOB", and quoted bodies all + # read identically to _project_flags (and the compile line). + tokens = join_flag_args(split_flag_entry(flag, "esphome"), "esphome") for tok in tokens: if tok.startswith("-D") and len(tok) > 2: body = tok[2:] @@ -266,6 +264,15 @@ def _resolve_build_config(defines: dict[str, str]) -> _BuildConfig: body for name, body in defines.items() if name.startswith("MMU_") ) else: + if "MMU_IRAM_SIZE" in defines or "MMU_ICACHE_SIZE" in defines: + # Same diagnostic the PlatformIO builder prints: without the + # knob the linker script keeps the default layout while the + # compile line carries the custom sizes + _LOGGER.warning( + "Detected custom MMU flags; use " + "-DPIO_FRAMEWORK_ARDUINO_MMU_CUSTOM to disable the " + "default configuration" + ) mmu = list(_MMU_DEFAULT) return _BuildConfig( @@ -495,6 +502,9 @@ def write_project(paths: InstalledPaths) -> bool: cache_key="arduino8266", ) + if not src_dir.is_dir(): + # Generated project state, not install state: clean-all would not help + raise EsphomeError(f"Generated source directory {src_dir} is missing") # A missing install directory would otherwise surface as a wall of # include errors; failing here names the path instead. include_dirs = [ @@ -505,7 +515,7 @@ def write_project(paths: InstalledPaths) -> bool: sdk / "lwip2" / "include", variant_dir, ] - for required in include_dirs: + for required in include_dirs[1:]: if not required.is_dir(): raise EsphomeError( f"{_INCOMPLETE_INSTALL}: missing {required}; {_CLEAN_HINT}" @@ -534,7 +544,14 @@ def write_project(paths: InstalledPaths) -> bool: + common + get_project_cxx_compile_flags() ) - asflags = _ASFLAGS + defines + includes + project_compile_flags + # PlatformIO's ASPPCOM carries defines and includes but not CCFLAGS, + # so only -D/-I user flags reach assembly there; match it. + asflags = ( + _ASFLAGS + + defines + + includes + + [f for f in project_compile_flags if f.startswith(("-D", "-I"))] + ) # build_unflags applies to the framework flag sets too (compile and link), # as under PlatformIO (a silently ignored ``build_unflags: -Os`` would @@ -546,7 +563,7 @@ def write_project(paths: InstalledPaths) -> bool: if esp8266_data[KEY_SCANF_FLOAT]: link_flags += ["-u", "_scanf_float"] link_flags += project_link_flags - link_flags += [flag for lib in libraries for flag in lib.link_flags] + link_flags += [_shell_token(flag) for lib in libraries for flag in lib.link_flags] flash_ld = _active_flash_ld_name(flash_ld_name) link_flags += ["-T", flash_ld] @@ -616,7 +633,7 @@ def write_project(paths: InstalledPaths) -> bool: f"asflags = {' '.join(asflags)}", f"linkflags = {' '.join(link_flags)}", f"libdirflags = {' '.join(f'-L{_q(d)}' for d in lib_dirs)}", - f"libflags = {' '.join(f'-l{lib}' for lib in system_libs)}", + f"libflags = {' '.join(_shell_token(f'-l{lib}') for lib in system_libs)}", "", ] @@ -625,7 +642,8 @@ def write_project(paths: InstalledPaths) -> bool: core_exclude |= _CORE_EXCLUDE_WAVEFORM archives = [] - variant_sources = _collect_sources(variant_dir) if variant_dir.is_dir() else [] + # variant_dir existence was already enforced with the include dirs + variant_sources = _collect_sources(variant_dir) if variant_sources: objs = _ninja_compile_edges(lines, variant_sources, variant_dir, "variant") lines.append(f"build libFrameworkArduinoVariant.a: ar {' '.join(objs)}") diff --git a/esphome/build_gen/build_tool.py b/esphome/build_gen/build_tool.py index ef6b1d4cc2..692d006661 100644 --- a/esphome/build_gen/build_tool.py +++ b/esphome/build_gen/build_tool.py @@ -24,9 +24,13 @@ def main() -> int: # Expand the response file here instead of passing @rspfile: GNU ar # treats backslashes in response files as escapes, corrupting Windows # paths ("sub\a.o" -> "suba.o"). - objects = Path(rspfile).read_text(encoding="utf-8").split() + # One path per line (rspfile_content = $in_newline, written without + # escaping), so a path containing a space survives. Expanding into + # argv trades away the OS command-line length limit rspfiles dodge; + # the relative object paths used here stay far below it. + objects = Path(rspfile).read_text(encoding="utf-8").splitlines() return subprocess.run( - [ar, "rc", archive, *objects], check=False, close_fds=False + [ar, "rc", archive, *filter(None, objects)], check=False, close_fds=False ).returncode if mode == "copy": src, dst = sys.argv[2:4] diff --git a/esphome/build_helpers/ccache.py b/esphome/build_helpers/ccache.py new file mode 100644 index 0000000000..11d40ac3c6 --- /dev/null +++ b/esphome/build_helpers/ccache.py @@ -0,0 +1,98 @@ +"""Shared ccache policy for build backends. + +One place for the probe, the enable/override rules, and the ``CCACHE_*`` +defaults, so the backends cannot drift apart. +""" + +from __future__ import annotations + +import logging +import os +from pathlib import Path +import subprocess + +from esphome.framework_helpers import strip_win_long_path_prefix + +_LOGGER = logging.getLogger(__name__) + + +def _ccache_runs(ccache: str) -> bool: + """Return True when the ``ccache`` found on PATH actually runs. + + ``shutil.which`` proves existence, not runnability: on Windows it also + matches ``.bat``/``.cmd`` wrappers and stale package-manager shims whose + target is gone. Wrapping compiles around such a find fails every compile + step with an opaque OS error, so probe once and fall back to compiling + without ccache when the probe fails. + """ + try: + subprocess.run( + [ccache, "--version"], + check=True, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + timeout=15, + close_fds=False, + ) + except (OSError, subprocess.SubprocessError): + _LOGGER.warning( + "Ignoring ccache at %s because it failed to run; compiling without ccache", + ccache, + ) + return False + return True + + +def resolve_ccache_path() -> str | None: + """The ccache binary to wrap compiles with, or None when disabled. + + Shared policy for every backend: on by default when a runnable ccache is + on PATH, ``ESPHOME_CCACHE_ENABLE=0`` opts out, and an explicit ``=1`` + warns when no binary is found and skips the runnability probe. The + Windows extended-length prefix is stripped before probing so the probe + validates the exact string the build will execute (#18399). + """ + import shutil + + from esphome.helpers import get_bool_env + + explicit = "ESPHOME_CCACHE_ENABLE" in os.environ + if explicit and not get_bool_env("ESPHOME_CCACHE_ENABLE"): + return None + ccache = shutil.which("ccache") + if ccache is None: + if explicit: + _LOGGER.warning( + "ESPHOME_CCACHE_ENABLE is set but no ccache binary is on PATH; " + "compiling without ccache" + ) + return None + ccache = strip_win_long_path_prefix(ccache) + if not explicit and not _ccache_runs(ccache): + return None + return ccache + + +def ccache_defaults_env(cache_dir: Path) -> dict[str, str]: + """Default ``CCACHE_*`` values for a build subprocess (not os.environ). + + Values the user already set in the environment are respected. Depend + mode is on: both native backends emit depfiles (-MMD / CMake), which + keeps cache-miss overhead low. + """ + from esphome.core import CORE + + # build_path is set during preload for every config-loading command; unset + # means the caller built the environment too early. Fail loudly rather + # than silently drop CCACHE_BASEDIR (losing cross-device cache hits). + if CORE.build_path is None: + raise ValueError( + "CORE.build_path must be set before constructing the build environment" + ) + defaults = { + "CCACHE_DIR": str(cache_dir), + "CCACHE_NOHASHDIR": "true", + "CCACHE_DEPEND": "1", + "CCACHE_BASEDIR": str(Path(CORE.build_path).resolve()), + } + return {k: v for k, v in defaults.items() if k not in os.environ} diff --git a/esphome/build_helpers/ninja.py b/esphome/build_helpers/ninja.py index 9a9189e8ee..6a7bac1f56 100644 --- a/esphome/build_helpers/ninja.py +++ b/esphome/build_helpers/ninja.py @@ -52,7 +52,10 @@ def quote_arg(tok: str) -> str: return f'"{quoted}"' -_NEEDS_QUOTE = re.compile(r'[\s"\']') +# Force-quote any token containing a character outside the shlex.quote-style +# safe set: ninja hands POSIX commands to /bin/sh -c, so bare (, ;, <, *, ` +# and friends would be re-parsed as shell syntax. +_NEEDS_QUOTE = re.compile(r"[^\w@%+=:,./-]") def shell_token(tok: str, force: bool = False) -> str: diff --git a/esphome/build_helpers/tools_cache.py b/esphome/build_helpers/tools_cache.py new file mode 100644 index 0000000000..1960db458f --- /dev/null +++ b/esphome/build_helpers/tools_cache.py @@ -0,0 +1,22 @@ +"""Machine-global tools cache location shared by the native backends.""" + +from __future__ import annotations + +from pathlib import Path + + +def tools_cache_path(env_var: str, subdir: str) -> Path: + """A backend's machine-global tools directory, with an env override. + + A blank/whitespace override is treated as unset: ``Path("")`` resolves + to the CWD, which ``clean-all`` would then delete. + """ + import platformdirs + + from esphome.helpers import get_str_env + + if prefix := get_str_env(env_var, "").strip(): + return Path(prefix).expanduser().resolve() + return ( + Path(platformdirs.user_cache_dir("esphome", appauthor=False)) / subdir + ).resolve() diff --git a/esphome/components/esp8266/boards.py b/esphome/components/esp8266/boards.py index 2646682766..4f137e95cd 100644 --- a/esphome/components/esp8266/boards.py +++ b/esphome/components/esp8266/boards.py @@ -361,31 +361,30 @@ BOARDS = { }, } -""" -ESP8266_BOARD_BUILD generate with (v4.2.1 is the platform version the -native toolchain mirrors; regenerate against the tag when bumping it): - -git clone -b v4.2.1 https://github.com/platformio/platform-espressif8266 -python3 - <<'EOF' -import json, glob, os -for f in sorted(glob.glob("platform-espressif8266/boards/*.json")): - b = json.load(open(f))["build"] - extra = b["extra_flags"] - extra = extra.split() if isinstance(extra, str) else extra - defines = [ - e[2:] for e in extra if e not in ("-DESP8266", "-DARDUINO_ARCH_ESP8266") - ] - entries = ", ".join(f'"{d}"' for d in defines) + ("," if len(defines) == 1 else "") - board = os.path.splitext(os.path.basename(f))[0] - print(f' "{board}": {{"variant": "{b["variant"]}", "defines": ({entries})}},') -EOF -""" # Per-board Arduino core build metadata for the native (PlatformIO-free) # toolchain: the variant directory (supplies pins_arduino.h) and the # board-identity defines the PlatformIO builder passes via build.extra_flags. # -DESP8266 and -DARDUINO_ARCH_ESP8266 are shared by every board and added by # the generator; only the per-board defines are listed here. +# +# ESP8266_BOARD_BUILD generate with (v4.2.1 is the platform version the +# native toolchain mirrors; regenerate against the tag when bumping it): +# +# git clone -b v4.2.1 https://github.com/platformio/platform-espressif8266 +# python3 - <<'EOF' +# import json, glob, os +# for f in sorted(glob.glob("platform-espressif8266/boards/*.json")): +# b = json.load(open(f))["build"] +# extra = b["extra_flags"] +# extra = extra.split() if isinstance(extra, str) else extra +# defines = [ +# e[2:] for e in extra if e not in ("-DESP8266", "-DARDUINO_ARCH_ESP8266") +# ] +# entries = ", ".join(f'"{d}"' for d in defines) + ("," if len(defines) == 1 else "") +# board = os.path.splitext(os.path.basename(f))[0] +# print(f' "{board}": {{"variant": "{b["variant"]}", "defines": ({entries})}},') +# EOF ESP8266_BOARD_BUILD = { "agruminolemon": { "variant": "agruminolemonv4", diff --git a/esphome/components/esp8266/build_surgery.py b/esphome/components/esp8266/build_surgery.py index 47f9042f08..97ce750dd5 100644 --- a/esphome/components/esp8266/build_surgery.py +++ b/esphome/components/esp8266/build_surgery.py @@ -82,6 +82,14 @@ def apply_testing_memory_patches(content: str, segments: Collection[str]) -> str raises, since a silently kept real memory limit would fail grouped builds far from the cause. """ + for segment in _TESTING_SEGMENT_SIZES: + if segment not in segments and _segment_line_re(segment).search(content): + # A known segment left unpatched would keep its real memory limit + # and silently under-provision the testing build + raise RuntimeError( + f"Testing-mode segment {segment} is present in the linker " + "script but was not selected for patching" + ) for segment in segments: if segment not in _TESTING_SEGMENT_SIZES: raise RuntimeError(f"Unknown testing-mode segment {segment!r}") @@ -103,15 +111,14 @@ def segment_length(content: str, segment_name: str) -> int | None: def surgery_fingerprint() -> str: - """Fingerprint of every behavioral input to the surgeries. + """Fingerprint of this module's source, covering every behavioral input. - Linker-script caches include it so an edit here invalidates them. + Linker-script caches include it so an edit here invalidates them; hashing + the source over-invalidates on comment edits, which is the safe direction. Native-toolchain-only, like ``segment_length``; no script twin. """ - parts = ( - RATETABLE_RULE, - _RATETABLE_COMMENT, - _RATETABLE_ANCHOR.pattern, - repr(sorted(_TESTING_SEGMENT_SIZES.items())), - ) - return hashlib.sha256("|".join(parts).encode()).hexdigest() + import inspect + import sys + + source = inspect.getsource(sys.modules[__name__]) + return hashlib.sha256(source.encode()).hexdigest() diff --git a/esphome/components/nrf52/framework.py b/esphome/components/nrf52/framework.py index 48963fa89f..578e91fe40 100644 --- a/esphome/components/nrf52/framework.py +++ b/esphome/components/nrf52/framework.py @@ -7,6 +7,7 @@ import shutil import sys import tempfile +from esphome.build_helpers.tools_cache import tools_cache_path import esphome.config_validation as cv from esphome.const import KEY_CORE, KEY_FRAMEWORK_VERSION from esphome.core import CORE, EsphomeError @@ -18,7 +19,6 @@ from esphome.framework_helpers import ( rmdir, run_command_ok, str_to_lst_of_str, - tools_cache_path, ) _LOGGER = logging.getLogger(__name__) diff --git a/esphome/espidf/framework.py b/esphome/espidf/framework.py index b4b03ce113..773fccdf18 100644 --- a/esphome/espidf/framework.py +++ b/esphome/espidf/framework.py @@ -11,11 +11,12 @@ import re import shutil from typing import Any, NoReturn +from esphome.build_helpers.ccache import ccache_defaults_env +from esphome.build_helpers.tools_cache import tools_cache_path from esphome.core import Version from esphome.framework_helpers import ( PathType, archive_extract_all, - ccache_defaults_env, create_venv, download_from_mirrors, download_with_resume, @@ -25,7 +26,6 @@ from esphome.framework_helpers import ( run_command, run_command_ok, str_to_lst_of_str, - tools_cache_path, ) from esphome.helpers import get_bool_env, write_file_if_changed @@ -89,7 +89,7 @@ def get_idf_tools_path() -> Path: Path object pointing to the ESP-IDF tools directory """ # Machine-global so all projects share the multi-GB install instead of - # a per-config-directory copy; see framework_helpers.tools_cache_path + # a per-config-directory copy; see build_helpers.tools_cache.tools_cache_path # for the env-override and normalization rules. return tools_cache_path("ESPHOME_ESP_IDF_PREFIX", "idf") diff --git a/esphome/framework_helpers.py b/esphome/framework_helpers.py index 4da53adada..1ceafdf9c2 100644 --- a/esphome/framework_helpers.py +++ b/esphome/framework_helpers.py @@ -1171,23 +1171,6 @@ def download_from_mirrors( raise ValueError("download_from_mirrors called with an empty mirrors list") -def tools_cache_path(env_var: str, subdir: str) -> Path: - """A backend's machine-global tools directory, with an env override. - - A blank/whitespace override is treated as unset: ``Path("")`` resolves - to the CWD, which ``clean-all`` would then delete. - """ - import platformdirs - - from esphome.helpers import get_str_env - - if prefix := get_str_env(env_var, "").strip(): - return Path(prefix).expanduser().resolve() - return ( - Path(platformdirs.user_cache_dir("esphome", appauthor=False)) / subdir - ).resolve() - - def strip_win_long_path_prefix(path: str) -> str: r"""Strip the Windows extended-length path prefix from ``path``. @@ -1220,85 +1203,3 @@ def strip_win_long_path_prefix(path: str) -> str: if path.startswith("\\\\?\\"): return path[len("\\\\?\\") :] return path - - -def _ccache_runs(ccache: str) -> bool: - """Return True when the ``ccache`` found on PATH actually runs. - - ``shutil.which`` proves existence, not runnability: on Windows it also - matches ``.bat``/``.cmd`` wrappers and stale package-manager shims whose - target is gone. Wrapping compiles around such a find fails every compile - step with an opaque OS error, so probe once and fall back to compiling - without ccache when the probe fails. - """ - try: - subprocess.run( - [ccache, "--version"], - check=True, - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, - timeout=15, - close_fds=False, - ) - except (OSError, subprocess.SubprocessError): - _LOGGER.warning( - "Ignoring ccache at %s because it failed to run; compiling without ccache", - ccache, - ) - return False - return True - - -def resolve_ccache_path() -> str | None: - """The ccache binary to wrap compiles with, or None when disabled. - - Shared policy for every backend: on by default when a runnable ccache is - on PATH, ``ESPHOME_CCACHE_ENABLE=0`` opts out, and an explicit ``=1`` - warns when no binary is found and skips the runnability probe. The - Windows extended-length prefix is stripped before probing so the probe - validates the exact string the build will execute (#18399). - """ - import shutil - - from esphome.helpers import get_bool_env - - explicit = "ESPHOME_CCACHE_ENABLE" in os.environ - if explicit and not get_bool_env("ESPHOME_CCACHE_ENABLE"): - return None - ccache = shutil.which("ccache") - if ccache is None: - if explicit: - _LOGGER.warning( - "ESPHOME_CCACHE_ENABLE is set but no ccache binary is on PATH; " - "compiling without ccache" - ) - return None - ccache = strip_win_long_path_prefix(ccache) - if not explicit and not _ccache_runs(ccache): - return None - return ccache - - -def ccache_defaults_env(cache_dir: Path) -> dict[str, str]: - """Default ``CCACHE_*`` values for a build subprocess (not os.environ). - - Values the user already set in the environment are respected. Depend - mode is on: both native backends emit depfiles (-MMD / CMake), which - keeps cache-miss overhead low. - """ - from esphome.core import CORE - - # build_path is set during preload for every config-loading command; unset - # means the caller built the environment too early. Fail loudly rather - # than silently drop CCACHE_BASEDIR (losing cross-device cache hits). - if CORE.build_path is None: - raise ValueError( - "CORE.build_path must be set before constructing the build environment" - ) - defaults = { - "CCACHE_DIR": str(cache_dir), - "CCACHE_NOHASHDIR": "true", - "CCACHE_DEPEND": "1", - "CCACHE_BASEDIR": str(Path(CORE.build_path).resolve()), - } - return {k: v for k, v in defaults.items() if k not in os.environ} diff --git a/esphome/platformio/extra_script.py b/esphome/platformio/extra_script.py index 726c3e04bb..ef75421671 100644 --- a/esphome/platformio/extra_script.py +++ b/esphome/platformio/extra_script.py @@ -35,7 +35,7 @@ import os from pathlib import Path from typing import TYPE_CHECKING -from esphome.platformio.library import ensure_list +from esphome.core import EsphomeError if TYPE_CHECKING: from esphome.platformio.library import ConvertedLibrary @@ -89,9 +89,17 @@ def apply_extra_script( extra_flags = captured_as_build_flags(result, library_dir=source_path) if not extra_flags: return - flags = ensure_list(component.data.setdefault("build", {}).setdefault("flags", [])) - flags.extend(extra_flags) - component.data["build"]["flags"] = flags + flags = component.data.setdefault("build", {}).setdefault("flags", []) + if isinstance(flags, str): + flags = [flags] + elif not isinstance(flags, list): + # A null/dict value coerced through a list wrapper would inject a + # non-string into the compiler command line; fail naming the library + raise EsphomeError( + f"Library {component.name} has a malformed build.flags " + f"({type(flags).__name__}); expected a string or list" + ) + component.data["build"]["flags"] = [*flags, *extra_flags] # Keys we know how to translate back into ESPHome's build-flag pipeline. diff --git a/esphome/platformio/library.py b/esphome/platformio/library.py index 147d5293f0..ea9a45f388 100644 --- a/esphome/platformio/library.py +++ b/esphome/platformio/library.py @@ -459,7 +459,7 @@ def check_library_data(data: dict, platform: str | None, framework: str): ) -def _parse_library_json(library_json_path: PathType): +def parse_library_json(library_json_path: PathType): """ Load and parse a JSON file describing a library. @@ -893,7 +893,7 @@ def convert_libraries( has_json = library_json_path.is_file() has_properties = library_properties_path.is_file() if has_json: - component.data = _parse_library_json(library_json_path) + component.data = parse_library_json(library_json_path) elif has_properties: component.data = parse_library_properties(library_properties_path) else: diff --git a/esphome/platformio/toolchain.py b/esphome/platformio/toolchain.py index 3eeaecf7e4..cf2094dfe0 100644 --- a/esphome/platformio/toolchain.py +++ b/esphome/platformio/toolchain.py @@ -9,9 +9,10 @@ from typing import TYPE_CHECKING, Any import platformdirs +from esphome.build_helpers.ccache import resolve_ccache_path from esphome.const import CONF_COMPILE_PROCESS_LIMIT, CONF_ESPHOME, KEY_CORE from esphome.core import CORE, EsphomeError -from esphome.framework_helpers import resolve_ccache_path, strip_win_long_path_prefix +from esphome.framework_helpers import strip_win_long_path_prefix from esphome.helpers import ( add_git_ceiling_directory, copy_file_if_changed, diff --git a/tests/unit_tests/build_gen/test_arduino8266.py b/tests/unit_tests/build_gen/test_arduino8266.py index a3e99a8159..f3cba98248 100644 --- a/tests/unit_tests/build_gen/test_arduino8266.py +++ b/tests/unit_tests/build_gen/test_arduino8266.py @@ -442,7 +442,6 @@ def test_generate_ld_scripts_testing_mode(tmp_path: Path) -> None: (paths.framework / "tools" / "sdk" / "ld" / "eagle.flash.4m.ld").write_text( "MEMORY\n{\n" " dram0_0_seg : org = 0x3FFE8000, len = 0x14000\n" - " iram1_0_seg : org = 0x40100000, len = 0x8000\n" " irom0_0_seg : org = 0x40201010, len = 0xfeff0\n" "}\n" ) @@ -677,3 +676,67 @@ def test_ninja_path_escaping() -> None: """Build-statement paths and command-line paths escape differently.""" assert arduino8266._e("a b:$c") == "a$ b$:$$c" assert arduino8266._q("/a b/$x") == '"/a b/$$x"' + + +def test_write_project_asm_excludes_non_define_user_flags(tmp_path: Path) -> None: + """The ASPPCOM command under PlatformIO never sees CCFLAGS, so only -D/-I user flags + reach assembly compiles.""" + paths = _make_framework(tmp_path) + _set_flags("-DUSER_KNOB=1", "-Wno-volatile") + content = _write_ninja(paths) + asflags = next(line for line in content.splitlines() if line.startswith("asflags")) + assert "-DUSER_KNOB=1" in asflags + assert "-Wno-volatile" not in asflags + cxxflags = next( + line for line in content.splitlines() if line.startswith("cxxflags") + ) + assert "-Wno-volatile" in cxxflags + + +def test_write_project_returns_changed(tmp_path: Path) -> None: + """The documented contract: True when build.ninja changed, False on an + identical regeneration (pins byte-stable output too).""" + paths = _make_framework(tmp_path) + _set_flags() + src = CORE.relative_src_path() + (src / "esphome" / "components" / "esp8266").mkdir(parents=True, exist_ok=True) + (src / "main.cpp").write_text("") + with ( + patch.object(arduino8266, "generate_ld_scripts"), + patch("esphome.arduino.library.resolve_libraries", return_value=[]), + patch("esphome.arduino8266.framework.ccache_path", return_value=None), + ): + assert arduino8266.write_project(paths) is True + assert arduino8266.write_project(paths) is False + + +def test_write_project_missing_src_dir_raises(tmp_path: Path) -> None: + """A missing generated source tree is its own error, not an install one.""" + paths = _make_framework(tmp_path) + _set_flags() + with ( + patch.object(arduino8266, "generate_ld_scripts"), + patch("esphome.arduino.library.resolve_libraries", return_value=[]), + patch.object( + arduino8266.CORE, "relative_src_path", return_value=tmp_path / "nope" + ), + pytest.raises(EsphomeError, match="source directory"), + ): + arduino8266.write_project(paths) + + +def test_build_config_custom_mmu_without_knob_warns( + caplog: pytest.LogCaptureFixture, +) -> None: + """Custom MMU sizes without the CUSTOM knob keep the default layout and + warn, as the PlatformIO builder does.""" + _set_flags("-DMMU_IRAM_SIZE=0xC000") + config = _resolve_build_config(_flag_defines()) + assert config.mmu_defines == ["MMU_IRAM_SIZE=0x8000", "MMU_ICACHE_SIZE=0x8000"] + assert "Detected custom MMU flags" in caplog.text + + +def test_flag_defines_lexes_quoted_single_tokens() -> None: + """A quoted single-token define reads the same as on the compile line.""" + _set_flags('-DMMU_SEC_HEAP="0x40108000"') + assert _flag_defines()["MMU_SEC_HEAP"] == "MMU_SEC_HEAP=0x40108000" diff --git a/tests/unit_tests/build_helpers/test_idedata.py b/tests/unit_tests/build_helpers/test_idedata.py index 6c44192439..51f5b2384c 100644 --- a/tests/unit_tests/build_helpers/test_idedata.py +++ b/tests/unit_tests/build_helpers/test_idedata.py @@ -451,3 +451,17 @@ def test_load_or_build_idedata_never_caches_bad_compiler(tmp_path: Path) -> None ) assert data["cxx_path"] == "/usr/bin/python3" assert not cache.exists() + + +def test_load_or_build_idedata_cache_hit_skips_rebuild(tmp_path: Path) -> None: + """A valid cache newer than the compile DB is served without re-parsing.""" + compile_commands = _write_compile_commands(tmp_path) + cache = tmp_path / "c.json" + cache.write_text(json.dumps({"cc_path": "/tools/gcc", "cached": True})) + os.utime(cache, (compile_commands.stat().st_mtime + 5,) * 2) + with patch.object(idedata, "idedata_from_build") as mock_build: + data = idedata.load_or_build_idedata( + compile_commands, tmp_path / "f.elf", cache + ) + mock_build.assert_not_called() + assert data["cached"] is True diff --git a/tests/unit_tests/build_helpers/test_ninja.py b/tests/unit_tests/build_helpers/test_ninja.py index cf51d1b8d8..1dc49396c6 100644 --- a/tests/unit_tests/build_helpers/test_ninja.py +++ b/tests/unit_tests/build_helpers/test_ninja.py @@ -48,3 +48,31 @@ def test_find_ninja_missing_everywhere(tmp_path: Path) -> None: pytest.raises(EsphomeError, match="ninja not found"), ): ninja_helper.find_ninja() + + +def test_escape_ninja_specials() -> None: + assert ninja_helper.escape("a b:c$d") == "a$ b$:c$$d" + + +def test_quote_arg_windows_argv_rule() -> None: + # Backslash runs double only before a quote (subprocess.list2cmdline rule) + assert ninja_helper.quote_arg('-DX=a\\"b c') == '"-DX=a\\\\\\"b c"' + assert ninja_helper.quote_arg("a b\\") == '"a b\\\\"' + + +def test_shell_token_quotes_only_when_needed() -> None: + assert ninja_helper.shell_token("-Os") == "-Os" + assert ninja_helper.shell_token("-DP=C:\\x y") == '"-DP=C:\\x y"' + assert ninja_helper.shell_token("plain", force=True) == '"plain"' + + +def test_shell_token_quotes_shell_metacharacters() -> None: + """Tokens like -DMASK=(1<<3) must not reach /bin/sh -c bare.""" + assert ninja_helper.shell_token("-DMASK=(1<<3)") == '"-DMASK=(1<<3)"' + assert ninja_helper.shell_token("-DX=a;b") == '"-DX=a;b"' + assert ninja_helper.shell_token("-DX=$HOME") == '"-DX=$$HOME"' + + +def test_quote_path_force_quotes() -> None: + assert ninja_helper.quote_path(Path("a b")) == '"a b"' + assert ninja_helper.quote_path("simple") == '"simple"' diff --git a/tests/unit_tests/components/esp8266/test_build_surgery.py b/tests/unit_tests/components/esp8266/test_build_surgery.py index e62a2270fb..944518b1fa 100644 --- a/tests/unit_tests/components/esp8266/test_build_surgery.py +++ b/tests/unit_tests/components/esp8266/test_build_surgery.py @@ -49,7 +49,8 @@ def test_relocate_ratetable_inserts_after_data_start() -> None: patched = relocate_ratetable(_COMMON_LD_SNIPPET) assert RATETABLE_RULE in patched # Inserted after the .data section's anchor, not the .dport0.data one - assert patched.index("_data_start = ABSOLUTE(.);") < patched.index(RATETABLE_RULE) + # (whose closing brace bounds the decoy block) + assert RATETABLE_RULE not in patched[: patched.index("} >dport0_0_seg")] assert patched.index(RATETABLE_RULE) < patched.index("*(.data)") # Idempotent on an already-patched script assert relocate_ratetable(patched) == patched @@ -106,14 +107,20 @@ def test_board_build_covers_every_board() -> None: assert set(BOARDS) <= set(ESP8266_BOARD_BUILD) -def test_surgery_fingerprint_tracks_inputs() -> None: - """The fingerprint changes with any behavioral input, so linker-script - caches stamped with it self-invalidate on surgery edits.""" - from unittest.mock import patch +def test_surgery_fingerprint_covers_module_source() -> None: + """The fingerprint hashes the module source, so any surgery edit + invalidates linker-script caches stamped with it.""" + import hashlib + import inspect from esphome.components.esp8266 import build_surgery - base = build_surgery.surgery_fingerprint() - assert base == build_surgery.surgery_fingerprint() - with patch.object(build_surgery, "_TESTING_SEGMENT_SIZES", {"iram1_0_seg": "0x1"}): - assert build_surgery.surgery_fingerprint() != base + expected = hashlib.sha256(inspect.getsource(build_surgery).encode()).hexdigest() + assert build_surgery.surgery_fingerprint() == expected + + +def test_testing_memory_patches_present_but_unselected_raises() -> None: + """A known segment left off the caller's list must fail, not silently + keep its real memory limit.""" + with pytest.raises(RuntimeError, match="not selected"): + apply_testing_memory_patches(_FLASH_LD_SNIPPET, ("dram0_0_seg",)) diff --git a/tests/unit_tests/test_arduino8266_framework.py b/tests/unit_tests/test_arduino8266_framework.py index fac85964ce..e22b0b185c 100644 --- a/tests/unit_tests/test_arduino8266_framework.py +++ b/tests/unit_tests/test_arduino8266_framework.py @@ -103,7 +103,7 @@ def test_ccache_path_explicit_skips_probe(monkeypatch: pytest.MonkeyPatch) -> No monkeypatch.setenv("ESPHOME_CCACHE_ENABLE", "1") with ( patch("shutil.which", return_value="/usr/bin/ccache"), - patch("esphome.framework_helpers._ccache_runs", side_effect=AssertionError), + patch("esphome.build_helpers.ccache._ccache_runs", side_effect=AssertionError), ): assert framework.ccache_path() == "/usr/bin/ccache" diff --git a/tests/unit_tests/test_espidf_component.py b/tests/unit_tests/test_espidf_component.py index c4a910be23..b17136aa15 100644 --- a/tests/unit_tests/test_espidf_component.py +++ b/tests/unit_tests/test_espidf_component.py @@ -25,10 +25,10 @@ from esphome.platformio.library import ( GitSource, URLSource, _node_key, - _parse_library_json, _resolve_registry_version, collect_filtered_files, normalize_dependencies, + parse_library_json, parse_library_properties, split_list_by_condition, ) @@ -368,11 +368,11 @@ def test_generate_idf_component_yml_missing_path_raises(tmp_component): generate_idf_component_yml(tmp_component) -def test_parse_library_json(tmp_path): +def testparse_library_json(tmp_path): f = tmp_path / "library.json" f.write_text(json.dumps({"name": "test"})) - result = _parse_library_json(f) + result = parse_library_json(f) assert result["name"] == "test"