diff --git a/esphome/arduino/__init__.py b/esphome/arduino/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/esphome/arduino/library.py b/esphome/arduino/library.py new file mode 100644 index 0000000000..e224e62589 --- /dev/null +++ b/esphome/arduino/library.py @@ -0,0 +1,531 @@ +"""Arduino-core backend for the shared PlatformIO library converter. + +Bundled names build straight from the framework tree; everything else goes +through ``esphome.platformio.library``. Mirrors ``lib_ldf_mode=off``: each +library builds its own archive; all include dirs join one global path. + +Deviations from PlatformIO: flat-layout libraries get the recursive default +source filter; ``dot_a_linkage`` is honored; bundled libraries never run a +manifest ``extraScript``; manifest ``-I`` flags join the global include path; +``precompiled``/``ldflags`` properties are refused by name. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +import logging +from pathlib import Path +import re + +from esphome.core import CORE, EsphomeError, Library +from esphome.helpers import walk_files +from esphome.platformio.extra_script import apply_extra_script +from esphome.platformio.library import ( + DEFAULT_BUILD_INCLUDE_DIR, + DEFAULT_BUILD_SRC_FILTER, + ESPHOME_DATA_KEY, + ESPHOME_DATA_LINK_FLAGS_KEY, + LIBRARY_HEADER_SUFFIXES, + SRC_FILE_EXTENSIONS, + ConvertedLibrary, + IncompatiblePlatform, + InvalidLibrary, + LibraryBackend, + _url_or_none, + check_library_data, + collect_filtered_files, + convert_libraries, + ensure_list, + is_lib_ignored, + lex_build_flags, + lib_ignore_set, + normalize_dependencies, + parse_library_json, + parse_library_properties, + warn_properties_depends, +) + +_LOGGER = logging.getLogger(__name__) + + +@dataclass +class ArduinoLibrary: + """One resolved library, ready for the ninja generator.""" + + name: str + sources: list[Path] = field(default_factory=list) + include_dirs: list[Path] = field(default_factory=list) + # Extra compile flags private to this library's own sources + flags: list[str] = field(default_factory=list) + # PlatformIO's build.libArchive / Arduino's dot_a_linkage: when False the + # objects go to the linker directly (symbols nothing references survive) + lib_archive: bool = True + # Link inputs the library contributes (-L dirs / -l libs, e.g. from + # precompiled vendor blobs) and -Wl, options for the firmware link + link_dirs: list[Path] = field(default_factory=list) + link_libs: list[str] = field(default_factory=list) + link_flags: list[str] = field(default_factory=list) + + +# Source-like suffixes the case-sensitive suffix map rejects +_UNMAPPED_SOURCE_SUFFIXES = frozenset( + {s.lower() for s in SRC_FILE_EXTENSIONS} | {".ino"} +) + +# Filename-plain names: an allowlist excludes separators, drive colons, +# and dot-only names by shape +_SAFE_LIBRARY_NAME_RE = re.compile(r"[A-Za-z0-9_][A-Za-z0-9_. +-]*\Z") + + +def _is_safe_library_name(name: object) -> bool: + """Whether a name may be joined under the framework's libraries dir.""" + return isinstance(name, str) and _SAFE_LIBRARY_NAME_RE.fullmatch(name) is not None + + +def _manifest_build(name: str, data: object) -> dict: + """The manifest's ``build`` section; malformed manifests fail by name.""" + build = data.get("build", {}) if isinstance(data, dict) else None + if not isinstance(build, dict): + raise EsphomeError(f"Library {name} has a malformed manifest") + return build + + +def _resolve_src_dir(name: str, read_path: Path, build: dict) -> str: + """Resolve PIO's source dir: manifest srcDir, else src/Src, else the root.""" + if "srcDir" not in build: + return next((d for d in ("src", "Src") if (read_path / d).is_dir()), ".") + # A declared srcDir (falsy included) that does not resolve is a manifest error + src_dir = build["srcDir"] + if not (isinstance(src_dir, str) and src_dir and (read_path / src_dir).is_dir()): + raise EsphomeError( + f"Library {name} declares srcDir {src_dir!r} which does not exist" + ) + return src_dir + + +def _reject_unsupported_link_fields(name: str, data: dict) -> None: + # PIO honors these; ignoring them would fail at link with no stated + # cause. Property values are strings, so "false" is not a declaration. + precompiled = data.get("precompiled") + if precompiled and str(precompiled).strip().lower() != "false": + raise EsphomeError( + f"Library {name} declares precompiled, which this backend does not support" + ) + if data.get("ldflags"): + raise EsphomeError( + f"Library {name} declares ldflags, which this backend does not support" + ) + + +def _resolve_lib_archive(name: str, data: dict, build: dict) -> bool: + """build.libArchive, else dot_a_linkage (an Arduino IDE property PIO + ignores; a deliberate extra), else archive.""" + + # Strict parse: bool("false") is True + def _parse(key: str, raw: object) -> bool: + if isinstance(raw, bool): + return raw + value = str(raw).strip().lower() + if value in ("true", "false"): + return value == "true" + raise EsphomeError(f"Library {name} has a malformed {key} value {raw!r}") + + if "libArchive" in build: + return _parse("libArchive", build["libArchive"]) + if "dot_a_linkage" in data: + return _parse("dot_a_linkage", data["dot_a_linkage"]) + return True + + +def _classify_build_flags( + name: str, read_path: Path, lib: ArduinoLibrary, flag_tokens: list[str] +) -> list[str]: + """Route the lexed build.flags into the library's flag lists. + + Returns the ``-I`` arguments for the include-dir resolution. + """ + include_flags: list[str] = [] + for tok in flag_tokens: + if tok.startswith("-I"): + include_flags.append(tok[2:]) + elif tok.startswith("-L"): + link_dir = (read_path / tok[2:]).resolve() + if not link_dir.is_dir(): + # Kept (the linker ignores missing -L dirs); the warning + # names the culprit before a bare "cannot find -lfoo" + _LOGGER.warning( + "Library %s declares library dir %s which does not exist", + name, + tok[2:], + ) + lib.link_dirs.append(link_dir) + elif tok.startswith("-l"): + lib.link_libs.append(tok[2:]) + elif tok.startswith("-Wl,"): + lib.link_flags.append(tok) + else: + lib.flags.append(tok) + return include_flags + + +def _resolve_include_dirs( + name: str, + read_path: Path, + lib: ArduinoLibrary, + build: dict, + src_dir: str, + include_flags: list[str], +) -> None: + include_dir = build.get("includeDir", DEFAULT_BUILD_INCLUDE_DIR) + if not isinstance(include_dir, str): + raise EsphomeError(f"Library {name} has a malformed includeDir") + for d, explicit in [ + (include_dir, "includeDir" in build), + (src_dir, False), # _resolve_src_dir already validated it + *((flag, True) for flag in include_flags), + ]: + if (path := (read_path / d)).is_dir(): + lib.include_dirs.append(path.resolve()) + elif explicit: + # Warn-and-drop (unlike srcDir): a missing include dir is + # harmless until a header is needed, and the compile names it + _LOGGER.warning( + "Library %s declares include dir %s which does not exist", name, d + ) + + +def _collect_lib_sources( + name: str, + read_path: Path, + lib: ArduinoLibrary, + src_dir: str, + src_filter: list[str], +) -> None: + sources: list[Path] = [] + dropped: list[str] = [] + saw_header = False + for f in collect_filtered_files(read_path / src_dir, src_filter): + path = Path(f) + suffix = path.suffix + if suffix in SRC_FILE_EXTENSIONS: + # resolve() per file: srcFilter patterns may escape src_dir + sources.append(path.resolve()) + elif suffix.lower() in _UNMAPPED_SOURCE_SUFFIXES: + # A source-like suffix the case-sensitive map rejects (.CPP, + # .ino) is a dropped compilation unit; headers fall through + dropped.append(path.name) + elif suffix.lower() in LIBRARY_HEADER_SUFFIXES: + saw_header = True + lib.sources = sorted(sources) + if dropped: + _LOGGER.warning( + "Library %s: %d file(s) with unmapped source suffixes are not compiled: %s", + name, + len(dropped), + ", ".join(sorted(dropped)), + ) + if not lib.sources and not saw_header: + # Matched headers mean header-only; a filter matching nothing is + # a manifest/tree problem (a truly empty tree raises elsewhere) + _LOGGER.warning("Library %s: no source files matched", name) + + +def _library_info(name: str, read_path: Path, data: dict) -> ArduinoLibrary: + """Resolve one library's sources, include dirs, and flags (PIO semantics).""" + build = _manifest_build(name, data) + _reject_unsupported_link_fields(name, data) + src_dir = _resolve_src_dir(name, read_path, build) + src_filter = ensure_list(build.get("srcFilter", DEFAULT_BUILD_SRC_FILTER)) + if not all(isinstance(entry, str) for entry in src_filter): + raise EsphomeError(f"Library {name} has a malformed srcFilter") + lib = ArduinoLibrary(name=name, lib_archive=_resolve_lib_archive(name, data, build)) + # PlatformIO shell-lexes each build.flags entry + include_flags = _classify_build_flags( + name, read_path, lib, lex_build_flags(build.get("flags", []), f"library {name}") + ) + _resolve_include_dirs(name, read_path, lib, build, src_dir, include_flags) + _collect_lib_sources(name, read_path, lib, src_dir, src_filter) + return lib + + +def _bundled_library(framework_path: Path, name: str) -> ArduinoLibrary: + """A library bundled with the Arduino core, read from the framework tree. + + ``library.json`` wins over ``library.properties`` when both exist, as in + PlatformIO's LibBuilderFactory; only the JSON manifest can carry a + ``build`` section (srcDir, srcFilter, flags). + """ + lib_dir = framework_path / "libraries" / name + manifest_json = lib_dir / "library.json" + if manifest_json.is_file(): + try: + data = parse_library_json(manifest_json) + except ValueError as err: # JSONDecodeError + raise EsphomeError( + f"Bundled library {name} has a corrupt library.json ({err}); " + "the framework install may be incomplete (run 'esphome clean-all')" + ) from err + elif (manifest := lib_dir / "library.properties").is_file(): + data = parse_library_properties(manifest) + else: + # Debug, not warning: the legacy manifest-less layout is legal and + # the 3.1.2 core ships one such library (FSTools), so a warning + # would be unactionable noise on every build using it + _LOGGER.debug("Bundled library %s has no manifest; using defaults", name) + data = {} + if isinstance(data, dict): + # Bundled manifest deps are never walked; make the skip visible + if data.get("dependencies"): + _LOGGER.warning( + "Bundled library %s declares dependencies, which are not " + "resolved automatically; add them with add_library() if needed", + name, + ) + warn_properties_depends(name, data) + build = data.get("build") + if isinstance(build, dict) and build.get("extraScript"): + # Scripts only run on the converted path; building without + # the script's flags would miscompile + raise EsphomeError( + f"Bundled library {name} declares an extraScript, which is " + "not run for bundled libraries" + ) + lib = _library_info(name, lib_dir, data) + _assert_tree_has_code( + name, + lib_dir, + "the framework install may be incomplete (run 'esphome clean-all')", + ) + return lib + + +def _assert_tree_has_code(name: str, root: Path, hint: str) -> None: + """An empty or half-extracted tree can never link; fail by name (a + warning would scroll away and resurface as undefined symbols).""" + if not any( + Path(p).suffix in SRC_FILE_EXTENSIONS + or Path(p).suffix.lower() in LIBRARY_HEADER_SUFFIXES + for p in walk_files(root) + ): + raise EsphomeError(f"Library {name} has no sources or headers; {hint}") + + +def _external_short_name(name: str) -> str: + """The short library name of a requested spec. + + "owner/Name" and plain names take the last path segment; "Name=" + takes the declared name. Git tails (".git", "#ref") are stripped like + the walk's URL normalization; the comparand is a manifest dependency + name, never a spec. + """ + head, sep, tail = name.partition("=") + if sep and "://" in tail: + return head + short = name.rsplit("/", maxsplit=1)[-1] + return short.partition("#")[0].removesuffix(".git") + + +def _check_unfulfilled_provides( + provided_requests: set[str], satisfied: set[str], still_requested: set[str] +) -> None: + """Fail by name when a walk-skipped dependency was never added. + + An unfulfilled provides() promise only surfaces as undefined symbols + at link. The walk records across re-resolutions, so a name no final + manifest still requests is stale state, never a failure. + """ + if missing := sorted((provided_requests & still_requested) - satisfied): + raise EsphomeError( + "provides() skipped these dependencies but nothing added them: " + f"{', '.join(missing)}; the build is missing libraries" + ) + + +def resolve_libraries( + framework_path: Path, *, pio_platform: str, board_mcu: str, cache_key: str +) -> list[ArduinoLibrary]: + """Resolve every ``cg.add_library()`` entry into an :class:`ArduinoLibrary`. + + ``pio_platform``/``board_mcu`` filter manifests the way PlatformIO would + for that core (e.g. ``espressif8266``/``esp8266``); ``cache_key`` keys the + shared converter's download cache. + + The returned list is not topologically sorted, so the caller must link + the archives inside one ``--start-group``/``--end-group`` pair (the + bundled-first grouping is incidental). + """ + bundled: list[ArduinoLibrary] = [] + external: list[Library] = [] + # PlatformIO's lib_ignore covers framework-bundled libraries too; the + # shared converter only filters the registry/git ones. + lib_ignore = lib_ignore_set() + # Exact directory names keep membership case-sensitive everywhere + # (an is_dir() probe would match "wire" on macOS/Windows and build + # the bundled Wire twice) + libraries_dir = framework_path / "libraries" + if not libraries_dir.is_dir(): + # A registry fallback would fail later with a misleading + # package-not-found error per bundled name + raise EsphomeError( + f"{libraries_dir} is missing; the framework install may be " + "incomplete (run 'esphome clean-all')" + ) + bundled_dir_names = frozenset(p.name for p in libraries_dir.iterdir() if p.is_dir()) + + def _provided(name: object) -> bool: + return _is_safe_library_name(name) and name in bundled_dir_names + + for library in CORE.platformio_libraries.values(): + if is_lib_ignored(library.name, lib_ignore): + continue + # Bundled only for a bare name with a matching framework dir; pinned + # or unmatched names resolve from the registry, as under PlatformIO. + if not library.repository and not library.version and _provided(library.name): + # Bundled manifest deps are not walked; _bundled_library warns + bundled.append(_bundled_library(framework_path, library.name)) + else: + external.append(library) + + converted: list[ArduinoLibrary] = [] + bundled_names = {lib.name for lib in bundled} + converted_manifest_names: set[str] = set() + # Bundled candidates skipped on purpose (platform filter); the + # provides() reconciliation must count them as satisfied + knowingly_skipped: set[str] = set() + # Dependency names of the manifests actually emitted; a walk recording + # for a since-re-resolved manifest must not fail the reconciliation + final_dep_names: set[str] = set() + # Ordered set of bundled dependency names to add once conversion is done + pending_bundled: dict[str, None] = {} + # Deps matching a separately-requested external are already in the build + # (a duplicate archive means duplicate-symbol link errors) + external_short_names = { + _external_short_name(lib.name) for lib in external if lib.name + } + + def _add_bundled_dependencies(component: ConvertedLibrary) -> None: + # A version-less bare name ("Hash") is a core-bundled library the + # shared converter cannot resolve from the registry + for dep in normalize_dependencies( + component.data.get("dependencies"), component.name + ): + # normalize_dependencies guarantees a non-empty str name + name = dep["name"] + final_dep_names.add(name) + if "/" in name: + owner, _, pkg = name.partition("/") + if _is_safe_library_name(owner) and _is_safe_library_name(pkg): + # Owner-qualified; the converter resolves it from the registry + continue + if not _is_safe_library_name(name): + # The name becomes a path component; never join a traversal + _LOGGER.warning( + "Ignoring malformed dependency entry %r of library %s", + dep, + component.name, + ) + continue + if name in external_short_names: + if _provided(name): + # A bundled copy is suppressed; a coincidental name + # collision would surface as link errors + _LOGGER.warning( + "Dependency %s of %s is assumed satisfied by a " + "requested external library; the bundled copy is " + "not added", + name, + component.name, + ) + else: + _LOGGER.debug( + "Dependency %s of %s assumed satisfied by a requested " + "external library", + name, + component.name, + ) + continue + if name in bundled_names or is_lib_ignored(name, lib_ignore): + continue + if _url_or_none(dep.get("version")) is not None: + # A URL names one specific source; never add the bundled copy + continue + if dep.get("owner") or not _provided(name): + # Only owner-less framework-tree names take the bundled + # copy (PIO's process_dependencies); the walk reports drops + continue + try: + # framework=None: the walk already warned for non-platform + # causes; debug keeps one fault from warning twice (pinned + # by test_nonplatform_rejection_warns_once_through_real_converter) + check_library_data(dep, pio_platform, None) + except IncompatiblePlatform as err: + # A knowing skip (platform filter), not a broken promise + knowingly_skipped.add(name) + _LOGGER.debug("Skip bundled candidate %s: %s", name, err) + continue + except InvalidLibrary as err: + # Malformed manifest data never counts as satisfied; the + # walk owns the warning (see the warns-once test above) + _LOGGER.debug("Skip malformed bundled candidate %s: %s", name, err) + continue + # Deferred: a later manifest name may satisfy this + pending_bundled.setdefault(name) + + def _emit(component: ConvertedLibrary) -> None: + apply_extra_script( + component, board_mcu=lambda: board_mcu, pio_platform=pio_platform + ) + _assert_tree_has_code( + component.get_require_name(), + component.source_dir, + "the download may be incomplete (run 'esphome clean-all')", + ) + if isinstance(manifest_name := component.data.get("name"), str): + converted_manifest_names.add(manifest_name) + lib = _library_info( + component.get_require_name(), component.source_dir, component.data + ) + # Extra-script LINKFLAGS travel outside build.flags; dropping + # them would link wrong with no stated cause + lib.link_flags.extend( + component.data.get(ESPHOME_DATA_KEY, {}).get( + ESPHOME_DATA_LINK_FLAGS_KEY, [] + ) + ) + converted.append(lib) + _add_bundled_dependencies(component) + + backend = LibraryBackend( + platform=pio_platform, + framework="arduino", + emit=_emit, + cache_key=cache_key, + # The walk must not resolve bundled names from the registry; + # _add_bundled_dependencies adds them after emit + provides=_provided, + ) + if external: + convert_libraries(external, backend) + for name in pending_bundled: + if name in converted_manifest_names: + # The converted library is this one; the bundled copy would + # double the archive. Warn like the external_short_names twin. + _LOGGER.warning( + "Dependency %s is assumed satisfied by a converted library's " + "manifest name; the bundled copy is not added", + name, + ) + continue + bundled_names.add(name) + bundled.append(_bundled_library(framework_path, name)) + + _check_unfulfilled_provides( + backend.provided_requests, + bundled_names + | converted_manifest_names + | external_short_names + | knowingly_skipped, + final_dep_names, + ) + + return bundled + converted diff --git a/esphome/build_gen/build_tool.py b/esphome/build_gen/build_tool.py new file mode 100644 index 0000000000..00aa1ec69d --- /dev/null +++ b/esphome/build_gen/build_tool.py @@ -0,0 +1,108 @@ +"""Tiny cross-platform build steps invoked from the generated ninja file. + +Plain script (not ``python -m``): it runs from ninja with whatever Python +started esphome and must not depend on the package being importable. + +Subcommands: + ar remove stale archive, then ``ar rcs`` + copy copy a file + +The ar rspfile carries one object path per line (the generating rule must +use ``$in_newline``, never ``$in``). +""" + +from pathlib import Path +import shutil +import subprocess +import sys + + +def _read_rspfile(rspfile: str) -> list[str]: + r"""The object paths listed in ``rspfile``, unquoted. + + GNU ar treats backslashes in response files as escapes (corrupts + Windows paths), so the caller expands the list into argv; strip the + simple surrounding quote ninja adds to special paths, then undo + ninja's POSIX escape for an embedded quote ('a'\\''b.o' -> a'b.o). + """ + return [ + line[1:-1].replace("'\\''", "'") + if len(line) >= 2 and line[0] == line[-1] and line[0] in "'\"" + else line + for line in Path(rspfile).read_text(encoding="utf-8").splitlines() + if line + ] + + +def _run_ar(ar: str, archive: str, rspfile: str) -> int: + # Remove first: ``ar rcs`` replaces members but never drops ones whose + # source was removed from the build, which would leak stale objects. + Path(archive).unlink(missing_ok=True) + objects = _read_rspfile(rspfile) + if not objects: + # An empty archive would "succeed" here and fail far away at link + print(f"ar: no objects listed in {rspfile} for {archive}", file=sys.stderr) + return 1 + # Batch by argv length: expanding the rspfile gives back the Windows + # 32767-char command-line limit it existed to avoid. "rcs" creates, + # "qs" appends; the s keeps the symbol index explicit on every ar. + op = "rcs" + ok = False + try: + while objects: + batch = [objects.pop(0)] + batch_len = len(batch[0]) + while objects and batch_len + len(objects[0]) < 25000: + batch_len += len(objects[0]) + 1 + batch.append(objects.pop(0)) + rc = subprocess.run( + [ar, op, archive, *batch], check=False, close_fds=False + ).returncode + if rc != 0: + return rc + op = "qs" + ok = True + return 0 + finally: + if not ok: + # Any failure (bad exit, missing ar binary, interrupt) must not + # leave a truncated archive behind + Path(archive).unlink(missing_ok=True) + + +def _run_copy(src: str, dst: str) -> int: + try: + shutil.copyfile(src, dst) + except OSError as err: + # Never leave a partially written output (e.g. a firmware image); + # SameFileError means dst IS src, where unlinking destroys the input + if not isinstance(err, shutil.SameFileError): + Path(dst).unlink(missing_ok=True) + print(f"copy: {src} -> {dst} failed: {err}", file=sys.stderr) + return 1 + return 0 + + +# mode -> (handler, expected operand count); surplus argv means a +# mis-specified ninja rule and must error, not silently drop operands +_MODES = {"ar": (_run_ar, 3), "copy": (_run_copy, 2)} + + +def main() -> int: + mode = sys.argv[1] if len(sys.argv) > 1 else "" + if entry := _MODES.get(mode): + handler, argc = entry + args = sys.argv[2:] + if len(args) != argc: + print( + f"build_tool {mode}: expected {argc} arguments, got {len(args)}", + file=sys.stderr, + ) + return 1 + return handler(*args) + print(f"unknown build_tool mode: {mode}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": # pragma: no cover + sys.exit(main()) diff --git a/esphome/platformio/library.py b/esphome/platformio/library.py index 306f07854e..0402311a9a 100644 --- a/esphome/platformio/library.py +++ b/esphome/platformio/library.py @@ -74,6 +74,11 @@ SOURCE_KIND_FOR_SUFFIX: dict[str, str] = { ".ASM": "asm", } SRC_FILE_EXTENSIONS = list(SOURCE_KIND_FOR_SUFFIX) +# Suffixes that count as headers when probing whether a library has any +# usable files at all (compare against Path.suffix.lower()) +LIBRARY_HEADER_SUFFIXES = frozenset( + {".h", ".hpp", ".hh", ".hxx", ".inc", ".ipp", ".tcc"} +) DOMAIN = "pio_components" @@ -329,6 +334,11 @@ class LibraryBackend: framework: str emit: Callable[["ConvertedLibrary"], None] cache_key: str + # Owner-less names this returns True for are skipped by the walk; + # the backend supplies them itself (e.g. core-bundled libraries) and + # reconciles provided_requests after resolving + provides: Callable[[str], bool] | None = None + provided_requests: set[str] = field(default_factory=set) def ensure_list[T](obj: T | list[T]) -> list[T]: @@ -469,7 +479,7 @@ def _valid_manifest_shape(data: Any) -> bool: ) -def check_library_data(data: dict, platform: str | None, framework: str): +def check_library_data(data: dict, platform: str | None, framework: str | None): """ Check whether a library manifest is compatible with the target toolchain. @@ -486,7 +496,8 @@ def check_library_data(data: dict, platform: str | None, framework: str): for targets (e.g. Zephyr) where PIO manifests rarely declare the platform yet portable libraries still build. framework: The active framework name (e.g. ``espidf``, ``arduino``, - ``zephyr``) the manifest is expected to declare. + ``zephyr``) the manifest is expected to declare. ``None`` skips + the framework check (and its warning), mirroring ``platform``. Raises: InvalidLibrary: If the library does not support the target platform. @@ -517,7 +528,7 @@ def check_library_data(data: dict, platform: str | None, framework: str): # under the target framework, and there's no way to opt out of the check at # this layer. Warn instead of failing so the user isn't forced to fork the # library to fix the manifest. - valid_framework = "*" in frameworks or framework in frameworks + valid_framework = framework is None or "*" in frameworks or framework in frameworks if not valid_framework: _LOGGER.warning( @@ -914,6 +925,56 @@ def is_lib_ignored(name: str | None, lib_ignore: set[str]) -> bool: ) +def _reconcile_versionless_skips( + skipped_versionless: list[tuple[Any, Any, str]], + components: dict[str, ConvertedLibrary], + backend: LibraryBackend, +) -> None: + """Warn for version-less deps nothing satisfied, and record the + backend-provided ones in ``backend.provided_requests`` for its + post-emit reconciliation; a silent drop surfaces as link errors far + from the cause.""" + resolved_manifest_names = {c.data.get("name") for c in components.values()} + # A treeless backend can never supply a bundled name; noise for it + log = _LOGGER.warning if backend.provides is not None else _LOGGER.debug + warned: set[str] = set() + for dep_name, dep_owner, requester in skipped_versionless: + if not isinstance(dep_name, str) or not dep_name or dep_name in warned: + continue + if dep_name in components: + # A version-less dep's request key is the name itself + continue + if ( + not dep_owner + and backend.provides is not None + and backend.provides(dep_name) + ): + # provides() only satisfies owner-less names (same guard as + # the walk's skip); record for the post-emit reconciliation. + # Checked before the manifest-name evidence so the overlap + # case warns once, in the backend's own suppression loop + backend.provided_requests.add(dep_name) + continue + if dep_name in resolved_manifest_names: + # Name-only evidence: a coincidental collision must stay + # visible where the user could pin it + warned.add(dep_name) + log( + "Version-less dependency %s of %s assumed satisfied by a " + "resolved library's manifest name only", + dep_name, + requester, + ) + continue + warned.add(dep_name) + log( + "Dependency %s of %s has no version to resolve and nothing " + "provides it; skipping", + dep_name, + requester, + ) + + def _fetch_source( component: ConvertedLibrary, salt: str, @@ -1083,6 +1144,8 @@ def convert_libraries( components: dict[str, ConvertedLibrary] = {} resolved_requirements: dict[str, frozenset[str]] = {} top_level_keys = set(top_level) + # (name, owner, requester) reconciled against the final resolution set + skipped_versionless: list[tuple[Any, Any, str]] = [] worklist = deque(dict.fromkeys(top_level)) while worklist: # Drain the frontier sequentially (spec resolution mutates shared @@ -1187,13 +1250,23 @@ def convert_libraries( component.data.get("dependencies"), component.name ): if "version" not in dependency: - # Cannot resolve from the registry; common for bundled - # names (Wire, SPI) -- unactionable noise above debug + # Cannot resolve from the registry; the post-emit + # reconciliation owns the drop warning + dep_name = dependency.get("name") _LOGGER.debug( "Skip version-less dependency %r of %s", - dependency.get("name"), + dep_name, component.name, ) + if not is_lib_ignored( + dep_name, lib_ignore + ) and dependency_is_usable( + dependency, backend.platform, backend.framework, component.name + ): + # Filtered or ignored deps are deliberately absent + skipped_versionless.append( + (dep_name, dependency.get("owner"), component.name) + ) continue if not dependency_is_usable( dependency, backend.platform, backend.framework, component.name @@ -1205,11 +1278,31 @@ def convert_libraries( if is_lib_ignored(dep_name, lib_ignore): _LOGGER.debug("Skip ignored dependency %s", dep_name) continue - # The version field may actually be a URL (git/archive dependency). + # The version may be a URL (git/archive), which names one + # specific source; never substitute a bundled library for it dep_version = dependency["version"] dep_url = _url_or_none(dep_version) if dep_url is not None: dep_version = None + elif ( + backend.provides is not None + and not dependency.get("owner") + and backend.provides(dep_name) + ): + # The backend adds it from its own tree; resolving here + # would fetch a same-named registry package + if dep_version and dep_version != "*": + # The pin is discarded; make the substitution visible + _LOGGER.warning( + "Dependency %s pins version %s; using the library " + "bundled with the framework instead", + dep_name, + dep_version, + ) + else: + _LOGGER.debug("Skip backend-provided dependency %s", dep_name) + backend.provided_requests.add(dep_name) + continue dep_key = add_spec(dep_name, dep_version, dep_url) node.edges.add(dep_key) worklist.append(dep_key) @@ -1263,4 +1356,6 @@ def convert_libraries( for component in components.values(): backend.emit(component) + _reconcile_versionless_skips(skipped_versionless, components, backend) + return [components[key] for key in top_level if key in components] diff --git a/tests/unit_tests/build_gen/test_build_tool.py b/tests/unit_tests/build_gen/test_build_tool.py new file mode 100644 index 0000000000..b029c647ab --- /dev/null +++ b/tests/unit_tests/build_gen/test_build_tool.py @@ -0,0 +1,246 @@ +"""Tests for the ninja build-tool helper script.""" + +from __future__ import annotations + +from pathlib import Path +import subprocess +import sys +from unittest.mock import MagicMock, patch + +import pytest + +from esphome.build_gen import build_tool + + +def test_ar_removes_stale_archive(tmp_path: Path) -> None: + archive = tmp_path / "lib.a" + archive.write_text("stale") + rsp = tmp_path / "lib.a.rsp" + rsp.write_text("a.o\n") + with ( + patch.object( + build_tool.sys, + "argv", + ["build_tool", "ar", "ar-bin", str(archive), str(rsp)], + ), + patch.object( + build_tool.subprocess, "run", return_value=MagicMock(returncode=0) + ) as mock_run, + ): + assert build_tool.main() == 0 + assert not archive.exists() + # The rspfile is expanded by the shim (GNU ar would escape backslashes) + assert mock_run.call_args[0][0] == ["ar-bin", "rcs", str(archive), "a.o"] + + +def test_copy(tmp_path: Path) -> None: + src = tmp_path / "firmware.bin" + src.write_text("data") + dst = tmp_path / "firmware.factory.bin" + with patch.object( + build_tool.sys, "argv", ["build_tool", "copy", str(src), str(dst)] + ): + assert build_tool.main() == 0 + assert dst.read_text() == "data" + + +def test_unknown_mode(capsys: pytest.CaptureFixture[str]) -> None: + with patch.object(build_tool.sys, "argv", ["build_tool", "bogus"]): + assert build_tool.main() == 1 + assert "unknown build_tool mode" in capsys.readouterr().err + + +def test_runs_as_script(tmp_path: Path) -> None: + """The ninja rules invoke the file as a plain script.""" + + src = tmp_path / "a.bin" + src.write_text("x") + dst = tmp_path / "b.bin" + result = subprocess.run( + [sys.executable, build_tool.__file__, "copy", str(src), str(dst)], + check=False, + ) + assert result.returncode == 0 + assert dst.read_text() == "x" + + +def test_ar_expands_rspfile_without_escaping(tmp_path) -> None: + """Backslash paths survive: the shim expands the rspfile itself instead + of letting GNU ar treat backslashes as escapes.""" + rsp = tmp_path / "objs.rsp" + rsp.write_text("obj/a.o\nsub\\b.o\n") + with ( + patch.object( + build_tool.sys, + "argv", + ["build_tool", "ar", "ar-bin", str(tmp_path / "lib.a"), str(rsp)], + ), + patch.object( + build_tool.subprocess, "run", return_value=MagicMock(returncode=0) + ) as mock_run, + ): + assert build_tool.main() == 0 + assert mock_run.call_args[0][0] == [ + "ar-bin", + "rcs", + str(tmp_path / "lib.a"), + "obj/a.o", + "sub\\b.o", + ] + + +def test_ar_unquotes_ninja_escaped_paths(tmp_path: Path) -> None: + """The shim strips a simple surrounding quote, since ninja shell- + quotes special rsp paths, so ar sees the real filename.""" + rsp = tmp_path / "t.rsp" + rsp.write_text("'obj/a b.o'\nobj/c.o\n") + with ( + patch.object( + build_tool.sys, "argv", ["bt", "ar", "/usr/bin/ar", "lib.a", str(rsp)] + ), + patch.object(build_tool.subprocess, "run") as mock_run, + ): + mock_run.return_value.returncode = 0 + rc = build_tool.main() + assert rc == 0 + assert mock_run.call_args.args[0] == [ + "/usr/bin/ar", + "rcs", + "lib.a", + "obj/a b.o", + "obj/c.o", + ] + + +def test_ar_empty_object_list_fails( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + """A lost object list is an error here, not undefined symbols at link.""" + rsp = tmp_path / "t.rsp" + rsp.write_text("\n\n") + with patch.object( + build_tool.sys, "argv", ["bt", "ar", "/usr/bin/ar", "lib.a", str(rsp)] + ): + rc = build_tool.main() + assert rc == 1 + assert "no objects listed" in capsys.readouterr().err + + +def test_ar_batches_long_object_lists(tmp_path: Path) -> None: + """The expanded argv must stay under the Windows 32767-char limit: a + long object list creates with rcs, then appends with qs.""" + archive = tmp_path / "lib.a" + rsp = tmp_path / "lib.a.rsp" + objects = [f"dir/{'x' * 120}_{i}.o" for i in range(400)] + rsp.write_text("\n".join(objects) + "\n") + with ( + patch.object( + build_tool.sys, + "argv", + ["build_tool", "ar", "ar-bin", str(archive), str(rsp)], + ), + patch.object( + build_tool.subprocess, "run", return_value=MagicMock(returncode=0) + ) as mock_run, + ): + assert build_tool.main() == 0 + calls = [c[0][0] for c in mock_run.call_args_list] + assert len(calls) > 1 + assert calls[0][1] == "rcs" + assert all(c[1] == "qs" for c in calls[1:]) + assert [o for c in calls for o in c[3:]] == objects + assert all(sum(len(a) + 1 for a in c) < 32000 for c in calls) + + +def test_ar_batch_failure_stops(tmp_path: Path) -> None: + """A failing batch propagates its exit code without running the rest.""" + archive = tmp_path / "lib.a" + rsp = tmp_path / "lib.a.rsp" + rsp.write_text("\n".join(f"{'y' * 200}_{i}.o" for i in range(300)) + "\n") + with ( + patch.object( + build_tool.sys, + "argv", + ["build_tool", "ar", "ar-bin", str(archive), str(rsp)], + ), + patch.object( + build_tool.subprocess, + "run", + side_effect=lambda cmd, **kw: ( + archive.write_text("partial"), + MagicMock(returncode=3), + )[1], + ) as mock_run, + ): + assert build_tool.main() == 3 + assert mock_run.call_count == 1 + # The failed batch must not leave a truncated archive behind + assert not archive.exists() + + +def test_ar_exception_leaves_no_partial_archive(tmp_path: Path) -> None: + """A missing ar binary mid-loop must not leave a truncated archive from + earlier successful batches.""" + archive = tmp_path / "lib.a" + rsp = tmp_path / "lib.a.rsp" + rsp.write_text("a.o\n") + with ( + patch.object( + build_tool.sys, + "argv", + ["build_tool", "ar", "ar-bin", str(archive), str(rsp)], + ), + patch.object( + build_tool.subprocess, + "run", + side_effect=lambda cmd, **kw: ( + archive.write_text("partial"), + (_ for _ in ()).throw(FileNotFoundError("no ar")), + ), + ), + pytest.raises(FileNotFoundError), + ): + build_tool.main() + assert not archive.exists() + + +def test_surplus_arguments_error(capsys: pytest.CaptureFixture[str]) -> None: + """A mis-specified ninja rule passing extra operands errors instead of + silently dropping them.""" + with patch.object( + build_tool.sys, "argv", ["build_tool", "copy", "a", "b", "extra"] + ): + assert build_tool.main() == 1 + assert "expected 2 arguments, got 3" in capsys.readouterr().err + + +def test_copy_same_file_keeps_the_input( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + """A same-file copy (dst IS src) must not unlink the input, and fails + with a message and exit code like the other shim paths.""" + src = tmp_path / "firmware.bin" + src.write_bytes(b"image") + with patch.object( + build_tool.sys, "argv", ["build_tool", "copy", str(src), str(src)] + ): + assert build_tool.main() == 1 + assert src.read_bytes() == b"image" + assert "failed" in capsys.readouterr().err + + +def test_copy_failure_leaves_no_partial_output(tmp_path: Path) -> None: + """A failed copy unlinks the destination; a partial firmware image must + never be left on disk.""" + dst = tmp_path / "firmware.factory.bin" + dst.write_text("stale") + with ( + patch.object(build_tool.shutil, "copyfile", side_effect=OSError("disk full")), + patch.object( + build_tool.sys, + "argv", + ["build_tool", "copy", str(tmp_path / "src.bin"), str(dst)], + ), + ): + assert build_tool.main() == 1 + assert not dst.exists() diff --git a/tests/unit_tests/test_arduino_library.py b/tests/unit_tests/test_arduino_library.py new file mode 100644 index 0000000000..87de28cf32 --- /dev/null +++ b/tests/unit_tests/test_arduino_library.py @@ -0,0 +1,1162 @@ +"""Tests for esphome.arduino.library (Arduino-core library resolution).""" + +from __future__ import annotations + +from contextlib import contextmanager +import json +import logging +from pathlib import Path +from unittest.mock import patch + +import pytest + +from esphome.arduino import library as component +from esphome.const import KEY_CORE, KEY_TARGET_PLATFORM, PLATFORM_ESP8266 +from esphome.core import CORE, EsphomeError, Library +import esphome.platformio.library as pio_library +from esphome.platformio.library import ( + ConvertedLibrary, + IncompatiblePlatform, + InvalidLibrary, + LibraryBackend, +) + + +@pytest.fixture(autouse=True) +def _reset_libraries() -> None: + # conftest's reset_core fixture clears platformio_libraries after each test + CORE.data[KEY_CORE] = {KEY_TARGET_PLATFORM: PLATFORM_ESP8266} + + +def _add_library(name: str, version: str | None, repository: str | None = None) -> None: + CORE.add_library(Library(name=name, version=version, repository=repository)) + + +def _make_framework(tmp_path: Path) -> Path: + framework = tmp_path / "framework" + lib = framework / "libraries" / "ESP8266WiFi" / "src" + lib.mkdir(parents=True) + (lib / "ESP8266WiFi.cpp").write_text("") + (lib / "ESP8266WiFi.h").write_text("") + (lib.parent / "library.properties").write_text("name=ESP8266WiFi\nversion=1.0\n") + root_lib = framework / "libraries" / "Wire" + root_lib.mkdir(parents=True) + (root_lib / "Wire.cpp").write_text("") + (root_lib / "examples").mkdir() + (root_lib / "examples" / "scan.ino").write_text("") + return framework + + +@contextmanager +def _emitting_converter(*converted): + """Patch convert_libraries to emit the given components via the backend.""" + + def fake_convert(libraries: list, backend: LibraryBackend) -> list: + assert backend.platform == "espressif8266" + assert backend.framework == "arduino" + assert backend.cache_key == "arduino8266" + for c in converted: + backend.emit(c) + return list(converted) + + with ( + patch.object(component, "convert_libraries", side_effect=fake_convert), + patch.object(component, "apply_extra_script") as mock_extra, + ): + yield mock_extra + + +def _converted(name: str, source_dir: Path, data: dict) -> ConvertedLibrary: + converted = ConvertedLibrary(name, "1.0.0", source=None) + converted.path = source_dir + converted.data = data + return converted + + +def _resolve(framework: Path) -> list[component.ArduinoLibrary]: + return component.resolve_libraries( + framework, + pio_platform="espressif8266", + board_mcu="esp8266", + cache_key="arduino8266", + ) + + +def _webserver(tmp_path: Path, data: dict) -> ConvertedLibrary: + """Register ESPAsyncWebServer and return its converted stand-in.""" + _add_library("ESP32Async/ESPAsyncWebServer", "3.9.6") + lib_dir = tmp_path / "converted" / "webserver" + (lib_dir / "src").mkdir(parents=True) + (lib_dir / "src" / "server.cpp").write_text("") + return _converted("esp32async__ESPAsyncWebServer", lib_dir, data) + + +def _local_lib(tmp_path: Path, dependencies: dict | list) -> None: + """Register a local file:// library declaring the given dependencies.""" + local_lib = tmp_path / "locallib" + (local_lib / "src").mkdir(parents=True) + (local_lib / "src" / "local.cpp").write_text("") + (local_lib / "library.json").write_text( + json.dumps( + {"name": "LocalLib", "version": "1.0.0", "dependencies": dependencies} + ) + ) + # as_uri() forms a valid file:// URL on every platform (file:///C:/... + # on Windows; a bare f-string would embed backslashes) + _add_library(local_lib.as_uri(), None) + + +def _ws_tcp_pair(tmp_path: Path) -> tuple[ConvertedLibrary, ConvertedLibrary]: + """Build ESPAsyncWebServer (depending on ESPAsyncTCP) plus resolved TCP.""" + ws_dir = tmp_path / "converted" / "webserver" + (ws_dir / "src").mkdir(parents=True) + (ws_dir / "src" / "server.cpp").write_text("") + tcp_dir = tmp_path / "converted" / "tcp" + (tcp_dir / "src").mkdir(parents=True) + (tcp_dir / "src" / "tcp.cpp").write_text("") + ws = _converted( + "esp32async__ESPAsyncWebServer", + ws_dir, + {"build": {}, "dependencies": [{"name": "ESPAsyncTCP"}]}, + ) + tcp = _converted("esp32async__ESPAsyncTCP", tcp_dir, {"build": {}}) + return ws, tcp + + +def test_library_info_src_layout(tmp_path: Path) -> None: + framework = _make_framework(tmp_path) + lib = component._bundled_library(framework, "ESP8266WiFi") + assert lib.name == "ESP8266WiFi" + assert [p.name for p in lib.sources] == ["ESP8266WiFi.cpp"] + assert lib.include_dirs == [(framework / "libraries/ESP8266WiFi/src").resolve()] + + +def test_library_info_root_layout_excludes_examples(tmp_path: Path) -> None: + framework = _make_framework(tmp_path) + lib = component._bundled_library(framework, "Wire") + assert [p.name for p in lib.sources] == ["Wire.cpp"] + assert lib.include_dirs == [(framework / "libraries/Wire").resolve()] + + +def test_library_info_flags_parsing(tmp_path: Path) -> None: + read_path = tmp_path / "lib" + (read_path / "src").mkdir(parents=True) + (read_path / "src" / "a.cpp").write_text("") + (read_path / "inc").mkdir() + (read_path / "blobs").mkdir() + data = { + "build": { + "flags": [ + "-DFOO=1 -I inc", + "-lalgobsec", + "-fno-lto", + "-Wl,--wrap=malloc", + # Bare flags join their argument within one entry only, as + # ParseFlags lexes each entry independently + "-l m", + "-L blobs", + ], + } + } + lib = component._library_info("x", read_path, data) + assert lib.flags == ["-DFOO=1", "-fno-lto"] + assert lib.include_dirs == [ + (read_path / "src").resolve(), + (read_path / "inc").resolve(), + ] + assert lib.link_dirs == [(read_path / "blobs").resolve()] + assert lib.link_libs == ["algobsec", "m"] + assert lib.link_flags == ["-Wl,--wrap=malloc"] + + +def test_library_info_missing_link_dir_warns( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + read_path = tmp_path / "lib" + read_path.mkdir() + data = {"build": {"flags": ["-Lmissing_blobs"]}} + lib = component._library_info("x", read_path, data) + assert "declares library dir missing_blobs which does not exist" in caplog.text + # Kept anyway: the linker ignores missing -L dirs + assert lib.link_dirs == [(read_path / "missing_blobs").resolve()] + + +def test_library_info_declared_filter_matches_nothing_warns( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + read_path = tmp_path / "lib" + (read_path / "src").mkdir(parents=True) + (read_path / "src" / "stub.cpp").write_text("") + data = {"build": {"srcFilter": ["+"]}} + lib = component._library_info("x", read_path, data) + assert not lib.sources + assert "no source files matched" in caplog.text + + +def test_empty_converted_tree_raises_at_emit(tmp_path: Path) -> None: + """A converted tree with no sources and no headers is a broken download; + fail by name like the bundled case.""" + framework = _make_framework(tmp_path) + _add_library("Some/Empty", "1.0.0") + lib_dir = tmp_path / "converted" / "empty" + (lib_dir / "src").mkdir(parents=True) + converted = _converted("some__Empty", lib_dir, {"build": {}}) + with ( + _emitting_converter(converted), + pytest.raises(EsphomeError, match="no sources or headers; the download"), + ): + _resolve(framework) + + +def test_library_info_no_src_dir(tmp_path: Path) -> None: + read_path = tmp_path / "empty" + read_path.mkdir() + lib = component._library_info("x", read_path, {}) + # With no manifest hints the source dir falls back to the library root + assert lib.sources == [] + assert lib.include_dirs == [read_path.resolve()] + + +def test_resolve_libraries_bundled(tmp_path: Path) -> None: + framework = _make_framework(tmp_path) + _add_library("ESP8266WiFi", None) + libs = _resolve(framework) + assert [lib.name for lib in libs] == ["ESP8266WiFi"] + + +@pytest.mark.parametrize("version", [None, "1.1.0"]) +def test_resolve_libraries_registry_name_is_external( + tmp_path: Path, version: str | None +) -> None: + """A name that is not bundled reaches the converter, bare or pinned.""" + framework = _make_framework(tmp_path) + _add_library("pngle", version) + with patch.object(component, "convert_libraries", return_value=[]) as mock_convert: + _resolve(framework) + (libraries, _backend), _ = mock_convert.call_args + assert [lib.name for lib in libraries] == ["pngle"] + + +def test_resolve_libraries_external_and_bundled_deps(tmp_path: Path) -> None: + framework = _make_framework(tmp_path) + _add_library("ESP32Async/ESPAsyncWebServer", "3.9.6") + + lib_dir = tmp_path / "converted" / "webserver" + (lib_dir / "src").mkdir(parents=True) + (lib_dir / "src" / "server.cpp").write_text("") + converted = _converted( + "esp32async__ESPAsyncWebServer", + lib_dir, + { + "build": {}, + "dependencies": [ + # Version-less bundled dependency: resolved from the framework + {"name": "Wire", "platforms": "espressif8266"}, + # Wrong platform: skipped + {"name": "ESP8266WiFi", "platforms": "espressif32"}, + # Registry dependency with a version: handled by the converter + {"name": "ESPAsyncTCP", "owner": "ESP32Async", "version": "^2.0.0"}, + # Not bundled: skipped + {"name": "NotBundled"}, + ], + }, + ) + + with _emitting_converter(converted) as mock_extra: + libs = _resolve(framework) + + mock_extra.assert_called_once() + assert mock_extra.call_args.args == (converted,) + assert mock_extra.call_args.kwargs["pio_platform"] == "espressif8266" + # board_mcu is passed lazily, as the shared helper requires + assert mock_extra.call_args.kwargs["board_mcu"]() == "esp8266" + assert [lib.name for lib in libs] == [ + "Wire", + "esp32async__ESPAsyncWebServer", + ] + + +def test_resolve_libraries_bundled_dep_already_present(tmp_path: Path) -> None: + framework = _make_framework(tmp_path) + _add_library("Wire", None) + _add_library("Some/External", "1.0.0") + + lib_dir = tmp_path / "converted" / "external" + lib_dir.mkdir(parents=True) + (lib_dir / "main.cpp").write_text("") + converted = _converted( + "some__External", lib_dir, {"dependencies": [{"name": "Wire"}]} + ) + + with _emitting_converter(converted): + libs = _resolve(framework) + + # Wire appears once (from the explicit registration), not twice + assert [lib.name for lib in libs] == ["Wire", "some__External"] + + +def test_library_info_trailing_bare_flag_warns( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + read_path = tmp_path / "lib" + (read_path / "src").mkdir(parents=True) + (read_path / "src" / "stub.cpp").write_text("") + lib = component._library_info("x", read_path, {"build": {"flags": ["-DA=1 -l"]}}) + assert lib.flags == ["-DA=1"] + assert lib.link_libs == [] + assert "Ignoring trailing '-l'" in caplog.text + + +def test_library_info_missing_explicit_include_warns( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + read_path = tmp_path / "lib" + (read_path / "src").mkdir(parents=True) + (read_path / "src" / "stub.cpp").write_text("") + lib = component._library_info("x", read_path, {"build": {"flags": ["-Inope"]}}) + assert lib.include_dirs == [(read_path / "src").resolve()] + assert "include dir nope which does not exist" in caplog.text + + +def test_library_info_missing_declared_src_dir_raises(tmp_path: Path) -> None: + """An explicitly declared srcDir that does not exist is a manifest error.""" + read_path = tmp_path / "lib" + read_path.mkdir() + with pytest.raises(EsphomeError, match="srcDir 'nosrc' which does not exist"): + component._library_info("x", read_path, {"build": {"srcDir": "nosrc"}}) + + +def test_library_info_missing_declared_include_dir_warns( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + read_path = tmp_path / "lib" + read_path.mkdir() + component._library_info("x", read_path, {"build": {"includeDir": "noinc"}}) + assert "include dir noinc which does not exist" in caplog.text + + +def test_resolve_libraries_lib_ignore_covers_bundled(tmp_path: Path) -> None: + """lib_ignore applies to framework-bundled libraries, as under PlatformIO.""" + framework = _make_framework(tmp_path) + _add_library("ESP8266WiFi", None) + _add_library("Wire", None) + CORE.platformio_options = {"lib_ignore": ["Wire"]} + libs = _resolve(framework) + assert [lib.name for lib in libs] == ["ESP8266WiFi"] + + +def test_resolve_libraries_lib_ignore_covers_bundled_dependencies( + tmp_path: Path, +) -> None: + framework = _make_framework(tmp_path) + _add_library("Some/External", "1.0.0") + CORE.platformio_options = {"lib_ignore": ["Wire"]} + + lib_dir = tmp_path / "converted" / "external" + lib_dir.mkdir(parents=True) + (lib_dir / "main.cpp").write_text("") + converted = _converted( + "some__External", lib_dir, {"dependencies": [{"name": "Wire"}]} + ) + + with _emitting_converter(converted): + libs = _resolve(framework) + + assert [lib.name for lib in libs] == ["some__External"] + + +def test_bundled_library_prefers_library_json(tmp_path: Path) -> None: + """A bundled library.json wins over library.properties (PIO semantics); + its build section is honored.""" + framework = _make_framework(tmp_path) + lib_dir = framework / "libraries" / "GDBStub" + (lib_dir / "custom").mkdir(parents=True) + (lib_dir / "custom" / "gdb.cpp").write_text("") + (lib_dir / "library.properties").write_text("name=GDBStub\n") + (lib_dir / "library.json").write_text( + '{"name": "GDBStub", "build": {"srcDir": "custom"}}' + ) + lib = component._bundled_library(framework, "GDBStub") + assert [s.name for s in lib.sources] == ["gdb.cpp"] + + +def test_library_info_lib_archive_flag(tmp_path: Path) -> None: + """Both libArchive (library.json) and dot_a_linkage (properties) reach + the generator's contract; default is archive.""" + read_path = tmp_path / "lib" + (read_path / "src").mkdir(parents=True) + (read_path / "src" / "stub.cpp").write_text("") + assert component._library_info("x", read_path, {}).lib_archive is True + assert ( + component._library_info( + "x", read_path, {"build": {"libArchive": False}} + ).lib_archive + is False + ) + assert ( + component._library_info("x", read_path, {"dot_a_linkage": "false"}).lib_archive + is False + ) + assert ( + component._library_info("x", read_path, {"dot_a_linkage": "true"}).lib_archive + is True + ) + + +def test_resolve_libraries_dep_warnings( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + """A nameless dependency entry warns in the shared normalizer; an + owner-without-version entry is left to the walk's reconciliation.""" + framework = _make_framework(tmp_path) + converted = _webserver( + tmp_path, + { + "build": {}, + "dependencies": [ + {"owner": "someone"}, + {"name": "Orphan", "owner": "someone"}, + ], + }, + ) + with _emitting_converter(converted): + _resolve(framework) + assert "Ignoring unrecognized dependency entry" in caplog.text + assert "Orphan" not in caplog.text + + +def test_bundled_dependency_nonplatform_rejection_is_silent_here( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + """The shared walk owns the rejection warning; the backend-side filter + stays at debug so one manifest fault never warns twice.""" + framework = _make_framework(tmp_path) + converted = _webserver(tmp_path, {"build": {}, "dependencies": [{"name": "Wire"}]}) + with ( + _emitting_converter(converted), + patch.object( + component, + "check_library_data", + side_effect=InvalidLibrary("manifest is corrupt"), + ), + ): + libs = _resolve(framework) + assert "Wire" not in [lib.name for lib in libs] + assert "manifest is corrupt" not in caplog.text + + +def test_nonplatform_rejection_warns_once_through_real_converter( + tmp_path: Path, + caplog: pytest.LogCaptureFixture, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """One manifest fault produces exactly one warning across the walk and + the backend-side bundled filter.""" + framework = _make_framework(tmp_path) + _local_lib(tmp_path, [{"name": "Wire"}]) + monkeypatch.setenv("ESPHOME_DATA_DIR", str(tmp_path / ".esphome")) + real = pio_library.check_library_data + + def flaky(data, platform, framework_name): + if data.get("name") == "Wire": + raise InvalidLibrary("manifest is corrupt") + return real(data, platform, framework_name) + + monkeypatch.setattr(pio_library, "check_library_data", flaky) + monkeypatch.setattr(component, "check_library_data", flaky) + with patch.object( + pio_library, + "_resolve_registry_version", + side_effect=AssertionError("registry touched"), + ): + _resolve(framework) + assert caplog.text.count("manifest is corrupt") == 1 + + +def test_url_pinned_bundled_name_not_doubled(tmp_path: Path) -> None: + """A URL-pinned dependency names one specific source; the bundled copy + of the same short name must never be added on top of the fork.""" + framework = _make_framework(tmp_path) + converted = _webserver( + tmp_path, + { + "build": {}, + "dependencies": [ + {"name": "Wire", "version": "https://github.com/x/wire-fork.git"} + ], + }, + ) + with _emitting_converter(converted): + libs = _resolve(framework) + assert "Wire" not in [lib.name for lib in libs] + + +def test_versioned_bundled_candidate_fault_warns_once( + tmp_path: Path, + caplog: pytest.LogCaptureFixture, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A versioned bundled-name dependency with a manifest fault warns once, + from the walk's usability filter; the backend-side re-check stays quiet.""" + framework = _make_framework(tmp_path) + _local_lib(tmp_path, [{"name": "Wire", "version": "*"}]) + monkeypatch.setenv("ESPHOME_DATA_DIR", str(tmp_path / ".esphome")) + real = pio_library.check_library_data + + def flaky(data, platform, framework_name): + if data.get("name") == "Wire": + raise InvalidLibrary("manifest is corrupt") + return real(data, platform, framework_name) + + monkeypatch.setattr(pio_library, "check_library_data", flaky) + monkeypatch.setattr(component, "check_library_data", flaky) + with patch.object( + pio_library, + "_resolve_registry_version", + side_effect=AssertionError("registry touched"), + ): + libs = _resolve(framework) + assert "Wire" not in [lib.name for lib in libs] + assert caplog.text.count("manifest is corrupt") == 1 + + +def test_short_name_collision_with_bundled_name_warns( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + """Suppressing a genuinely bundled name on a short-name match warns; + an accidental collision would otherwise surface at link.""" + framework = _make_framework(tmp_path) + _add_library("Someone/Wire", "1.0.0") + converted = _converted( + "someone__Wire", + tmp_path / "conv", + {"build": {}, "dependencies": [{"name": "Wire"}]}, + ) + (tmp_path / "conv" / "src").mkdir(parents=True) + (tmp_path / "conv" / "src" / "a.cpp").write_text("") + with _emitting_converter(converted): + libs = _resolve(framework) + assert "Wire" not in [lib.name for lib in libs] + assert "assumed satisfied by a requested external library" in caplog.text + assert any(r.levelname == "WARNING" for r in caplog.records) + + +def test_missing_libraries_dir_is_a_broken_install(tmp_path: Path) -> None: + """A framework tree without libraries/ must fail by name, not silently + reroute every bundled name to the registry.""" + framework = tmp_path / "framework" + framework.mkdir() + _add_library("Wire", None) + with pytest.raises(EsphomeError, match="framework install may be incomplete"): + _resolve(framework) + + +def test_provided_is_case_sensitive(tmp_path: Path) -> None: + """Membership uses the exact on-disk names, so a case-insensitive + filesystem cannot add the same bundled library twice.""" + framework = _make_framework(tmp_path) + converted = _webserver(tmp_path, {"build": {}, "dependencies": [{"name": "wire"}]}) + with _emitting_converter(converted): + libs = _resolve(framework) + assert "wire" not in [lib.name for lib in libs] + assert "Wire" not in [lib.name for lib in libs] + + +@pytest.mark.parametrize("declared", ["", None]) +def test_library_info_falsy_declared_src_dir_raises( + tmp_path: Path, declared: str | None +) -> None: + """A declared-but-falsy srcDir must not silently fall back to the probe.""" + read_path = tmp_path / "lib" + (read_path / "src").mkdir(parents=True) + (read_path / "src" / "stub.cpp").write_text("") + with pytest.raises(EsphomeError, match="does not exist"): + component._library_info("x", read_path, {"build": {"srcDir": declared}}) + + +@pytest.mark.parametrize( + ("value", "expected"), + [ + (False, False), + ("false", False), + ("False", False), + ("true", True), + ], +) +def test_library_info_lib_archive_parse( + tmp_path: Path, + value: object, + expected: bool, +) -> None: + """bool("false") is True; the string forms must parse, not coerce.""" + read_path = tmp_path / "lib" + (read_path / "src").mkdir(parents=True) + (read_path / "src" / "stub.cpp").write_text("") + lib = component._library_info("x", read_path, {"build": {"libArchive": value}}) + assert lib.lib_archive is expected + + +def test_library_info_unsupported_link_fields_raise(tmp_path: Path) -> None: + """precompiled/ldflags properties are not supported; refuse by name.""" + read_path = tmp_path / "lib" + (read_path / "src").mkdir(parents=True) + (read_path / "src" / "stub.cpp").write_text("") + with pytest.raises(EsphomeError, match="declares precompiled"): + component._library_info("x", read_path, {"precompiled": "true", "build": {}}) + with pytest.raises(EsphomeError, match="declares ldflags"): + component._library_info("x", read_path, {"ldflags": "-lfoo", "build": {}}) + + +@pytest.mark.parametrize("value", ["false", "False", " false ", "", False, None]) +def test_library_info_precompiled_opt_out_accepted( + tmp_path: Path, value: object +) -> None: + """Manifest values are strings; precompiled=false is the spec's + explicit opt-out, not a declaration.""" + read_path = tmp_path / "lib" + (read_path / "src").mkdir(parents=True) + (read_path / "src" / "stub.cpp").write_text("") + data = {"build": {}} + if value is not None: + data["precompiled"] = value + component._library_info("x", read_path, data) + + +@pytest.mark.parametrize("value", ["full", True, "weird"]) +def test_library_info_precompiled_set_raises(tmp_path: Path, value: object) -> None: + """Both full (Arduino's other legal value) and unknown spellings fail safe.""" + read_path = tmp_path / "lib" + (read_path / "src").mkdir(parents=True) + (read_path / "src" / "stub.cpp").write_text("") + with pytest.raises(EsphomeError, match="declares precompiled"): + component._library_info("x", read_path, {"precompiled": value, "build": {}}) + + +def test_library_info_default_filter_matching_nothing_warns( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + """The empty-match warning is not gated on a declared srcFilter/srcDir; + a default-filter src/ holding only inert files warns too.""" + read_path = tmp_path / "lib" + (read_path / "src").mkdir(parents=True) + (read_path / "src" / "keywords.txt").write_text("") + lib = component._library_info("x", read_path, {"build": {}}) + assert not lib.sources + assert "no source files matched" in caplog.text + + +def test_library_info_unmapped_sources_warn( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + """Source-like files the case-sensitive suffix map rejects are named, + even when other sources compiled (a partial drop links with undefined + symbols far from the cause).""" + read_path = tmp_path / "lib" + (read_path / "src").mkdir(parents=True) + (read_path / "src" / "impl.CPP").write_text("") + (read_path / "src" / "sketch.ino").write_text("") + (read_path / "src" / "ok.cpp").write_text("") + lib = component._library_info("x", read_path, {"build": {}}) + assert [s.name for s in lib.sources] == ["ok.cpp"] + assert "not compiled: impl.CPP, sketch.ino" in caplog.text + + +def test_library_info_inert_only_filter_warns( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + """A declared srcFilter matching only inert files (no sources, no + headers) warns like one matching nothing at all.""" + read_path = tmp_path / "lib" + (read_path / "src").mkdir(parents=True) + (read_path / "src" / "keywords.txt").write_text("") + component._library_info("x", read_path, {"build": {"srcFilter": ["+<*>"]}}) + assert "no source files matched" in caplog.text + + +def test_library_info_declared_filter_matching_headers_stays_quiet( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + """A declared filter matching real headers is a header-only library.""" + read_path = tmp_path / "lib" + (read_path / "src").mkdir(parents=True) + (read_path / "src" / "api.h").write_text("") + component._library_info("x", read_path, {"build": {"srcFilter": ["+<*>"]}}) + assert "no source files matched" not in caplog.text + + +def test_library_info_header_only_src_stays_quiet( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + """A header-only library (real headers in src/) is routine, not a + warning (the default +<*> filter matches the headers too).""" + read_path = tmp_path / "lib" + (read_path / "src").mkdir(parents=True) + (read_path / "src" / "ArduinoJson.h").write_text("") + (read_path / "keywords.txt").write_text("") + lib = component._library_info("x", read_path, {"build": {}}) + assert lib.sources == [] + assert "not compiled" not in caplog.text + assert "srcFilter" not in caplog.text + + +def test_library_info_lib_archive_malformed_raises(tmp_path: Path) -> None: + """A typo'd libArchive fails by name like the other build fields.""" + read_path = tmp_path / "lib" + (read_path / "src").mkdir(parents=True) + (read_path / "src" / "stub.cpp").write_text("") + with pytest.raises(EsphomeError, match="malformed libArchive value 'archive-me'"): + component._library_info("x", read_path, {"build": {"libArchive": "archive-me"}}) + + +def test_bundled_dependency_dict_shorthand_prefers_bundled(tmp_path: Path) -> None: + """The {"Wire": "*"} dict shorthand resolves to the bundled library.""" + framework = _make_framework(tmp_path) + converted = _webserver(tmp_path, {"build": {}, "dependencies": {"Wire": "*"}}) + with _emitting_converter(converted): + libs = _resolve(framework) + assert "Wire" in [lib.name for lib in libs] + + +def test_unfulfilled_provides_promise_raises(tmp_path: Path) -> None: + """A provides()-skipped dependency nothing added can only surface as + undefined symbols at link, so it fails here by name; satisfied ones + pass silently.""" + with pytest.raises(EsphomeError, match="Wire") as err: + component._check_unfulfilled_provides( + {"Wire", "Hash"}, {"Hash"}, {"Wire", "Hash"} + ) + assert str(err.value).count("Wire") == 1 + assert "Hash" not in str(err.value) + component._check_unfulfilled_provides({"Hash"}, {"Hash"}, {"Hash"}) + # A recording for a since-re-resolved manifest is stale walk state, + # never a failure: no final manifest still requests Wire + component._check_unfulfilled_provides({"Wire"}, set(), set()) + + +def test_extra_script_link_flags_reach_the_library(tmp_path: Path) -> None: + """LINKFLAGS captured by an extra script travel outside build.flags and + must reach the library's link flags, matching the ESP-IDF backend.""" + framework = _make_framework(tmp_path) + converted = _webserver( + tmp_path, + { + "build": {}, + component.ESPHOME_DATA_KEY: { + component.ESPHOME_DATA_LINK_FLAGS_KEY: ["-Wl,--wrap=foo"] + }, + }, + ) + with _emitting_converter(converted): + libs = _resolve(framework) + (webserver,) = (lib for lib in libs if "ESPAsyncWebServer" in lib.name) + assert "-Wl,--wrap=foo" in webserver.link_flags + + +def test_bundled_dependency_platform_rejection_is_debug( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + """The typed IncompatiblePlatform (the routine cross-platform skip) + stays at debug regardless of message wording.""" + framework = _make_framework(tmp_path) + converted = _webserver(tmp_path, {"build": {}, "dependencies": [{"name": "Wire"}]}) + with ( + _emitting_converter(converted), + patch.object( + component, + "check_library_data", + side_effect=IncompatiblePlatform("nothing about the p-word here"), + ), + ): + libs = _resolve(framework) + assert "Wire" not in [lib.name for lib in libs] + assert "Skipping dependency Wire" not in caplog.text + + +@pytest.mark.parametrize("data", [{"build": "src"}, [], "nope"]) +def test_library_info_malformed_manifest_is_named(tmp_path: Path, data: object) -> None: + """A malformed manifest names the library, never an AttributeError.""" + read_path = tmp_path / "lib" + read_path.mkdir() + with pytest.raises(EsphomeError, match="Library x has a malformed manifest"): + component._library_info("x", read_path, data) + + +def test_bundled_library_with_declared_dependencies_warns( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + """A bundled manifest that declares dependencies is visible, not + silently skipped (a no-op for the ESP8266 core, not for every core).""" + framework = _make_framework(tmp_path) + wire = framework / "libraries" / "Wire" + (wire / "library.json").write_text( + '{"name": "Wire", "dependencies": [{"name": "SPI"}]}' + ) + _add_library("Wire", None) + _resolve(framework) + assert "Bundled library Wire declares dependencies" in caplog.text + + +@pytest.mark.parametrize( + ("build", "match"), + [ + ({"includeDir": ["a", "b"]}, "malformed includeDir"), + ({"srcFilter": [123]}, "malformed srcFilter"), + ], +) +def test_library_info_malformed_build_fields_are_named( + tmp_path: Path, build: dict, match: str +) -> None: + """Malformed includeDir/srcFilter fail naming the library like srcDir.""" + read_path = tmp_path / "lib" + (read_path / "src").mkdir(parents=True) + (read_path / "src" / "stub.cpp").write_text("") + with pytest.raises(EsphomeError, match=match): + component._library_info("x", read_path, {"build": build}) + + +@pytest.mark.parametrize( + ("value", "expected"), + [ + ("true", True), + ("False", False), + ], +) +def test_library_info_dot_a_linkage_parses_strictly( + tmp_path: Path, + value: str, + expected: bool, +) -> None: + """The dot_a_linkage property uses the same strict table as libArchive.""" + read_path = tmp_path / "lib" + (read_path / "src").mkdir(parents=True) + (read_path / "src" / "stub.cpp").write_text("") + lib = component._library_info("x", read_path, {"dot_a_linkage": value, "build": {}}) + assert lib.lib_archive is expected + + +def test_library_info_dot_a_linkage_malformed_raises(tmp_path: Path) -> None: + """A typo'd dot_a_linkage must not silently flip link semantics.""" + read_path = tmp_path / "lib" + (read_path / "src").mkdir(parents=True) + (read_path / "src" / "stub.cpp").write_text("") + with pytest.raises(EsphomeError, match="malformed dot_a_linkage value 'yes'"): + component._library_info("x", read_path, {"dot_a_linkage": "yes", "build": {}}) + + +def test_bundled_library_properties_depends_warns( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + """The library.properties depends= spelling reaches the visibility + warning too; the shared parser returns it raw.""" + framework = _make_framework(tmp_path) + wire = framework / "libraries" / "Wire" + (wire / "library.properties").write_text("name=Wire\nversion=1.0\ndepends=SPI\n") + _add_library("Wire", None) + caplog.set_level("INFO") + _resolve(framework) + assert "Library Wire declares dependencies via library.properties" in caplog.text + + +def test_bundled_library_extra_script_raises(tmp_path: Path) -> None: + """A bundled manifest relying on an extraScript would miscompile; + refuse by name.""" + framework = _make_framework(tmp_path) + wire = framework / "libraries" / "Wire" + (wire / "library.json").write_text( + '{"name": "Wire", "build": {"extraScript": "extra.py"}}' + ) + _add_library("Wire", None) + with pytest.raises(EsphomeError, match="Wire declares an extraScript"): + _resolve(framework) + + +def test_dependency_requested_top_level_is_not_a_drop( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + """A version-less manifest dependency the config separately requests is + already in the build; it is not probed as a bundled library.""" + framework = _make_framework(tmp_path) + _add_library("ESP32Async/ESPAsyncWebServer", "3.9.6") + _add_library("ESP32Async/ESPAsyncTCP", "2.0.0") + ws, tcp = _ws_tcp_pair(tmp_path) + with _emitting_converter(ws, tcp): + libs = _resolve(framework) + # Exactly the two converted libraries; no bundled stand-in was added + assert [lib.name for lib in libs] == [ + "esp32async__ESPAsyncWebServer", + "esp32async__ESPAsyncTCP", + ] + assert "Skipping" not in caplog.text + + +def test_bundled_library_non_dict_manifest_skips_probes_and_raises( + tmp_path: Path, +) -> None: + """A bundled library.json that is a JSON array skips the dependency and + extraScript probes and fails in _library_info naming the library.""" + framework = _make_framework(tmp_path) + wire = framework / "libraries" / "Wire" + (wire / "library.json").write_text('["not", "a", "manifest"]') + with pytest.raises(EsphomeError, match="Library Wire has a malformed manifest"): + component._bundled_library(framework, "Wire") + + +def test_bundled_missing_manifest_is_debug_only( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + """The legacy manifest-less layout is legal (the core ships FSTools + without one), so the diagnostic must stay below warning level.""" + framework = _make_framework(tmp_path) + with caplog.at_level(logging.DEBUG): + component._bundled_library(framework, "Wire") + record = next(r for r in caplog.records if "has no manifest" in r.message) + assert record.levelno == logging.DEBUG + + +def test_bundled_corrupt_library_json_fails_by_name(tmp_path: Path) -> None: + """A truncated bundled library.json fails with the library name and the + clean-all hint, not a raw JSONDecodeError.""" + framework = _make_framework(tmp_path) + (framework / "libraries" / "Wire" / "library.json").write_text("{truncated") + with pytest.raises(EsphomeError, match="Wire has a corrupt library.json"): + component._bundled_library(framework, "Wire") + + +def test_dict_shorthand_dependency_skips_registry_through_real_converter( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """{"Wire": "*"} resolves to the bundled copy without touching the + registry (real converter).""" + framework = _make_framework(tmp_path) + _local_lib(tmp_path, {"Wire": "*"}) + # Pin the component cache to tmp_path (data_dir honors an ambient + # ESPHOME_DATA_DIR otherwise) + monkeypatch.setenv("ESPHOME_DATA_DIR", str(tmp_path / ".esphome")) + with patch.object( + pio_library, + "_resolve_registry_version", + side_effect=AssertionError("registry touched"), + ): + libs = _resolve(framework) + names = [lib.name for lib in libs] + assert "Wire" in names + assert any("locallib" in n.lower() for n in names) + # The walk populated provided_requests for the skip; the backend added + # the bundled copy, so the reconciliation passed without raising + + +def test_versionless_provides_skip_is_reconciled_through_real_converter( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """A truly version-less bare-name dependency the walk skips on the + backend's promise is recorded and fulfilled by the bundled copy.""" + framework = _make_framework(tmp_path) + _local_lib(tmp_path, ["Wire"]) + monkeypatch.setenv("ESPHOME_DATA_DIR", str(tmp_path / ".esphome")) + with patch.object( + pio_library, + "_resolve_registry_version", + side_effect=AssertionError("registry touched"), + ): + libs = _resolve(framework) + assert "Wire" in [lib.name for lib in libs] + + +def test_platform_filtered_bundled_candidate_does_not_break_reconciliation( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """A bundled candidate the backend knowingly skips (platform filter) + counts as satisfied; the promise reconciliation must not raise.""" + framework = _make_framework(tmp_path) + _local_lib(tmp_path, [{"name": "Wire", "platforms": ["espressif32"]}]) + monkeypatch.setenv("ESPHOME_DATA_DIR", str(tmp_path / ".esphome")) + with patch.object( + pio_library, + "_resolve_registry_version", + side_effect=AssertionError("registry touched"), + ): + libs = _resolve(framework) + assert "Wire" not in [lib.name for lib in libs] + + +@pytest.mark.parametrize( + ("bad_name", "message"), + [ + # A non-string name never leaves the shared normalizer + (1, "Ignoring unrecognized dependency entry"), + ("../escape", "Ignoring malformed dependency entry"), + ("..", "Ignoring malformed dependency entry"), + ], +) +def test_bundled_dependency_bad_name_is_malformed( + tmp_path: Path, bad_name: object, message: str, caplog: pytest.LogCaptureFixture +) -> None: + """A dependency name becomes a path component; a traversal or a + non-string is a malformed entry, never joined.""" + framework = _make_framework(tmp_path) + converted = _webserver( + tmp_path, {"build": {}, "dependencies": [{"name": bad_name}]} + ) + with _emitting_converter(converted): + _resolve(framework) + assert message in caplog.text + + +def test_owner_qualified_dependency_is_silent( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + """The owner-qualified dependency spelling (PIO's Owner/Pkg) resolves via + the converter; it must not draw the malformed-entry warning.""" + framework = _make_framework(tmp_path) + converted = _webserver( + tmp_path, + { + "build": {}, + "dependencies": [{"name": "ESP32Async/AsyncTCP", "version": "^3.0"}], + }, + ) + with _emitting_converter(converted): + _resolve(framework) + assert "malformed" not in caplog.text + + +def test_bundled_dependency_string_list_form(tmp_path: Path) -> None: + """The bare string-list dependency form (PIO-legal) resolves to the + bundled library instead of vanishing in normalization.""" + framework = _make_framework(tmp_path) + converted = _webserver(tmp_path, {"build": {}, "dependencies": ["Wire"]}) + with _emitting_converter(converted): + libs = _resolve(framework) + assert "Wire" in [lib.name for lib in libs] + + +def test_pinned_bundled_dependency_substitution_warns( + tmp_path: Path, + caplog: pytest.LogCaptureFixture, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A non-* version pin on a backend-provided dependency is discarded + for the bundled copy; the substitution must be visible.""" + framework = _make_framework(tmp_path) + _local_lib(tmp_path, {"Wire": "^2.0.0"}) + monkeypatch.setenv("ESPHOME_DATA_DIR", str(tmp_path / ".esphome")) + with patch.object( + pio_library, + "_resolve_registry_version", + side_effect=AssertionError("registry touched"), + ): + libs = _resolve(framework) + assert "Wire" in [lib.name for lib in libs] + assert "pins version ^2.0.0; using the library bundled" in caplog.text + + +def test_transitively_resolved_dependency_does_not_warn( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + """A dependency the walk already resolved does not warn.""" + framework = _make_framework(tmp_path) + _add_library("ESP32Async/ESPAsyncWebServer", "3.9.6") + ws, tcp = _ws_tcp_pair(tmp_path) + with _emitting_converter(ws, tcp): + libs = _resolve(framework) + assert [lib.name for lib in libs] == [ + "esp32async__ESPAsyncWebServer", + "esp32async__ESPAsyncTCP", + ] + assert "Skipping" not in caplog.text + + +@pytest.mark.parametrize( + ("spec", "expected"), + [ + ("owner/Name", "Name"), + ("Name", "Name"), + ("Foo=file:///srv/Wire", "Foo"), + ("Foo=https://github.com/x/Wire", "Foo"), + # An "=" without a URL is a registry name, not the custom-name form + ("FOO=BAR", "FOO=BAR"), + ("https://github.com/x/Wire", "Wire"), + # Git tails are stripped like the walk's URL normalization + ("https://github.com/x/Wire.git", "Wire"), + ("git+https://github.com/x/Wire.git#v1", "Wire"), + ], +) +def test_external_short_name(spec: str, expected: str) -> None: + assert component._external_short_name(spec) == expected + + +def test_converted_manifest_name_suppresses_bundled_dependency( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + """A name a converted library's manifest provides is not also added + from the framework tree, even when the provider emits later; the + suppression warns like its external_short_names twin.""" + framework = _make_framework(tmp_path) + _add_library("ESP32Async/ESPAsyncWebServer", "3.9.6") + # Requested under a different short name; only the manifest says "Wire" + _add_library("Someone/WireLib", "9.9.9") + ws_dir = tmp_path / "converted" / "webserver" + (ws_dir / "src").mkdir(parents=True) + (ws_dir / "src" / "stub.cpp").write_text("") + wire_dir = tmp_path / "converted" / "wire" + (wire_dir / "src").mkdir(parents=True) + (wire_dir / "src" / "wire.cpp").write_text("") + ws = _converted( + "esp32async__ESPAsyncWebServer", + ws_dir, + {"build": {}, "dependencies": [{"name": "Wire"}]}, + ) + registry_wire = _converted( + "someone__WireLib", wire_dir, {"name": "Wire", "build": {}} + ) + with _emitting_converter(ws, registry_wire): + libs = _resolve(framework) + # The bundled Wire is not added alongside the registry-resolved one + assert [lib.name for lib in libs] == [ + "esp32async__ESPAsyncWebServer", + "someone__WireLib", + ] + assert "Dependency Wire is assumed satisfied by a converted" in caplog.text + + +def test_bundled_library_root_headers_pass_the_probe(tmp_path: Path) -> None: + """Headers anywhere in the bundled tree (uncommon suffixes and case + included) prove the install is intact, even with an empty src dir.""" + framework = _make_framework(tmp_path) + lib_dir = framework / "libraries" / "HeaderOnly" + (lib_dir / "src").mkdir(parents=True) + (lib_dir / "impl.HXX").write_text("") + lib = component._bundled_library(framework, "HeaderOnly") + assert lib.sources == [] + + +def test_empty_bundled_library_warns( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + """A bundled directory with no sources or headers is a broken install + that can never link; fail by name instead of warning into it.""" + framework = _make_framework(tmp_path) + (framework / "libraries" / "Empty").mkdir() + _add_library("Empty", None) + with pytest.raises(EsphomeError, match="Library Empty has no sources or headers"): + _resolve(framework) + + +def test_versionless_dependency_with_provider_stays_quiet( + tmp_path: Path, + caplog: pytest.LogCaptureFixture, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """With a provides backend the version-less skip is routine (debug) and + the bundled copy is picked up after emit.""" + framework = _make_framework(tmp_path) + _local_lib(tmp_path, [{"name": "Wire"}]) + monkeypatch.setenv("ESPHOME_DATA_DIR", str(tmp_path / ".esphome")) + with patch.object( + pio_library, + "_resolve_registry_version", + side_effect=AssertionError("registry touched"), + ): + libs = _resolve(framework) + assert "Wire" in [lib.name for lib in libs] + assert "has no version to resolve" not in caplog.text diff --git a/tests/unit_tests/test_platformio_library.py b/tests/unit_tests/test_platformio_library.py index 0c873dc3fe..0a16b118fc 100644 --- a/tests/unit_tests/test_platformio_library.py +++ b/tests/unit_tests/test_platformio_library.py @@ -10,7 +10,7 @@ from pathlib import Path import pytest -from esphome.core import EsphomeError, Library +from esphome.core import CORE, EsphomeError, Library import esphome.platformio.library as lib from esphome.platformio.library import ( SOURCE_KIND_FOR_SUFFIX, @@ -29,9 +29,13 @@ from esphome.platformio.library import ( ) -def _backend(emit=lambda component: None) -> LibraryBackend: +def _backend(emit=lambda component: None, provides=None) -> LibraryBackend: return LibraryBackend( - platform="espressif32", framework="espidf", emit=emit, cache_key="idf" + platform="espressif32", + framework="espidf", + emit=emit, + cache_key="idf", + provides=provides, ) @@ -952,3 +956,202 @@ def test_source_kind_map_shape() -> None: assert SOURCE_KIND_FOR_SUFFIX[".S"] == "aspp" assert SOURCE_KIND_FOR_SUFFIX[".c"] == "c" assert SOURCE_KIND_FOR_SUFFIX[".cpp"] == "cxx" + # SCons's case-sensitive C++ suffixes: PIO compiles .C as C++ + assert SOURCE_KIND_FOR_SUFFIX[".C"] == "cxx" + assert SOURCE_KIND_FOR_SUFFIX[".C++"] == "cxx" + + +def test_versionless_platform_filtered_dependency_stays_quiet( + tmp_path, monkeypatch, caplog: pytest.LogCaptureFixture +) -> None: + """A version-less dependency the platform filter excludes is + deliberately absent, not a drop to warn about.""" + _patch_download_with_manifests( + monkeypatch, + tmp_path, + { + "esphome/A": { + "name": "A", + "dependencies": [{"name": "Hash", "platforms": "espressif8266"}], + } + }, + ) + convert_libraries([Library("esphome/A", None, None)], _backend()) + assert "has no version to resolve" not in caplog.text + + +def test_versionless_ignored_dependency_stays_quiet( + tmp_path, monkeypatch, caplog: pytest.LogCaptureFixture +) -> None: + """A lib_ignore'd version-less dependency is deliberately excluded, not + a drop; no reconciliation warning.""" + _patch_download_with_manifests( + monkeypatch, + tmp_path, + {"esphome/A": {"name": "A", "dependencies": [{"name": "Hash"}]}}, + ) + CORE.platformio_options = {"lib_ignore": ["Hash"]} + convert_libraries([Library("esphome/A", None, None)], _backend()) + assert "has no version to resolve" not in caplog.text + + +def test_versionless_dependency_without_provider_warns( + tmp_path, monkeypatch, caplog: pytest.LogCaptureFixture +) -> None: + """A backend whose tree could supply the name warns on the drop; one + without provides() can never act on it, so it stays at debug.""" + _patch_download_with_manifests( + monkeypatch, + tmp_path, + { + "esphome/A": { + "name": "A", + # The duplicate entry warns once (reconciliation dedup) + "dependencies": [{"name": "Hash"}, {"name": "Hash"}], + } + }, + ) + convert_libraries( + [Library("esphome/A", None, None)], _backend(provides=lambda name: False) + ) + assert ( + caplog.text.count( + "Hash of esphome/A has no version to resolve and nothing provides it" + ) + == 1 + ) + caplog.clear() + with caplog.at_level(logging.DEBUG): + convert_libraries([Library("esphome/A", None, None)], _backend()) + records = [ + r + for r in caplog.records + if "has no version to resolve and nothing provides it" in r.message + ] + assert records and all(r.levelno == logging.DEBUG for r in records) + + +def test_url_version_dependency_is_not_substituted_by_provides( + tmp_path, monkeypatch, caplog: pytest.LogCaptureFixture +) -> None: + """A URL-valued version names one specific source; the backend-provided + skip must not replace it with the bundled copy.""" + _patch_download_with_manifests( + monkeypatch, + tmp_path, + { + "esphome/A": { + "name": "A", + "dependencies": [ + {"name": "Hash", "version": "https://github.com/o/Hash.git"} + ], + }, + "o/Hash": {"name": "Hash"}, + }, + ) + emitted: list[str] = [] + convert_libraries( + [Library("esphome/A", "1.0.0", None)], + _backend(emit=lambda c: emitted.append(c.name), provides=lambda name: True), + ) + assert "Skip backend-provided" not in caplog.text + assert "using the library bundled" not in caplog.text + assert any("o/hash" in n.lower() for n in emitted) + + +def test_versionless_owner_qualified_dependency_warns_despite_provides( + tmp_path, monkeypatch, caplog: pytest.LogCaptureFixture +) -> None: + """An owner-qualified version-less dependency is not satisfied by + provides(); it must still warn.""" + _patch_download_with_manifests( + monkeypatch, + tmp_path, + { + "esphome/A": { + "name": "A", + "dependencies": [{"name": "Wire", "owner": "Foo"}], + } + }, + ) + convert_libraries( + [Library("esphome/A", None, None)], + _backend(provides=lambda name: name == "Wire"), + ) + assert "Wire of esphome/A has no version to resolve" in caplog.text + + +def test_versionless_provided_dependency_stays_quiet( + tmp_path, monkeypatch, caplog: pytest.LogCaptureFixture +) -> None: + """An owner-less version-less dependency the backend provides is added + by the backend after emit; no reconciliation warning.""" + _patch_download_with_manifests( + monkeypatch, + tmp_path, + {"esphome/A": {"name": "A", "dependencies": [{"name": "Wire"}]}}, + ) + convert_libraries( + [Library("esphome/A", None, None)], + _backend(provides=lambda name: name == "Wire"), + ) + assert "has no version to resolve" not in caplog.text + + +def test_versionless_dependency_requested_top_level_stays_quiet( + tmp_path, monkeypatch, caplog: pytest.LogCaptureFixture +) -> None: + """A version-less dependency the config also requests top-level is in + the build; no drop warning even without a provides backend.""" + _patch_download_with_manifests( + monkeypatch, + tmp_path, + { + "esphome/A": {"name": "A", "dependencies": [{"name": "Hash"}]}, + "Hash": {"name": "Hash"}, + }, + ) + convert_libraries( + [Library("esphome/A", None, None), Library("Hash", None, None)], + _backend(), + ) + assert "has no version to resolve" not in caplog.text + + +def test_versionless_url_ish_dependency_name_warns_cleanly( + tmp_path, monkeypatch, caplog: pytest.LogCaptureFixture +) -> None: + """A malformed URL-ish dependency name falls to the drop warning, never + a RuntimeError out of the key parser.""" + _patch_download_with_manifests( + monkeypatch, + tmp_path, + {"esphome/A": {"name": "A", "dependencies": [{"name": "file://"}]}}, + ) + convert_libraries( + [Library("esphome/A", None, None)], _backend(provides=lambda name: False) + ) + assert ( + "file:// of esphome/A has no version to resolve and nothing provides it" + in caplog.text + ) + + +def test_versionless_dependency_matching_resolved_manifest_name_stays_quiet( + tmp_path, monkeypatch, caplog: pytest.LogCaptureFixture +) -> None: + """A bare name satisfied by an owner-qualified component's manifest + name is not a drop.""" + _patch_download_with_manifests( + monkeypatch, + tmp_path, + { + "esphome/A": {"name": "A", "dependencies": [{"name": "B"}]}, + "esphome/B": {"name": "B"}, + }, + ) + convert_libraries( + [Library("esphome/A", None, None), Library("esphome/B", None, None)], + _backend(), + ) + assert "has no version to resolve" not in caplog.text