diff --git a/esphome/arduino8266/build_tool.py b/esphome/arduino8266/build_tool.py new file mode 100644 index 0000000000..bfcf4128eb --- /dev/null +++ b/esphome/arduino8266/build_tool.py @@ -0,0 +1,36 @@ +"""Tiny cross-platform build steps invoked from the generated ninja file. + +Plain script (not ``python -m``): it runs from ninja with whatever Python +started esphome and must not depend on the package being importable. + +Subcommands: + ar remove stale archive, then ``ar rc`` + copy copy a file +""" + +from pathlib import Path +import shutil +import subprocess +import sys + + +def main() -> int: + mode = sys.argv[1] + if mode == "ar": + ar, archive, rspfile = sys.argv[2:5] + # Remove first: ``ar rc`` replaces members but never drops ones whose + # source was removed from the build, which would leak stale objects. + Path(archive).unlink(missing_ok=True) + return subprocess.run( + [ar, "rc", archive, f"@{rspfile}"], check=False + ).returncode + if mode == "copy": + src, dst = sys.argv[2:4] + shutil.copyfile(src, dst) + return 0 + print(f"unknown build_tool mode: {mode}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": # pragma: no cover + sys.exit(main()) diff --git a/esphome/arduino8266/component.py b/esphome/arduino8266/component.py new file mode 100644 index 0000000000..17fd5471b4 --- /dev/null +++ b/esphome/arduino8266/component.py @@ -0,0 +1,232 @@ +"""Arduino ESP8266 backend for the shared PlatformIO library converter. + +Turns the libraries registered via ``cg.add_library()`` into build inputs for +the ninja generator. Bare names that exist under the framework's bundled +``libraries/`` directory (ESP8266WiFi, Wire, SPI, ...) are read straight from +the framework tree; everything else goes through the shared +resolution/download pipeline in ``esphome.platformio.library``. + +Mirrors PlatformIO's ``lib_ldf_mode=off`` behavior: each library builds into +its own static archive and every library's include dir joins one global +include path. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +import logging +from pathlib import Path + +from esphome.build_helpers.extra_script import apply_extra_script +from esphome.core import CORE, EsphomeError, Library +from esphome.platformio.library import ( + DEFAULT_BUILD_INCLUDE_DIR, + DEFAULT_BUILD_SRC_FILTER, + SRC_FILE_EXTENSIONS, + ConvertedLibrary, + InvalidLibrary, + LibraryBackend, + check_library_data, + collect_filtered_files, + convert_libraries, + ensure_list, + is_lib_ignored, + join_flag_args, + lib_ignore_set, + normalize_dependencies, + parse_library_properties, + split_flag_entry, +) + +_LOGGER = logging.getLogger(__name__) + +ESP8266_PLATFORM = "espressif8266" + + +@dataclass +class ArduinoLibrary: + """One resolved library, ready for the ninja generator.""" + + name: str + sources: list[Path] = field(default_factory=list) + include_dirs: list[Path] = field(default_factory=list) + # Extra compile flags private to this library's own sources + flags: list[str] = field(default_factory=list) + # Link inputs the library contributes (-L dirs / -l libs, e.g. from + # precompiled vendor blobs) and -Wl, options for the firmware link + link_dirs: list[Path] = field(default_factory=list) + link_libs: list[str] = field(default_factory=list) + link_flags: list[str] = field(default_factory=list) + + +def _library_info(name: str, read_path: Path, data: dict) -> ArduinoLibrary: + """Resolve one library's sources, include dirs, and flags (PIO semantics).""" + build = data.get("build", {}) + + # PIO's source-dir resolution: manifest srcDir, else src/Src, else the root + src_dir = build.get("srcDir") or next( + (d for d in ("src", "Src") if (read_path / d).is_dir()), "." + ) + if "srcDir" in build and not (read_path / src_dir).is_dir(): + # Unlike the default probes, an explicitly declared srcDir that does + # not resolve is unambiguously a manifest/tree error; a silently + # empty source set would surface as link errors far from the cause + raise EsphomeError( + f"Library {name} declares srcDir {src_dir} which does not exist" + ) + + src_filter = ensure_list(build.get("srcFilter", DEFAULT_BUILD_SRC_FILTER)) + # PlatformIO shell-lexes each build.flags entry + flag_tokens = join_flag_args( + ( + token + for entry in ensure_list(build.get("flags", [])) + for token in split_flag_entry(entry, f"library {name}") + ), + f"library {name}", + ) + + lib = ArduinoLibrary(name=name) + include_flags: list[str] = [] + for tok in flag_tokens: + if tok.startswith("-I"): + include_flags.append(tok[2:]) + elif tok.startswith("-L"): + link_dir = (read_path / tok[2:]).resolve() + if not link_dir.is_dir(): + # Kept anyway (the linker ignores missing -L dirs); the + # warning names the culprit before a bare "cannot find -lfoo" + _LOGGER.warning( + "Library %s declares library dir %s which does not exist", + name, + tok[2:], + ) + lib.link_dirs.append(link_dir) + elif tok.startswith("-l"): + lib.link_libs.append(tok[2:]) + elif tok.startswith("-Wl,"): + lib.link_flags.append(tok) + else: + lib.flags.append(tok) + + include_dir = build.get("includeDir", DEFAULT_BUILD_INCLUDE_DIR) + for d in [include_dir, src_dir, *include_flags]: + if (path := (read_path / d)).is_dir(): + lib.include_dirs.append(path.resolve()) + elif d in include_flags or (d == include_dir and "includeDir" in build): + # The includeDir/srcDir defaults are probes; an explicitly + # declared path that does not resolve is a manifest error + _LOGGER.warning( + "Library %s declares include dir %s which does not exist", name, d + ) + + lib.sources = sorted( + path.resolve() + for f in collect_filtered_files(read_path / src_dir, src_filter) + if (path := Path(f)).suffix in SRC_FILE_EXTENSIONS + ) + if not lib.sources and ("srcFilter" in build or "srcDir" in build): + # A default probe finding nothing is a header-only library; a + # declared filter matching nothing is a manifest/tree problem. + _LOGGER.warning( + "Library %s declares srcFilter/srcDir but no source files matched", + name, + ) + return lib + + +def _bundled_library(framework_path: Path, name: str) -> ArduinoLibrary: + """A library bundled with the Arduino core, read from the framework tree.""" + lib_dir = framework_path / "libraries" / name + manifest = lib_dir / "library.properties" + data = parse_library_properties(manifest) if manifest.is_file() else {} + return _library_info(name, lib_dir, {"name": name, **data}) + + +def resolve_libraries(framework_path: Path) -> list[ArduinoLibrary]: + """Resolve every ``cg.add_library()`` entry into an :class:`ArduinoLibrary`.""" + bundled: list[ArduinoLibrary] = [] + external: list[Library] = [] + # PlatformIO's lib_ignore covers framework-bundled libraries too; the + # shared converter only filters the registry/git ones. + lib_ignore = lib_ignore_set() + for library in CORE.platformio_libraries.values(): + if is_lib_ignored(library.name, lib_ignore): + continue + # A version pin means a registry package ("pngle@1.1.0"), never a + # framework-bundled library. + if ( + library.repository + or library.version + or not library.name + or "/" in library.name + ): + external.append(library) + elif (framework_path / "libraries" / library.name).is_dir(): + bundled.append(_bundled_library(framework_path, library.name)) + else: + # A bare registry name; resolved at the latest version, matching + # PlatformIO (a typo fails loudly as a registry lookup error). + external.append(library) + + converted: list[ArduinoLibrary] = [] + bundled_names = {lib.name for lib in bundled} + + def _add_bundled_dependencies(component: ConvertedLibrary) -> None: + # A version-less bare-name dependency ("Hash" in ESPAsyncWebServer) + # is a core-bundled library; the shared converter skips it because + # it cannot be resolved from the registry. + for dep in normalize_dependencies(component.data.get("dependencies")): + name = dep.get("name") + if ( + not name + or dep.get("owner") + or "version" in dep + or name in bundled_names + or is_lib_ignored(name, lib_ignore) + ): + continue + if not (framework_path / "libraries" / name).is_dir(): + # The shared converter skips version-less deps too, so this + # is the only place the drop can be made visible before the + # missing sources surface as link errors. + _LOGGER.warning( + "Dependency %s of library %s is not bundled with the " + "framework and has no version to resolve; skipping", + name, + component.name, + ) + continue + try: + check_library_data(dep, ESP8266_PLATFORM, "arduino") + except InvalidLibrary as err: + # check_library_data's only raise is the platform filter, and + # rejecting another platform's dependency of a cross-platform + # manifest is routine (every ESPAsyncWebServer build hits it); + # a warning here would be noise, and the reason is in the log. + _LOGGER.debug("Skipping bundled dependency %s: %s", name, err) + continue + bundled_names.add(name) + bundled.append(_bundled_library(framework_path, name)) + + def _emit(component: ConvertedLibrary) -> None: + apply_extra_script(component, "esp8266", pio_platform=ESP8266_PLATFORM) + converted.append( + _library_info( + component.get_require_name(), component.source_dir, component.data + ) + ) + _add_bundled_dependencies(component) + + if external: + convert_libraries( + external, + LibraryBackend( + platform=ESP8266_PLATFORM, + framework="arduino", + emit=_emit, + cache_key="arduino8266", + ), + ) + + return bundled + converted diff --git a/tests/unit_tests/test_arduino8266_build_tool.py b/tests/unit_tests/test_arduino8266_build_tool.py new file mode 100644 index 0000000000..66d11a6700 --- /dev/null +++ b/tests/unit_tests/test_arduino8266_build_tool.py @@ -0,0 +1,63 @@ +"""Tests for the ninja build-tool helper script.""" + +from __future__ import annotations + +from pathlib import Path +from unittest.mock import MagicMock, patch + +import pytest + +from esphome.arduino8266 import build_tool + + +def test_ar_removes_stale_archive(tmp_path: Path) -> None: + archive = tmp_path / "lib.a" + archive.write_text("stale") + rsp = tmp_path / "lib.a.rsp" + rsp.write_text("a.o\n") + with ( + patch.object( + build_tool.sys, + "argv", + ["build_tool", "ar", "ar-bin", str(archive), str(rsp)], + ), + patch.object( + build_tool.subprocess, "run", return_value=MagicMock(returncode=0) + ) as mock_run, + ): + assert build_tool.main() == 0 + assert not archive.exists() + assert mock_run.call_args[0][0] == ["ar-bin", "rc", str(archive), f"@{rsp}"] + + +def test_copy(tmp_path: Path) -> None: + src = tmp_path / "firmware.bin" + src.write_text("data") + dst = tmp_path / "firmware.factory.bin" + with patch.object( + build_tool.sys, "argv", ["build_tool", "copy", str(src), str(dst)] + ): + assert build_tool.main() == 0 + assert dst.read_text() == "data" + + +def test_unknown_mode(capsys: pytest.CaptureFixture[str]) -> None: + with patch.object(build_tool.sys, "argv", ["build_tool", "bogus"]): + assert build_tool.main() == 1 + assert "unknown build_tool mode" in capsys.readouterr().err + + +def test_runs_as_script(tmp_path: Path) -> None: + """The ninja rules invoke the file as a plain script.""" + import subprocess + import sys + + src = tmp_path / "a.bin" + src.write_text("x") + dst = tmp_path / "b.bin" + result = subprocess.run( + [sys.executable, build_tool.__file__, "copy", str(src), str(dst)], + check=False, + ) + assert result.returncode == 0 + assert dst.read_text() == "x" diff --git a/tests/unit_tests/test_arduino8266_component.py b/tests/unit_tests/test_arduino8266_component.py new file mode 100644 index 0000000000..efe38a68dc --- /dev/null +++ b/tests/unit_tests/test_arduino8266_component.py @@ -0,0 +1,310 @@ +"""Tests for esphome.arduino8266.component (library resolution).""" + +from __future__ import annotations + +from pathlib import Path +from unittest.mock import patch + +import pytest + +from esphome.arduino8266 import component +from esphome.const import KEY_CORE, KEY_TARGET_PLATFORM, PLATFORM_ESP8266 +from esphome.core import CORE, EsphomeError, Library +from esphome.platformio.library import ConvertedLibrary, LibraryBackend + + +@pytest.fixture(autouse=True) +def _reset_libraries() -> None: + CORE.platformio_libraries = {} + CORE.data[KEY_CORE] = {KEY_TARGET_PLATFORM: PLATFORM_ESP8266} + + +def _add_library(name: str, version: str | None, repository: str | None = None) -> None: + CORE.add_library(Library(name=name, version=version, repository=repository)) + + +def _make_framework(tmp_path: Path) -> Path: + framework = tmp_path / "framework" + lib = framework / "libraries" / "ESP8266WiFi" / "src" + lib.mkdir(parents=True) + (lib / "ESP8266WiFi.cpp").write_text("") + (lib / "ESP8266WiFi.h").write_text("") + (lib.parent / "library.properties").write_text("name=ESP8266WiFi\nversion=1.0\n") + root_lib = framework / "libraries" / "Wire" + root_lib.mkdir(parents=True) + (root_lib / "Wire.cpp").write_text("") + (root_lib / "examples").mkdir() + (root_lib / "examples" / "scan.ino").write_text("") + return framework + + +def test_library_info_src_layout(tmp_path: Path) -> None: + framework = _make_framework(tmp_path) + lib = component._bundled_library(framework, "ESP8266WiFi") + assert lib.name == "ESP8266WiFi" + assert [p.name for p in lib.sources] == ["ESP8266WiFi.cpp"] + assert lib.include_dirs == [(framework / "libraries/ESP8266WiFi/src").resolve()] + + +def test_library_info_root_layout_excludes_examples(tmp_path: Path) -> None: + framework = _make_framework(tmp_path) + lib = component._bundled_library(framework, "Wire") + assert [p.name for p in lib.sources] == ["Wire.cpp"] + assert lib.include_dirs == [(framework / "libraries/Wire").resolve()] + + +def test_library_info_flags_parsing(tmp_path: Path) -> None: + read_path = tmp_path / "lib" + (read_path / "src").mkdir(parents=True) + (read_path / "src" / "a.cpp").write_text("") + (read_path / "inc").mkdir() + (read_path / "blobs").mkdir() + data = { + "build": { + "flags": [ + "-DFOO=1 -I inc", + "-lalgobsec", + "-fno-lto", + "-Wl,--wrap=malloc", + "-l", + "m", + "-L", + "blobs", + ], + } + } + lib = component._library_info("x", read_path, data) + assert lib.flags == ["-DFOO=1", "-fno-lto"] + assert lib.include_dirs == [ + (read_path / "src").resolve(), + (read_path / "inc").resolve(), + ] + assert lib.link_dirs == [(read_path / "blobs").resolve()] + assert lib.link_libs == ["algobsec", "m"] + assert lib.link_flags == ["-Wl,--wrap=malloc"] + + +def test_library_info_missing_link_dir_warns( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + read_path = tmp_path / "lib" + read_path.mkdir() + data = {"build": {"flags": ["-Lmissing_blobs"]}} + lib = component._library_info("x", read_path, data) + assert "declares library dir missing_blobs which does not exist" in caplog.text + # Kept anyway: the linker ignores missing -L dirs + assert lib.link_dirs == [(read_path / "missing_blobs").resolve()] + + +def test_library_info_declared_filter_matches_nothing_warns( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + read_path = tmp_path / "lib" + (read_path / "src").mkdir(parents=True) + data = {"build": {"srcFilter": ["+"]}} + lib = component._library_info("x", read_path, data) + assert not lib.sources + assert "declares srcFilter/srcDir but no source files matched" in caplog.text + + +def test_library_info_header_only_does_not_warn( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + read_path = tmp_path / "lib" + (read_path / "src").mkdir(parents=True) + lib = component._library_info("x", read_path, {}) + assert not lib.sources + assert "no source files matched" not in caplog.text + + +def test_library_info_no_src_dir(tmp_path: Path) -> None: + read_path = tmp_path / "empty" + read_path.mkdir() + lib = component._library_info("x", read_path, {}) + # With no manifest hints the source dir falls back to the library root + assert lib.sources == [] + assert lib.include_dirs == [read_path.resolve()] + + +def test_resolve_libraries_bundled(tmp_path: Path) -> None: + framework = _make_framework(tmp_path) + _add_library("ESP8266WiFi", None) + libs = component.resolve_libraries(framework) + assert [lib.name for lib in libs] == ["ESP8266WiFi"] + + +def test_resolve_libraries_bare_registry_name_is_external(tmp_path: Path) -> None: + """A bare name that is not bundled resolves from the registry at the + latest version, matching PlatformIO and the documented libraries: key.""" + framework = _make_framework(tmp_path) + _add_library("pngle", None) + with patch.object(component, "convert_libraries", return_value=[]) as mock_convert: + component.resolve_libraries(framework) + (libraries, _backend), _ = mock_convert.call_args + assert [lib.name for lib in libraries] == ["pngle"] + + +def _converted(name: str, source_dir: Path, data: dict) -> ConvertedLibrary: + converted = ConvertedLibrary(name, "1.0.0", source=None) + converted.path = source_dir + converted.data = data + return converted + + +def test_resolve_libraries_external_and_bundled_deps(tmp_path: Path) -> None: + framework = _make_framework(tmp_path) + _add_library("ESP32Async/ESPAsyncWebServer", "3.9.6") + + lib_dir = tmp_path / "converted" / "webserver" + (lib_dir / "src").mkdir(parents=True) + (lib_dir / "src" / "server.cpp").write_text("") + converted = _converted( + "esp32async__ESPAsyncWebServer", + lib_dir, + { + "build": {}, + "dependencies": [ + # Version-less bundled dependency: resolved from the framework + {"name": "Wire", "platforms": "espressif8266"}, + # Wrong platform: skipped + {"name": "ESP8266WiFi", "platforms": "espressif32"}, + # Registry dependency with a version: handled by the converter + {"name": "ESPAsyncTCP", "owner": "ESP32Async", "version": "^2.0.0"}, + # Not bundled: skipped + {"name": "NotBundled"}, + ], + }, + ) + + def fake_convert(libraries: list, backend: LibraryBackend) -> list: + assert backend.platform == "espressif8266" + assert backend.framework == "arduino" + assert backend.cache_key == "arduino8266" + backend.emit(converted) + return [converted] + + with ( + patch.object(component, "convert_libraries", side_effect=fake_convert), + patch.object(component, "apply_extra_script") as mock_extra, + ): + libs = component.resolve_libraries(framework) + + mock_extra.assert_called_once_with( + converted, "esp8266", pio_platform="espressif8266" + ) + assert [lib.name for lib in libs] == [ + "Wire", + "esp32async__ESPAsyncWebServer", + ] + + +def test_resolve_libraries_bundled_dep_already_present(tmp_path: Path) -> None: + framework = _make_framework(tmp_path) + _add_library("Wire", None) + _add_library("Some/External", "1.0.0") + + lib_dir = tmp_path / "converted" / "external" + lib_dir.mkdir(parents=True) + converted = _converted( + "some__External", lib_dir, {"dependencies": [{"name": "Wire"}]} + ) + + def fake_convert(libraries: list, backend: LibraryBackend) -> list: + backend.emit(converted) + return [converted] + + with ( + patch.object(component, "convert_libraries", side_effect=fake_convert), + patch.object(component, "apply_extra_script"), + ): + libs = component.resolve_libraries(framework) + + # Wire appears once (from the explicit registration), not twice + assert [lib.name for lib in libs] == ["Wire", "some__External"] + + +def test_resolve_libraries_versioned_bare_name_is_external(tmp_path: Path) -> None: + """A bare name with a version pin ("pngle@1.1.0") is a registry package, + not a bundled library, and must reach the converter.""" + framework = _make_framework(tmp_path) + _add_library("pngle", "1.1.0") + + with patch.object(component, "convert_libraries", return_value=[]) as mock_convert: + component.resolve_libraries(framework) + + (libraries, _backend), _ = mock_convert.call_args + assert [lib.name for lib in libraries] == ["pngle"] + + +def test_library_info_trailing_bare_flag_warns( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + read_path = tmp_path / "lib" + (read_path / "src").mkdir(parents=True) + lib = component._library_info("x", read_path, {"build": {"flags": ["-DA=1 -l"]}}) + assert lib.flags == ["-DA=1"] + assert lib.link_libs == [] + assert "Ignoring trailing '-l'" in caplog.text + + +def test_library_info_missing_explicit_include_warns( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + read_path = tmp_path / "lib" + (read_path / "src").mkdir(parents=True) + lib = component._library_info("x", read_path, {"build": {"flags": ["-Inope"]}}) + assert lib.include_dirs == [(read_path / "src").resolve()] + assert "include dir nope which does not exist" in caplog.text + + +def test_library_info_missing_declared_src_dir_raises(tmp_path: Path) -> None: + """An explicitly declared srcDir that does not exist is a manifest error.""" + read_path = tmp_path / "lib" + read_path.mkdir() + with pytest.raises(EsphomeError, match="srcDir nosrc which does not exist"): + component._library_info("x", read_path, {"build": {"srcDir": "nosrc"}}) + + +def test_library_info_missing_declared_include_dir_warns( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + read_path = tmp_path / "lib" + read_path.mkdir() + component._library_info("x", read_path, {"build": {"includeDir": "noinc"}}) + assert "include dir noinc which does not exist" in caplog.text + + +def test_resolve_libraries_lib_ignore_covers_bundled(tmp_path: Path) -> None: + """lib_ignore applies to framework-bundled libraries, as under PlatformIO.""" + framework = _make_framework(tmp_path) + _add_library("ESP8266WiFi", None) + _add_library("Wire", None) + CORE.platformio_options = {"lib_ignore": ["Wire"]} + libs = component.resolve_libraries(framework) + assert [lib.name for lib in libs] == ["ESP8266WiFi"] + + +def test_resolve_libraries_lib_ignore_covers_bundled_dependencies( + tmp_path: Path, +) -> None: + framework = _make_framework(tmp_path) + _add_library("Some/External", "1.0.0") + CORE.platformio_options = {"lib_ignore": ["Wire"]} + + lib_dir = tmp_path / "converted" / "external" + lib_dir.mkdir(parents=True) + converted = _converted( + "some__External", lib_dir, {"dependencies": [{"name": "Wire"}]} + ) + + def fake_convert(libraries: list, backend: LibraryBackend) -> list: + backend.emit(converted) + return [converted] + + with ( + patch.object(component, "convert_libraries", side_effect=fake_convert), + patch.object(component, "apply_extra_script"), + ): + libs = component.resolve_libraries(framework) + + assert [lib.name for lib in libs] == ["some__External"]