Add the ESP8266 native library backend

This commit is contained in:
J. Nick Koston
2026-08-20 12:59:10 -05:00
parent 3e77654719
commit 82c3c1f50a
4 changed files with 641 additions and 0 deletions
+36
View File
@@ -0,0 +1,36 @@
"""Tiny cross-platform build steps invoked from the generated ninja file.
Plain script (not ``python -m``): it runs from ninja with whatever Python
started esphome and must not depend on the package being importable.
Subcommands:
ar <ar-binary> <archive> <rspfile> remove stale archive, then ``ar rc``
copy <src> <dst> copy a file
"""
from pathlib import Path
import shutil
import subprocess
import sys
def main() -> int:
mode = sys.argv[1]
if mode == "ar":
ar, archive, rspfile = sys.argv[2:5]
# Remove first: ``ar rc`` replaces members but never drops ones whose
# source was removed from the build, which would leak stale objects.
Path(archive).unlink(missing_ok=True)
return subprocess.run(
[ar, "rc", archive, f"@{rspfile}"], check=False
).returncode
if mode == "copy":
src, dst = sys.argv[2:4]
shutil.copyfile(src, dst)
return 0
print(f"unknown build_tool mode: {mode}", file=sys.stderr)
return 1
if __name__ == "__main__": # pragma: no cover
sys.exit(main())
+232
View File
@@ -0,0 +1,232 @@
"""Arduino ESP8266 backend for the shared PlatformIO library converter.
Turns the libraries registered via ``cg.add_library()`` into build inputs for
the ninja generator. Bare names that exist under the framework's bundled
``libraries/`` directory (ESP8266WiFi, Wire, SPI, ...) are read straight from
the framework tree; everything else goes through the shared
resolution/download pipeline in ``esphome.platformio.library``.
Mirrors PlatformIO's ``lib_ldf_mode=off`` behavior: each library builds into
its own static archive and every library's include dir joins one global
include path.
"""
from __future__ import annotations
from dataclasses import dataclass, field
import logging
from pathlib import Path
from esphome.build_helpers.extra_script import apply_extra_script
from esphome.core import CORE, EsphomeError, Library
from esphome.platformio.library import (
DEFAULT_BUILD_INCLUDE_DIR,
DEFAULT_BUILD_SRC_FILTER,
SRC_FILE_EXTENSIONS,
ConvertedLibrary,
InvalidLibrary,
LibraryBackend,
check_library_data,
collect_filtered_files,
convert_libraries,
ensure_list,
is_lib_ignored,
join_flag_args,
lib_ignore_set,
normalize_dependencies,
parse_library_properties,
split_flag_entry,
)
_LOGGER = logging.getLogger(__name__)
ESP8266_PLATFORM = "espressif8266"
@dataclass
class ArduinoLibrary:
"""One resolved library, ready for the ninja generator."""
name: str
sources: list[Path] = field(default_factory=list)
include_dirs: list[Path] = field(default_factory=list)
# Extra compile flags private to this library's own sources
flags: list[str] = field(default_factory=list)
# Link inputs the library contributes (-L dirs / -l libs, e.g. from
# precompiled vendor blobs) and -Wl, options for the firmware link
link_dirs: list[Path] = field(default_factory=list)
link_libs: list[str] = field(default_factory=list)
link_flags: list[str] = field(default_factory=list)
def _library_info(name: str, read_path: Path, data: dict) -> ArduinoLibrary:
"""Resolve one library's sources, include dirs, and flags (PIO semantics)."""
build = data.get("build", {})
# PIO's source-dir resolution: manifest srcDir, else src/Src, else the root
src_dir = build.get("srcDir") or next(
(d for d in ("src", "Src") if (read_path / d).is_dir()), "."
)
if "srcDir" in build and not (read_path / src_dir).is_dir():
# Unlike the default probes, an explicitly declared srcDir that does
# not resolve is unambiguously a manifest/tree error; a silently
# empty source set would surface as link errors far from the cause
raise EsphomeError(
f"Library {name} declares srcDir {src_dir} which does not exist"
)
src_filter = ensure_list(build.get("srcFilter", DEFAULT_BUILD_SRC_FILTER))
# PlatformIO shell-lexes each build.flags entry
flag_tokens = join_flag_args(
(
token
for entry in ensure_list(build.get("flags", []))
for token in split_flag_entry(entry, f"library {name}")
),
f"library {name}",
)
lib = ArduinoLibrary(name=name)
include_flags: list[str] = []
for tok in flag_tokens:
if tok.startswith("-I"):
include_flags.append(tok[2:])
elif tok.startswith("-L"):
link_dir = (read_path / tok[2:]).resolve()
if not link_dir.is_dir():
# Kept anyway (the linker ignores missing -L dirs); the
# warning names the culprit before a bare "cannot find -lfoo"
_LOGGER.warning(
"Library %s declares library dir %s which does not exist",
name,
tok[2:],
)
lib.link_dirs.append(link_dir)
elif tok.startswith("-l"):
lib.link_libs.append(tok[2:])
elif tok.startswith("-Wl,"):
lib.link_flags.append(tok)
else:
lib.flags.append(tok)
include_dir = build.get("includeDir", DEFAULT_BUILD_INCLUDE_DIR)
for d in [include_dir, src_dir, *include_flags]:
if (path := (read_path / d)).is_dir():
lib.include_dirs.append(path.resolve())
elif d in include_flags or (d == include_dir and "includeDir" in build):
# The includeDir/srcDir defaults are probes; an explicitly
# declared path that does not resolve is a manifest error
_LOGGER.warning(
"Library %s declares include dir %s which does not exist", name, d
)
lib.sources = sorted(
path.resolve()
for f in collect_filtered_files(read_path / src_dir, src_filter)
if (path := Path(f)).suffix in SRC_FILE_EXTENSIONS
)
if not lib.sources and ("srcFilter" in build or "srcDir" in build):
# A default probe finding nothing is a header-only library; a
# declared filter matching nothing is a manifest/tree problem.
_LOGGER.warning(
"Library %s declares srcFilter/srcDir but no source files matched",
name,
)
return lib
def _bundled_library(framework_path: Path, name: str) -> ArduinoLibrary:
"""A library bundled with the Arduino core, read from the framework tree."""
lib_dir = framework_path / "libraries" / name
manifest = lib_dir / "library.properties"
data = parse_library_properties(manifest) if manifest.is_file() else {}
return _library_info(name, lib_dir, {"name": name, **data})
def resolve_libraries(framework_path: Path) -> list[ArduinoLibrary]:
"""Resolve every ``cg.add_library()`` entry into an :class:`ArduinoLibrary`."""
bundled: list[ArduinoLibrary] = []
external: list[Library] = []
# PlatformIO's lib_ignore covers framework-bundled libraries too; the
# shared converter only filters the registry/git ones.
lib_ignore = lib_ignore_set()
for library in CORE.platformio_libraries.values():
if is_lib_ignored(library.name, lib_ignore):
continue
# A version pin means a registry package ("pngle@1.1.0"), never a
# framework-bundled library.
if (
library.repository
or library.version
or not library.name
or "/" in library.name
):
external.append(library)
elif (framework_path / "libraries" / library.name).is_dir():
bundled.append(_bundled_library(framework_path, library.name))
else:
# A bare registry name; resolved at the latest version, matching
# PlatformIO (a typo fails loudly as a registry lookup error).
external.append(library)
converted: list[ArduinoLibrary] = []
bundled_names = {lib.name for lib in bundled}
def _add_bundled_dependencies(component: ConvertedLibrary) -> None:
# A version-less bare-name dependency ("Hash" in ESPAsyncWebServer)
# is a core-bundled library; the shared converter skips it because
# it cannot be resolved from the registry.
for dep in normalize_dependencies(component.data.get("dependencies")):
name = dep.get("name")
if (
not name
or dep.get("owner")
or "version" in dep
or name in bundled_names
or is_lib_ignored(name, lib_ignore)
):
continue
if not (framework_path / "libraries" / name).is_dir():
# The shared converter skips version-less deps too, so this
# is the only place the drop can be made visible before the
# missing sources surface as link errors.
_LOGGER.warning(
"Dependency %s of library %s is not bundled with the "
"framework and has no version to resolve; skipping",
name,
component.name,
)
continue
try:
check_library_data(dep, ESP8266_PLATFORM, "arduino")
except InvalidLibrary as err:
# check_library_data's only raise is the platform filter, and
# rejecting another platform's dependency of a cross-platform
# manifest is routine (every ESPAsyncWebServer build hits it);
# a warning here would be noise, and the reason is in the log.
_LOGGER.debug("Skipping bundled dependency %s: %s", name, err)
continue
bundled_names.add(name)
bundled.append(_bundled_library(framework_path, name))
def _emit(component: ConvertedLibrary) -> None:
apply_extra_script(component, "esp8266", pio_platform=ESP8266_PLATFORM)
converted.append(
_library_info(
component.get_require_name(), component.source_dir, component.data
)
)
_add_bundled_dependencies(component)
if external:
convert_libraries(
external,
LibraryBackend(
platform=ESP8266_PLATFORM,
framework="arduino",
emit=_emit,
cache_key="arduino8266",
),
)
return bundled + converted
@@ -0,0 +1,63 @@
"""Tests for the ninja build-tool helper script."""
from __future__ import annotations
from pathlib import Path
from unittest.mock import MagicMock, patch
import pytest
from esphome.arduino8266 import build_tool
def test_ar_removes_stale_archive(tmp_path: Path) -> None:
archive = tmp_path / "lib.a"
archive.write_text("stale")
rsp = tmp_path / "lib.a.rsp"
rsp.write_text("a.o\n")
with (
patch.object(
build_tool.sys,
"argv",
["build_tool", "ar", "ar-bin", str(archive), str(rsp)],
),
patch.object(
build_tool.subprocess, "run", return_value=MagicMock(returncode=0)
) as mock_run,
):
assert build_tool.main() == 0
assert not archive.exists()
assert mock_run.call_args[0][0] == ["ar-bin", "rc", str(archive), f"@{rsp}"]
def test_copy(tmp_path: Path) -> None:
src = tmp_path / "firmware.bin"
src.write_text("data")
dst = tmp_path / "firmware.factory.bin"
with patch.object(
build_tool.sys, "argv", ["build_tool", "copy", str(src), str(dst)]
):
assert build_tool.main() == 0
assert dst.read_text() == "data"
def test_unknown_mode(capsys: pytest.CaptureFixture[str]) -> None:
with patch.object(build_tool.sys, "argv", ["build_tool", "bogus"]):
assert build_tool.main() == 1
assert "unknown build_tool mode" in capsys.readouterr().err
def test_runs_as_script(tmp_path: Path) -> None:
"""The ninja rules invoke the file as a plain script."""
import subprocess
import sys
src = tmp_path / "a.bin"
src.write_text("x")
dst = tmp_path / "b.bin"
result = subprocess.run(
[sys.executable, build_tool.__file__, "copy", str(src), str(dst)],
check=False,
)
assert result.returncode == 0
assert dst.read_text() == "x"
@@ -0,0 +1,310 @@
"""Tests for esphome.arduino8266.component (library resolution)."""
from __future__ import annotations
from pathlib import Path
from unittest.mock import patch
import pytest
from esphome.arduino8266 import component
from esphome.const import KEY_CORE, KEY_TARGET_PLATFORM, PLATFORM_ESP8266
from esphome.core import CORE, EsphomeError, Library
from esphome.platformio.library import ConvertedLibrary, LibraryBackend
@pytest.fixture(autouse=True)
def _reset_libraries() -> None:
CORE.platformio_libraries = {}
CORE.data[KEY_CORE] = {KEY_TARGET_PLATFORM: PLATFORM_ESP8266}
def _add_library(name: str, version: str | None, repository: str | None = None) -> None:
CORE.add_library(Library(name=name, version=version, repository=repository))
def _make_framework(tmp_path: Path) -> Path:
framework = tmp_path / "framework"
lib = framework / "libraries" / "ESP8266WiFi" / "src"
lib.mkdir(parents=True)
(lib / "ESP8266WiFi.cpp").write_text("")
(lib / "ESP8266WiFi.h").write_text("")
(lib.parent / "library.properties").write_text("name=ESP8266WiFi\nversion=1.0\n")
root_lib = framework / "libraries" / "Wire"
root_lib.mkdir(parents=True)
(root_lib / "Wire.cpp").write_text("")
(root_lib / "examples").mkdir()
(root_lib / "examples" / "scan.ino").write_text("")
return framework
def test_library_info_src_layout(tmp_path: Path) -> None:
framework = _make_framework(tmp_path)
lib = component._bundled_library(framework, "ESP8266WiFi")
assert lib.name == "ESP8266WiFi"
assert [p.name for p in lib.sources] == ["ESP8266WiFi.cpp"]
assert lib.include_dirs == [(framework / "libraries/ESP8266WiFi/src").resolve()]
def test_library_info_root_layout_excludes_examples(tmp_path: Path) -> None:
framework = _make_framework(tmp_path)
lib = component._bundled_library(framework, "Wire")
assert [p.name for p in lib.sources] == ["Wire.cpp"]
assert lib.include_dirs == [(framework / "libraries/Wire").resolve()]
def test_library_info_flags_parsing(tmp_path: Path) -> None:
read_path = tmp_path / "lib"
(read_path / "src").mkdir(parents=True)
(read_path / "src" / "a.cpp").write_text("")
(read_path / "inc").mkdir()
(read_path / "blobs").mkdir()
data = {
"build": {
"flags": [
"-DFOO=1 -I inc",
"-lalgobsec",
"-fno-lto",
"-Wl,--wrap=malloc",
"-l",
"m",
"-L",
"blobs",
],
}
}
lib = component._library_info("x", read_path, data)
assert lib.flags == ["-DFOO=1", "-fno-lto"]
assert lib.include_dirs == [
(read_path / "src").resolve(),
(read_path / "inc").resolve(),
]
assert lib.link_dirs == [(read_path / "blobs").resolve()]
assert lib.link_libs == ["algobsec", "m"]
assert lib.link_flags == ["-Wl,--wrap=malloc"]
def test_library_info_missing_link_dir_warns(
tmp_path: Path, caplog: pytest.LogCaptureFixture
) -> None:
read_path = tmp_path / "lib"
read_path.mkdir()
data = {"build": {"flags": ["-Lmissing_blobs"]}}
lib = component._library_info("x", read_path, data)
assert "declares library dir missing_blobs which does not exist" in caplog.text
# Kept anyway: the linker ignores missing -L dirs
assert lib.link_dirs == [(read_path / "missing_blobs").resolve()]
def test_library_info_declared_filter_matches_nothing_warns(
tmp_path: Path, caplog: pytest.LogCaptureFixture
) -> None:
read_path = tmp_path / "lib"
(read_path / "src").mkdir(parents=True)
data = {"build": {"srcFilter": ["+<nothing/*>"]}}
lib = component._library_info("x", read_path, data)
assert not lib.sources
assert "declares srcFilter/srcDir but no source files matched" in caplog.text
def test_library_info_header_only_does_not_warn(
tmp_path: Path, caplog: pytest.LogCaptureFixture
) -> None:
read_path = tmp_path / "lib"
(read_path / "src").mkdir(parents=True)
lib = component._library_info("x", read_path, {})
assert not lib.sources
assert "no source files matched" not in caplog.text
def test_library_info_no_src_dir(tmp_path: Path) -> None:
read_path = tmp_path / "empty"
read_path.mkdir()
lib = component._library_info("x", read_path, {})
# With no manifest hints the source dir falls back to the library root
assert lib.sources == []
assert lib.include_dirs == [read_path.resolve()]
def test_resolve_libraries_bundled(tmp_path: Path) -> None:
framework = _make_framework(tmp_path)
_add_library("ESP8266WiFi", None)
libs = component.resolve_libraries(framework)
assert [lib.name for lib in libs] == ["ESP8266WiFi"]
def test_resolve_libraries_bare_registry_name_is_external(tmp_path: Path) -> None:
"""A bare name that is not bundled resolves from the registry at the
latest version, matching PlatformIO and the documented libraries: key."""
framework = _make_framework(tmp_path)
_add_library("pngle", None)
with patch.object(component, "convert_libraries", return_value=[]) as mock_convert:
component.resolve_libraries(framework)
(libraries, _backend), _ = mock_convert.call_args
assert [lib.name for lib in libraries] == ["pngle"]
def _converted(name: str, source_dir: Path, data: dict) -> ConvertedLibrary:
converted = ConvertedLibrary(name, "1.0.0", source=None)
converted.path = source_dir
converted.data = data
return converted
def test_resolve_libraries_external_and_bundled_deps(tmp_path: Path) -> None:
framework = _make_framework(tmp_path)
_add_library("ESP32Async/ESPAsyncWebServer", "3.9.6")
lib_dir = tmp_path / "converted" / "webserver"
(lib_dir / "src").mkdir(parents=True)
(lib_dir / "src" / "server.cpp").write_text("")
converted = _converted(
"esp32async__ESPAsyncWebServer",
lib_dir,
{
"build": {},
"dependencies": [
# Version-less bundled dependency: resolved from the framework
{"name": "Wire", "platforms": "espressif8266"},
# Wrong platform: skipped
{"name": "ESP8266WiFi", "platforms": "espressif32"},
# Registry dependency with a version: handled by the converter
{"name": "ESPAsyncTCP", "owner": "ESP32Async", "version": "^2.0.0"},
# Not bundled: skipped
{"name": "NotBundled"},
],
},
)
def fake_convert(libraries: list, backend: LibraryBackend) -> list:
assert backend.platform == "espressif8266"
assert backend.framework == "arduino"
assert backend.cache_key == "arduino8266"
backend.emit(converted)
return [converted]
with (
patch.object(component, "convert_libraries", side_effect=fake_convert),
patch.object(component, "apply_extra_script") as mock_extra,
):
libs = component.resolve_libraries(framework)
mock_extra.assert_called_once_with(
converted, "esp8266", pio_platform="espressif8266"
)
assert [lib.name for lib in libs] == [
"Wire",
"esp32async__ESPAsyncWebServer",
]
def test_resolve_libraries_bundled_dep_already_present(tmp_path: Path) -> None:
framework = _make_framework(tmp_path)
_add_library("Wire", None)
_add_library("Some/External", "1.0.0")
lib_dir = tmp_path / "converted" / "external"
lib_dir.mkdir(parents=True)
converted = _converted(
"some__External", lib_dir, {"dependencies": [{"name": "Wire"}]}
)
def fake_convert(libraries: list, backend: LibraryBackend) -> list:
backend.emit(converted)
return [converted]
with (
patch.object(component, "convert_libraries", side_effect=fake_convert),
patch.object(component, "apply_extra_script"),
):
libs = component.resolve_libraries(framework)
# Wire appears once (from the explicit registration), not twice
assert [lib.name for lib in libs] == ["Wire", "some__External"]
def test_resolve_libraries_versioned_bare_name_is_external(tmp_path: Path) -> None:
"""A bare name with a version pin ("pngle@1.1.0") is a registry package,
not a bundled library, and must reach the converter."""
framework = _make_framework(tmp_path)
_add_library("pngle", "1.1.0")
with patch.object(component, "convert_libraries", return_value=[]) as mock_convert:
component.resolve_libraries(framework)
(libraries, _backend), _ = mock_convert.call_args
assert [lib.name for lib in libraries] == ["pngle"]
def test_library_info_trailing_bare_flag_warns(
tmp_path: Path, caplog: pytest.LogCaptureFixture
) -> None:
read_path = tmp_path / "lib"
(read_path / "src").mkdir(parents=True)
lib = component._library_info("x", read_path, {"build": {"flags": ["-DA=1 -l"]}})
assert lib.flags == ["-DA=1"]
assert lib.link_libs == []
assert "Ignoring trailing '-l'" in caplog.text
def test_library_info_missing_explicit_include_warns(
tmp_path: Path, caplog: pytest.LogCaptureFixture
) -> None:
read_path = tmp_path / "lib"
(read_path / "src").mkdir(parents=True)
lib = component._library_info("x", read_path, {"build": {"flags": ["-Inope"]}})
assert lib.include_dirs == [(read_path / "src").resolve()]
assert "include dir nope which does not exist" in caplog.text
def test_library_info_missing_declared_src_dir_raises(tmp_path: Path) -> None:
"""An explicitly declared srcDir that does not exist is a manifest error."""
read_path = tmp_path / "lib"
read_path.mkdir()
with pytest.raises(EsphomeError, match="srcDir nosrc which does not exist"):
component._library_info("x", read_path, {"build": {"srcDir": "nosrc"}})
def test_library_info_missing_declared_include_dir_warns(
tmp_path: Path, caplog: pytest.LogCaptureFixture
) -> None:
read_path = tmp_path / "lib"
read_path.mkdir()
component._library_info("x", read_path, {"build": {"includeDir": "noinc"}})
assert "include dir noinc which does not exist" in caplog.text
def test_resolve_libraries_lib_ignore_covers_bundled(tmp_path: Path) -> None:
"""lib_ignore applies to framework-bundled libraries, as under PlatformIO."""
framework = _make_framework(tmp_path)
_add_library("ESP8266WiFi", None)
_add_library("Wire", None)
CORE.platformio_options = {"lib_ignore": ["Wire"]}
libs = component.resolve_libraries(framework)
assert [lib.name for lib in libs] == ["ESP8266WiFi"]
def test_resolve_libraries_lib_ignore_covers_bundled_dependencies(
tmp_path: Path,
) -> None:
framework = _make_framework(tmp_path)
_add_library("Some/External", "1.0.0")
CORE.platformio_options = {"lib_ignore": ["Wire"]}
lib_dir = tmp_path / "converted" / "external"
lib_dir.mkdir(parents=True)
converted = _converted(
"some__External", lib_dir, {"dependencies": [{"name": "Wire"}]}
)
def fake_convert(libraries: list, backend: LibraryBackend) -> list:
backend.emit(converted)
return [converted]
with (
patch.object(component, "convert_libraries", side_effect=fake_convert),
patch.object(component, "apply_extra_script"),
):
libs = component.resolve_libraries(framework)
assert [lib.name for lib in libs] == ["some__External"]