Harden the shared library converter

Split out of #18662: the converter/manifest hardening without the
download parallelization, which stays there.

The PIO extra-script shim moves from espidf/ to platformio/ and gains
type checks for every manifest field, capture of the Append variants and
CPPPATH, warn-once diagnostics for unmodelled env reads, and uniform
shlex quoting so captured values survive the lex_build_flags round-trip.
The dependency walk normalizes every spelling PIO accepts, splits the
routine cross-platform skip (IncompatiblePlatform) from real manifest
problems, validates manifest shape once, and the shared flag lexer
warns and drops trailing or empty glued arguments.
This commit is contained in:
J. Nick Koston
2026-08-23 21:59:22 -05:00
committed by J. Nick Koston
parent 9974ad97fa
commit 0d75f597b3
11 changed files with 1294 additions and 411 deletions
+1
View File
@@ -1124,6 +1124,7 @@ def test_should_run_esp32_platformio_with_branch() -> None:
(["esphome/build_helpers/idedata.py"], True),
(["esphome/platformio/library.py"], True),
(["esphome/framework_helpers.py"], True),
(["esphome/platformio/extra_script.py"], True),
# PlatformIO build gen, its toolchain, and the esp32 component are
# NOT IDF-infra triggers
(["esphome/platformio/toolchain.py"], False),
+41 -132
View File
@@ -1,12 +1,12 @@
import glob
import hashlib
import json
import os
from pathlib import Path
from unittest.mock import MagicMock
import pytest
from esphome.components import esp32 as esp32_module
from esphome.const import (
KEY_CORE,
KEY_TARGET_FRAMEWORK,
@@ -16,6 +16,7 @@ from esphome.const import (
)
from esphome.core import CORE, Library
from esphome.espidf.component import (
_emit_idf_component,
generate_cmakelists_txt,
generate_idf_component_yml,
generate_idf_components,
@@ -26,11 +27,11 @@ from esphome.platformio.library import (
GitSource,
URLSource,
_node_key,
_normalize_dependencies,
_parse_library_json,
_parse_library_properties,
_resolve_registry_version,
collect_filtered_files,
normalize_dependencies,
parse_library_json,
parse_library_properties,
split_list_by_condition,
)
@@ -369,133 +370,11 @@ def test_generate_idf_component_yml_missing_path_raises(tmp_component):
generate_idf_component_yml(tmp_component)
def test_extra_script_captures_libpath_libs_and_defines(tmp_path):
from esphome.espidf.extra_script import captured_as_build_flags, run_extra_script
(tmp_path / "src" / "esp32").mkdir(parents=True)
script = tmp_path / "extra_script.py"
script.write_text(
"Import('env')\n"
"mcu = env.get('BOARD_MCU')\n"
"env.Append(\n"
" LIBPATH=[join('src', mcu)],\n"
" LIBS=['algobsec'],\n"
" CPPDEFINES=['FOO', ('BAR', '1')],\n"
" LINKFLAGS=['-Wl,--gc-sections'],\n"
")\n"
)
# The script uses bare ``join`` (PIO's extra-scripts run inside SCons
# where this is in scope). Inject it via the script header so the
# shim's exec namespace can resolve it.
script.write_text("from os.path import join\n" + script.read_text())
result = run_extra_script(script, library_dir=tmp_path, idf_target="esp32")
assert result.libpath == [str(Path("src") / "esp32")]
assert result.libs == ["algobsec"]
assert ("BAR", "1") in result.cppdefines
assert "FOO" in result.cppdefines
assert result.linkflags == ["-Wl,--gc-sections"]
flags = captured_as_build_flags(result, library_dir=tmp_path)
sep = os.sep
assert f"-Lsrc{sep}esp32" in flags
assert "-lalgobsec" in flags
assert "-DFOO" in flags
assert "-DBAR=1" in flags
assert "-Wl,--gc-sections" in flags
def test_extra_script_libpath_relative_resolves_against_library_dir(
tmp_path, monkeypatch
):
"""Relative LIBPATH entries must resolve against ``library_dir``, not the
caller's CWD (the shim restores CWD before ``captured_as_build_flags``
runs)."""
from esphome.espidf.extra_script import ExtraScriptResult, captured_as_build_flags
(tmp_path / "lib" / "esp32").mkdir(parents=True)
elsewhere = tmp_path.parent / "not_the_library_dir"
elsewhere.mkdir(exist_ok=True)
monkeypatch.chdir(elsewhere)
result = ExtraScriptResult(libpath=["lib/esp32"])
flags = captured_as_build_flags(result, library_dir=tmp_path)
sep = os.sep
assert flags == [f"-Llib{sep}esp32"]
def test_extra_script_libpath_absolute_outside_library_dir(tmp_path):
from esphome.espidf.extra_script import ExtraScriptResult, captured_as_build_flags
outside = tmp_path.parent / "system_lib"
outside.mkdir(exist_ok=True)
result = ExtraScriptResult(libpath=[str(outside)])
flags = captured_as_build_flags(result, library_dir=tmp_path)
assert flags == [f"-L{outside.resolve()}"]
def test_extra_script_failure_returns_empty_result(tmp_path, caplog):
from esphome.espidf.extra_script import run_extra_script
script = tmp_path / "broken.py"
script.write_text("raise RuntimeError('boom')\n")
with caplog.at_level("WARNING"):
result = run_extra_script(script, library_dir=tmp_path, idf_target="esp32")
assert result.libpath == []
assert result.libs == []
assert "broken.py" in caplog.text
def test_apply_extra_script_path_traversal_is_rejected(tmp_path):
from esphome.espidf.component import _apply_extra_script
library_dir = tmp_path / "lib"
library_dir.mkdir()
outside = tmp_path / "evil.py"
outside.write_text("env.Append(LIBS=['pwned'])\n")
c = IDFComponent("owner/name", "1.0", source=URLSource("http://dummy"))
c.path = library_dir
c.data = {"build": {"extraScript": "../evil.py"}}
_apply_extra_script(c)
# Nothing was folded into flags: the traversal was rejected before
# the script could run.
assert "flags" not in c.data["build"]
def test_apply_extra_script_merges_into_existing_flags(tmp_path, monkeypatch):
from esphome.components import esp32 as esp32_module
monkeypatch.setattr(esp32_module, "get_esp32_variant", lambda: "ESP32")
from esphome.espidf.component import _apply_extra_script
(tmp_path / "src").mkdir()
script = tmp_path / "extra.py"
script.write_text("env.Append(LIBS=['algobsec'])\n")
c = IDFComponent("owner/name", "1.0", source=URLSource("http://dummy"))
c.path = tmp_path
c.data = {"build": {"extraScript": "extra.py", "flags": ["-DEXISTING"]}}
_apply_extra_script(c)
assert "-DEXISTING" in c.data["build"]["flags"]
assert "-lalgobsec" in c.data["build"]["flags"]
def test_parse_library_json(tmp_path):
f = tmp_path / "library.json"
f.write_text(json.dumps({"name": "test"}))
result = _parse_library_json(f)
result = parse_library_json(f)
assert result["name"] == "test"
@@ -510,7 +389,7 @@ empty=
"""
)
result = _parse_library_properties(f)
result = parse_library_properties(f)
assert result["name"] == "Test"
assert result["version"] == "1.0"
@@ -680,22 +559,22 @@ def test_node_key_registry_bare_name():
def test_normalize_dependencies_none():
assert _normalize_dependencies(None) == []
assert normalize_dependencies(None) == []
def test_normalize_dependencies_list_form():
deps = [{"name": "foo", "version": "1.0"}]
assert _normalize_dependencies(deps) == [{"name": "foo", "version": "1.0"}]
assert normalize_dependencies(deps) == [{"name": "foo", "version": "1.0"}]
def test_normalize_dependencies_dict_form():
out = _normalize_dependencies({"nanopb/Nanopb": "^0.4.91", "BareName": "1.2.3"})
out = normalize_dependencies({"nanopb/Nanopb": "^0.4.91", "BareName": "1.2.3"})
assert {"name": "Nanopb", "owner": "nanopb", "version": "^0.4.91"} in out
assert {"name": "BareName", "owner": None, "version": "1.2.3"} in out
def test_normalize_dependencies_dict_form_nested_spec():
out = _normalize_dependencies(
out = normalize_dependencies(
{"nanopb/Nanopb": {"version": "^0.4.91", "platforms": "espidf"}}
)
assert out == [
@@ -1190,3 +1069,33 @@ def test_idf_component_download_passes_salt() -> None:
"owner/name", force=True, salt="abcd1234", namespace="idf"
)
assert c.path == Path("/converted/owner/name")
def test_emit_idf_component_wires_esp32_target(tmp_path, monkeypatch):
"""Emitting a component resolves the esp32 variant into the shared
extraScript helper."""
monkeypatch.setattr(esp32_module, "get_esp32_variant", lambda: "ESP32")
(tmp_path / "src").mkdir()
script = tmp_path / "extra.py"
script.write_text("env.Append(LIBS=[env.get('BOARD_MCU')])\n")
c = IDFComponent("owner/name", "1.0", source=URLSource("http://dummy"))
c.path = tmp_path
c.data = {"build": {"extraScript": "extra.py"}}
_emit_idf_component(c)
assert c.data["build"]["flags"] == ["-lesp32"]
def test_build_flags_dangling_flag_does_not_cross_entries(
tmp_path, caplog: pytest.LogCaptureFixture
) -> None:
"""Each entry is lexed independently, as ParseFlags does: a dangling -I ending one
entry warns instead of absorbing the next entry's first token."""
(tmp_path / "src").mkdir()
c = IDFComponent("owner/name", "1.0", source=URLSource("http://dummy"))
c.path = tmp_path
c.data = {"build": {"flags": ["-Wall -I", "-DFOO=1"]}}
content = generate_cmakelists_txt(c)
assert "FOO=1" in content
assert "-I-DFOO" not in content
assert "Ignoring trailing '-I'" in caplog.text
@@ -0,0 +1,478 @@
"""Tests for the shared extraScript machinery (platformio.extra_script)."""
from __future__ import annotations
import logging
import os
from pathlib import Path
from unittest.mock import patch
import pytest
from esphome.core import EsphomeError
from esphome.platformio.extra_script import (
ExtraScriptResult,
_FakeSConsEnv,
apply_extra_script,
captured_as_build_flags,
run_extra_script,
)
from esphome.platformio.library import (
ConvertedLibrary as IDFComponent,
URLSource,
lex_build_flags,
)
def test_extra_script_captures_libpath_libs_and_defines(tmp_path):
(tmp_path / "src" / "esp32").mkdir(parents=True)
script = tmp_path / "extra_script.py"
script.write_text(
"Import('env')\n"
"mcu = env.get('BOARD_MCU')\n"
"env.Append(\n"
" LIBPATH=[join('src', mcu)],\n"
" LIBS=['algobsec'],\n"
" CPPDEFINES=['FOO', ('BAR', '1')],\n"
" LINKFLAGS=['-Wl,--gc-sections'],\n"
")\n"
)
# The script uses bare ``join`` (PIO's extra-scripts run inside SCons
# where this is in scope). Inject it via the script header so the
# shim's exec namespace can resolve it.
script.write_text("from os.path import join\n" + script.read_text())
result = run_extra_script(
script, library_dir=tmp_path, board_mcu="esp32", pio_platform="espressif32"
)
assert result.libpath == [str(Path("src") / "esp32")]
assert result.libs == ["algobsec"]
assert ("BAR", "1") in result.cppdefines
assert "FOO" in result.cppdefines
assert result.linkflags == ["-Wl,--gc-sections"]
# Lex like the consumer does: quoting makes raw strings platform-varying
tokens = lex_build_flags(
captured_as_build_flags(result, library_dir=tmp_path), "test"
)
sep = os.sep
assert f"-Lsrc{sep}esp32" in tokens
assert "-lalgobsec" in tokens
assert "-DFOO" in tokens
assert "-DBAR=1" in tokens
assert "-Wl,--gc-sections" in tokens
def test_extra_script_libpath_relative_resolves_against_library_dir(
tmp_path, monkeypatch
):
"""Relative LIBPATH entries must resolve against ``library_dir``, not the
caller's CWD (the shim restores CWD before ``captured_as_build_flags``
runs)."""
(tmp_path / "lib" / "esp32").mkdir(parents=True)
elsewhere = tmp_path.parent / "not_the_library_dir"
elsewhere.mkdir(exist_ok=True)
monkeypatch.chdir(elsewhere)
result = ExtraScriptResult(libpath=["lib/esp32"])
flags = captured_as_build_flags(result, library_dir=tmp_path)
sep = os.sep
assert lex_build_flags(flags, "test") == [f"-Llib{sep}esp32"]
def test_extra_script_libpath_absolute_outside_library_dir(tmp_path):
outside = tmp_path.parent / "system_lib"
outside.mkdir(exist_ok=True)
result = ExtraScriptResult(libpath=[str(outside)])
flags = captured_as_build_flags(result, library_dir=tmp_path)
assert lex_build_flags(flags, "test") == [f"-L{outside.resolve()}"]
def test_extra_script_failure_returns_empty_result(tmp_path, caplog):
script = tmp_path / "broken.py"
script.write_text("raise RuntimeError('boom')\n")
with caplog.at_level("WARNING"):
result = run_extra_script(
script, library_dir=tmp_path, board_mcu="esp32", pio_platform="espressif32"
)
assert result.libpath == []
assert result.libs == []
assert "broken.py" in caplog.text
def test_apply_extra_script_path_traversal_is_rejected(tmp_path):
library_dir = tmp_path / "lib"
library_dir.mkdir()
outside = tmp_path / "evil.py"
outside.write_text("env.Append(LIBS=['pwned'])\n")
c = IDFComponent("owner/name", "1.0", source=URLSource("http://dummy"))
c.path = library_dir
c.data = {"build": {"extraScript": "../evil.py"}}
with pytest.raises(EsphomeError, match="escapes the library directory"):
apply_extra_script(c, board_mcu=lambda: "esp32", pio_platform="espressif32")
# Nothing was folded into flags: the traversal was rejected before
# the script could run.
assert "flags" not in c.data["build"]
def test_apply_extra_script_merges_into_existing_flags(tmp_path):
(tmp_path / "src").mkdir()
script = tmp_path / "extra.py"
script.write_text("env.Append(LIBS=['algobsec'])\n")
c = IDFComponent("owner/name", "1.0", source=URLSource("http://dummy"))
c.path = tmp_path
c.data = {"build": {"extraScript": "extra.py", "flags": ["-DEXISTING"]}}
apply_extra_script(c, board_mcu=lambda: "esp32", pio_platform="espressif32")
assert "-DEXISTING" in c.data["build"]["flags"]
assert "-lalgobsec" in c.data["build"]["flags"]
def test_apply_extra_script_malformed_flags_raises(tmp_path) -> None:
"""A null/dict build.flags fails naming the library instead of injecting
a non-string into the compiler command line."""
(tmp_path / "src").mkdir()
script = tmp_path / "extra.py"
script.write_text("env.Append(LIBS=['algobsec'])\n")
c = IDFComponent("owner/name", "1.0", source=URLSource("http://dummy"))
c.path = tmp_path
c.data = {"build": {"extraScript": "extra.py", "flags": None}}
with pytest.raises(EsphomeError, match="malformed build.flags"):
apply_extra_script(c, board_mcu=lambda: "esp32", pio_platform="espressif32")
def test_apply_extra_script_callable_target_and_str_flags(tmp_path) -> None:
"""The shared helper resolves the board_mcu callable lazily and normalizes
a string ``build.flags`` value into a list before extending it."""
(tmp_path / "src").mkdir()
script = tmp_path / "extra.py"
script.write_text("env.Append(LIBS=[env.get('BOARD_MCU')])\n")
c = IDFComponent("owner/name", "1.0", source=URLSource("http://dummy"))
c.path = tmp_path
c.data = {"build": {"extraScript": "extra.py", "flags": "-DBASE=1"}}
apply_extra_script(c, board_mcu=lambda: "esp8266", pio_platform="espressif8266")
assert c.data["build"]["flags"] == ["-DBASE=1", "-lesp8266"]
def test_captured_nonstring_buckets_warn_and_skip(tmp_path, caplog) -> None:
"""Non-string LIBS/LINKFLAGS/CPPFLAGS/LIBPATH entries (legal SCons
nodes) are skipped by name instead of stringified into garbage flags."""
(tmp_path / "src").mkdir()
script = tmp_path / "extra.py"
script.write_text(
"env.Append(LIBS=['m', 42], LINKFLAGS=['-Wl,-x', {'no': 1}], "
"CPPFLAGS=['-Os', 3.5], LIBPATH=['libs', 7])\n"
)
(tmp_path / "libs").mkdir()
c = IDFComponent("owner/name", "1.0", source=URLSource("http://dummy"))
c.path = tmp_path
c.data = {"build": {"extraScript": "extra.py"}}
apply_extra_script(c, board_mcu=lambda: "esp8266", pio_platform="espressif8266")
flags = c.data["build"]["flags"]
assert "-lm" in flags and "-Wl,-x" in flags and "-Os" in flags
assert not any("42" in f or "no" in f or "3.5" in f for f in flags)
assert "Ignoring unsupported LIBS entry 42" in caplog.text
assert "Ignoring unsupported LIBPATH entry 7" in caplog.text
def test_captured_dict_cppdefines_warn_and_skip(tmp_path, caplog) -> None:
"""A dict CPPDEFINES entry (legal SCons) must warn and skip; formatting
it blind would hand the compiler -D{'FOO': '1'} garbage."""
(tmp_path / "src").mkdir()
script = tmp_path / "extra.py"
script.write_text(
"env.Append(CPPDEFINES=[{'FOO': '1'}, ('BAR', 2), ['BAZ', 3], 'PLAIN'])\n"
)
c = IDFComponent("owner/name", "1.0", source=URLSource("http://dummy"))
c.path = tmp_path
c.data = {"build": {"extraScript": "extra.py"}}
apply_extra_script(c, board_mcu=lambda: "esp8266", pio_platform="espressif8266")
assert c.data["build"]["flags"] == ["-DBAR=2", "-DBAZ=3", "-DPLAIN"]
assert "Ignoring unsupported CPPDEFINES entry" in caplog.text
def test_apply_extra_script_subscript_env_read(tmp_path) -> None:
"""Scripts also read env["BOARD_MCU"]; the subscript form must work or
the broad handler discards every flag the script captured."""
(tmp_path / "src").mkdir()
script = tmp_path / "extra.py"
script.write_text("env.Append(LIBS=[env['BOARD_MCU']])\n")
c = IDFComponent("owner/name", "1.0", source=URLSource("http://dummy"))
c.path = tmp_path
c.data = {"build": {"extraScript": "extra.py"}}
apply_extra_script(c, board_mcu=lambda: "esp8266", pio_platform="espressif8266")
assert c.data["build"]["flags"] == ["-lesp8266"]
def test_apply_extra_script_no_script_and_no_flags(tmp_path) -> None:
# No extraScript declared: nothing happens, the target is never resolved
c = IDFComponent("owner/name", "1.0", source=URLSource("http://dummy"))
c.path = tmp_path
c.data = {"build": {}}
apply_extra_script(
c,
board_mcu=lambda: pytest.fail("target resolved without a script"),
pio_platform="espressif8266",
)
# A script that captures nothing leaves the flags untouched
script = tmp_path / "noop.py"
script.write_text("pass\n")
c.data = {"build": {"extraScript": "noop.py"}}
apply_extra_script(c, board_mcu=lambda: "esp8266", pio_platform="espressif8266")
assert "flags" not in c.data["build"]
def test_apply_extra_script_ignores_uncaptured_env_calls(tmp_path, caplog) -> None:
"""Un-captured env vars and unsupported env methods are skipped but
diagnosable from the build log."""
caplog.set_level(logging.DEBUG)
script = tmp_path / "extra.py"
script.write_text(
"env.Replace(CC='clang')\nenv.Append(UNCAPTURED=['x'], LIBS='single')\n"
)
c = IDFComponent("owner/name", "1.0", source=URLSource("http://dummy"))
c.path = tmp_path
c.data = {"build": {"extraScript": "extra.py"}}
apply_extra_script(c, board_mcu=lambda: "esp8266", pio_platform="espressif8266")
assert c.data["build"]["flags"] == ["-lsingle"]
assert "env.Append(UNCAPTURED=...) is not captured" in caplog.text
assert "env.Replace(...) is not supported" in caplog.text
def test_apply_extra_script_swallows_script_errors(tmp_path, caplog) -> None:
"""A raising extra-script is best-effort: logged and skipped."""
script = tmp_path / "extra.py"
script.write_text("raise RuntimeError('boom')\n")
c = IDFComponent("owner/name", "1.0", source=URLSource("http://dummy"))
c.path = tmp_path
c.data = {"build": {"extraScript": "extra.py"}}
apply_extra_script(c, board_mcu=lambda: "esp8266", pio_platform="espressif8266")
assert "flags" not in c.data["build"]
assert "ignoring its output" in caplog.text
def test_apply_extra_script_pio_platform(tmp_path) -> None:
"""The backend's platform token is exposed to the script as PIOPLATFORM."""
script = tmp_path / "extra.py"
script.write_text("env.Append(LIBS=[env.get('PIOPLATFORM')])\n")
c = IDFComponent("owner/name", "1.0", source=URLSource("http://dummy"))
c.path = tmp_path
c.data = {"build": {"extraScript": "extra.py"}}
apply_extra_script(c, board_mcu=lambda: "esp8266", pio_platform="espressif8266")
assert c.data["build"]["flags"] == ["-lespressif8266"]
def test_apply_extra_script_missing_script_raises(tmp_path) -> None:
"""A declared but absent extraScript is a broken package and fails by
name, as it would under PlatformIO."""
c = IDFComponent("owner/name", "1.0", source=URLSource("http://dummy"))
c.path = tmp_path
c.data = {"build": {"extraScript": "nope.py"}}
with pytest.raises(EsphomeError, match="nope.py of library owner/name not found"):
apply_extra_script(c, board_mcu=lambda: "esp8266", pio_platform="espressif8266")
@pytest.mark.parametrize("bad", (["a.py"], {"esp32": "a.py"}), ids=("list", "dict"))
def test_apply_extra_script_non_string_raises(tmp_path, bad) -> None:
"""A non-string extraScript fails naming the library, not with a TypeError."""
c = IDFComponent("owner/name", "1.0", source=URLSource("http://dummy"))
c.path = tmp_path
c.data = {"build": {"extraScript": bad}}
with pytest.raises(EsphomeError, match="of library owner/name must be a string"):
apply_extra_script(c, board_mcu=lambda: "esp8266", pio_platform="espressif8266")
def test_extra_script_cpppath_captured_as_include_flags(tmp_path, monkeypatch):
"""CPPPATH entries translate to -I flags anchored like LIBPATH."""
(tmp_path / "include").mkdir()
outside = tmp_path.parent / "system_inc"
outside.mkdir(exist_ok=True)
elsewhere = tmp_path.parent / "not_the_library_dir"
elsewhere.mkdir(exist_ok=True)
monkeypatch.chdir(elsewhere)
result = ExtraScriptResult(cpppath=["include", str(outside), 7])
flags = captured_as_build_flags(result, library_dir=tmp_path)
assert lex_build_flags(flags, "test") == ["-Iinclude", f"-I{outside.resolve()}"]
def test_extra_script_spaced_paths_survive_relexing(tmp_path):
"""-I/-L paths with spaces round-trip through lex_build_flags as one token."""
(tmp_path / "my libs").mkdir()
result = ExtraScriptResult(cpppath=["my libs"], libpath=["my libs"])
flags = captured_as_build_flags(result, library_dir=tmp_path)
assert lex_build_flags(flags, "test") == ["-Imy libs", "-Lmy libs"]
def test_run_extra_script_failure_discards_partial_capture(tmp_path, caplog) -> None:
"""A crashed script yields an empty result: half-applied flags could
build wrong-output firmware that links cleanly."""
script = tmp_path / "extra.py"
script.write_text("env.Append(LIBS=['algobsec'])\nraise RuntimeError('boom')\n")
result = run_extra_script(
script, library_dir=tmp_path, board_mcu="esp32", pio_platform="espressif32"
)
assert result.libs == []
assert "ignoring its output" in caplog.text
def test_run_extra_script_syntax_error_is_best_effort(tmp_path, caplog) -> None:
"""A vendored script that does not even compile warns and skips instead
of aborting the build."""
script = tmp_path / "extra.py"
script.write_text("def broken(:\n")
result = run_extra_script(
script, library_dir=tmp_path, board_mcu="esp32", pio_platform="espressif32"
)
assert result.libs == []
assert "ignoring its output" in caplog.text
def test_unsupported_env_method_warns_once(caplog) -> None:
"""Repeated calls to the same unsupported method warn only once."""
env = _FakeSConsEnv(
board_mcu="esp8266", pio_env="esphome_esp8266", pio_platform="espressif8266"
)
env.Replace(CC="clang")
env.Replace(CC="gcc")
assert caplog.text.count("env.Replace(...) is not supported") == 1
def test_run_extra_script_sys_exit_is_best_effort(tmp_path, caplog) -> None:
"""A nonzero sys.exit() in a vendored script must not kill the esphome
run, and its output is discarded."""
script = tmp_path / "extra.py"
script.write_text("import sys\nenv.Append(LIBS=['x'])\nsys.exit(3)\n")
result = run_extra_script(
script, library_dir=tmp_path, board_mcu="esp32", pio_platform="espressif32"
)
assert result.libs == []
assert "exited with status 3" in caplog.text
def test_run_extra_script_sys_exit_zero_is_success(tmp_path, caplog) -> None:
"""sys.exit(0) is a normal PlatformIO script ending: the capture is kept."""
script = tmp_path / "extra.py"
script.write_text("import sys\nenv.Append(LIBS=['algobsec'])\nsys.exit(0)\n")
result = run_extra_script(
script, library_dir=tmp_path, board_mcu="esp32", pio_platform="espressif32"
)
assert result.libs == ["algobsec"]
assert "ignoring its output" not in caplog.text
def test_run_extra_script_unreadable_raises(tmp_path) -> None:
"""An unreadable declared script is a broken package, like a missing one."""
script = tmp_path / "extra.py"
script.write_text("")
with (
patch("pathlib.Path.read_text", side_effect=OSError("denied")),
pytest.raises(EsphomeError, match="is unreadable"),
):
run_extra_script(
script, library_dir=tmp_path, board_mcu="esp32", pio_platform="espressif32"
)
def test_run_extra_script_bad_encoding_is_best_effort(tmp_path, caplog) -> None:
"""Undecodable content warns and skips, like a SyntaxError."""
script = tmp_path / "extra.py"
script.write_bytes(b"\xff\xfe\x00bad")
result = run_extra_script(
script, library_dir=tmp_path, board_mcu="esp32", pio_platform="espressif32"
)
assert result.libs == []
assert "is not UTF-8" in caplog.text
@pytest.mark.parametrize("method", ("Prepend", "AppendUnique", "PrependUnique"))
def test_append_variants_capture_like_append(method: str) -> None:
"""Prepend/AppendUnique/PrependUnique write the captured keys too."""
env = _FakeSConsEnv(
board_mcu="esp8266", pio_env="esphome_esp8266", pio_platform="espressif8266"
)
getattr(env, method)(LIBS=["algobsec"], LIBPATH=["lib"])
assert env.result.libs == ["algobsec"]
assert env.result.libpath == ["lib"]
def test_env_get_unknown_key_warns_once(caplog) -> None:
"""A script branching on an unmodelled env var is diagnosable."""
env = _FakeSConsEnv(
board_mcu="esp8266", pio_env="esphome_esp8266", pio_platform="espressif8266"
)
assert env.get("BOARD") is None
assert env.get("BOARD", "d1") == "d1"
assert env.get("BOARD_MCU") == "esp8266"
assert caplog.text.count("env.get('BOARD') is not modelled") == 1
assert "BOARD_MCU" not in caplog.text
def test_spaced_linkflag_survives_relexing(tmp_path) -> None:
"""A captured argv token with a space stays one token after lexing."""
result = ExtraScriptResult(
linkflags=["-Wl,-T my linker.ld"], cppflags=["-include my hdr.h"]
)
flags = captured_as_build_flags(result, library_dir=tmp_path)
assert lex_build_flags(flags, "test") == [
"-Wl,-T my linker.ld",
"-include my hdr.h",
]
def test_uncaptured_append_key_warns_once(caplog) -> None:
"""A loop of Appends to the same uncaptured key warns once."""
env = _FakeSConsEnv(
board_mcu="esp8266", pio_env="esphome_esp8266", pio_platform="espressif8266"
)
env.Append(RANLIBFLAGS=["a"])
env.Append(RANLIBFLAGS=["b"])
assert caplog.text.count("env.Append(RANLIBFLAGS=...) is not captured") == 1
+209 -3
View File
@@ -4,6 +4,7 @@ Covers the shared download/parse/resolve/dependency-walk paths in
``esphome.platformio.library`` directly (the ESP-IDF and Zephyr backends are
exercised in their own test modules)."""
from contextlib import contextmanager
import json
import logging
from pathlib import Path
@@ -13,6 +14,7 @@ import pytest
from esphome.core import EsphomeError, Library
import esphome.platformio.library as lib
from esphome.platformio.library import (
SOURCE_KIND_FOR_SUFFIX,
ConvertedLibrary,
GitSource,
InvalidLibrary,
@@ -23,6 +25,8 @@ from esphome.platformio.library import (
_resolve_registry_version,
check_library_data,
convert_libraries,
join_flag_args,
split_flag_entry,
)
@@ -150,11 +154,30 @@ def test_localsource_download_returns_empty_build_dir(setup_core: Path) -> None:
assert plain != out
@contextmanager
def caplog_at_info():
records: list[logging.LogRecord] = []
handler = logging.Handler()
handler.emit = records.append
logger = logging.getLogger("esphome.platformio.library")
logger.addHandler(handler)
# The level must actually admit INFO or the no-INFO assertions are vacuous
old_level = logger.level
logger.setLevel(logging.INFO)
try:
yield records
finally:
logger.setLevel(old_level)
logger.removeHandler(handler)
def test_urlsource_download_extracts_then_reuses_marker(setup_core, monkeypatch):
monkeypatch.setattr(lib, "rmdir", lambda path, msg="": None)
dl_calls: list[list[str]] = []
monkeypatch.setattr(
lib, "download_from_mirrors", lambda urls, headers, f: dl_calls.append(urls)
lib,
"download_from_mirrors",
lambda urls, headers, f: dl_calls.append(urls),
)
def fake_extract(fileobj, path):
@@ -213,7 +236,7 @@ def _patch_registry_resolve(monkeypatch: pytest.MonkeyPatch) -> None:
def _patch_download_with_manifests(monkeypatch, tmp_path, manifests, *, properties=()):
"""Fake ConvertedLibrary.download to materialize canned manifests on disk."""
def fake_download(self, force=False, salt="", namespace=""):
def fake_download(self, force=False, salt="", namespace="", progress=None):
self.path = tmp_path / self.get_require_name()
self.path.mkdir(parents=True, exist_ok=True)
if self.name in properties:
@@ -292,7 +315,11 @@ def _patch_download_without_manifest(
calls: list[bool] = []
def fake_download(
self: ConvertedLibrary, force: bool = False, salt: str = "", namespace: str = ""
self: ConvertedLibrary,
force: bool = False,
salt: str = "",
namespace: str = "",
progress=None,
) -> None:
calls.append(force)
self.path = tmp_path / self.get_require_name()
@@ -531,3 +558,182 @@ def test_convert_libraries_skips_incompatible_dependency(tmp_path, monkeypatch):
top = convert_libraries([Library("esphome/A", "1.0.0", None)], _backend())
assert top[0].dependencies == []
def test_split_flag_entry_unbalanced_quote_is_clean() -> None:
"""A malformed flags entry raises EsphomeError, not a raw ValueError."""
assert split_flag_entry('-DX="a b"', "library x") == ["-DX=a b"]
with pytest.raises(EsphomeError, match=r"Malformed build flag.*library x"):
split_flag_entry('-DX="unclosed', "library x")
def test_join_flag_args_reglues_spaced_define() -> None:
"""A spaced -D re-glues to its argument, as ParseFlags does."""
assert join_flag_args(["-D", "FOO=1", "-Os"], "x") == ["-DFOO=1", "-Os"]
def test_join_flag_args_trailing_bare_flag_warns(
caplog: pytest.LogCaptureFixture,
) -> None:
assert join_flag_args(["-Os", "-l"], "library x") == ["-Os"]
assert "Ignoring trailing '-l'" in caplog.text
def test_lex_build_flags_dangling_flag_does_not_cross_entries(
caplog: pytest.LogCaptureFixture,
) -> None:
"""Each entry is lexed independently, as ParseFlags does: a dangling -I
ending one entry warns instead of absorbing the next entry's first token."""
from esphome.platformio.library import lex_build_flags
assert lex_build_flags(["-Wall -I", "-DFOO=1"], "lib x") == ["-Wall", "-DFOO=1"]
assert "Ignoring trailing '-I'" in caplog.text
def test_join_flag_args_empty_argument_warns_and_drops(
caplog: pytest.LogCaptureFixture,
) -> None:
"""An empty glued argument is dropped: a bare -D would eat the next flag."""
assert lib.lex_build_flags('-D "" -DFOO', "build_flags") == ["-DFOO"]
assert "Ignoring '-D' with empty argument in build_flags" in caplog.text
def test_normalize_dependencies_forms(caplog) -> None:
"""Every PIO-legal spelling normalizes; unrecognizable entries warn."""
from esphome.platformio.library import normalize_dependencies
assert normalize_dependencies(
["Wire", {"name": "SPI"}, 5, "", {"version": "1.0"}], "libx"
) == [
{"name": "Wire"},
{"name": "SPI"},
]
# The int, the empty string, and the nameless dict all warn
assert caplog.text.count("unrecognized dependency entry") == 3
# A plain string is names, never iterated into characters
assert normalize_dependencies("Wire, SPI") == [
{"name": "Wire"},
{"name": "SPI"},
]
assert normalize_dependencies("Wire") == [{"name": "Wire"}]
# A non-iterable value fails by manifest name, never a bare TypeError
assert normalize_dependencies(5, "libx") == []
assert "Ignoring unrecognized dependencies 5 of libx" in caplog.text
# The dict-shorthand form validates names like the list form: an empty
# key and a spec overriding name with a non-string both warn and drop
assert normalize_dependencies(
{"": "1.0", "Wire": {"name": 123, "version": "1.0"}, "SPI": "*"}, "libx"
) == [{"name": "SPI", "owner": None, "version": "*"}]
assert caplog.text.count("unrecognized dependency entry") == 5
# A container or numeric version would raise from set.add() or fail
# opaquely in the registry; both spellings warn and drop
assert normalize_dependencies({"Foo": ["1.0", "2.0"]}, "libx") == []
assert normalize_dependencies([{"name": "Foo", "version": 1}], "libx") == []
assert caplog.text.count("unrecognized dependency entry") == 7
@pytest.mark.parametrize(
"manifest", [["not", "a", "manifest"], {"name": "A", "build": "src"}]
)
def test_convert_libraries_malformed_manifest_raises(
tmp_path, monkeypatch, manifest
) -> None:
"""A manifest without the expected dict shape fails by library name
before any backend dereferences data/build."""
_patch_download_with_manifests(monkeypatch, tmp_path, {"esphome/A": manifest})
with pytest.raises(EsphomeError, match="has a malformed manifest"):
convert_libraries([Library("esphome/A", None, None)], _backend())
def test_walk_warns_for_properties_only_depends(
tmp_path, monkeypatch, caplog: pytest.LogCaptureFixture
) -> None:
"""A manifest declaring dependencies only as library.properties depends=
warns in the shared walk, so every backend reports the drop."""
_patch_download_with_manifests(
monkeypatch,
tmp_path,
{"esphome/A": "name=A\nversion=1.0\ndepends=Wire, SPI\n"},
properties=("esphome/A",),
)
caplog.set_level("INFO")
convert_libraries([Library("esphome/A", "1.0.0", None)], _backend())
assert "declares dependencies via library.properties" in caplog.text
def test_walk_warns_for_nonplatform_invalid_library(
tmp_path, monkeypatch, caplog: pytest.LogCaptureFixture
) -> None:
"""A dependency dropped for any cause other than the routine platform
filter is visible in every backend."""
_patch_download_with_manifests(
monkeypatch,
tmp_path,
{"esphome/A": {"name": "A", "dependencies": [{"name": "B", "version": "1.0"}]}},
)
calls = {"n": 0}
real = lib.check_library_data
def flaky(data, platform, framework):
calls["n"] += 1
if calls["n"] > 1:
raise InvalidLibrary("manifest is corrupt")
return real(data, platform, framework)
monkeypatch.setattr(lib, "check_library_data", flaky)
convert_libraries([Library("esphome/A", None, None)], _backend())
assert "Skipping dependency B of esphome/A: manifest is corrupt" in caplog.text
def test_convert_libraries_warns_for_nonplatform_invalid_dependency_component(
tmp_path, monkeypatch, caplog: pytest.LogCaptureFixture
) -> None:
"""A dependency component dropped for any cause other than the platform
filter warns; only the routine cross-platform skip stays at debug."""
_patch_download_with_manifests(
monkeypatch,
tmp_path,
{
"esphome/A": {
"name": "A",
"dependencies": [{"name": "C", "owner": "esphome", "version": "1.0"}],
},
"esphome/C": {"name": "C"},
},
)
real = lib.check_library_data
def flaky(data, platform, framework):
# Fail only on C's resolved manifest, not on A's dependency entry
if data.get("name") == "C" and "version" not in data:
raise InvalidLibrary("manifest is corrupt")
return real(data, platform, framework)
monkeypatch.setattr(lib, "check_library_data", flaky)
convert_libraries([Library("esphome/A", "1.0.0", None)], _backend())
assert "manifest is corrupt" in caplog.text
assert "Skipping dependency" in caplog.text
def test_split_flag_entry_non_string_is_clean() -> None:
"""A dict or number from a third-party manifest fails naming the entry,
not with an opaque shlex traceback."""
with pytest.raises(EsphomeError, match="Malformed build flag"):
split_flag_entry({"esp32": ["-DX"]}, "lib x")
with pytest.raises(EsphomeError, match="Malformed build flag 5"):
split_flag_entry(5, "lib x")
def test_source_kind_map_shape() -> None:
"""The kind values the native compile rules key on, and the deliberate
AS/ASPP merge (.s and .S both map to asm)."""
assert set(SOURCE_KIND_FOR_SUFFIX.values()) == {"c", "cxx", "asm"}
assert SOURCE_KIND_FOR_SUFFIX[".s"] == "asm"
assert SOURCE_KIND_FOR_SUFFIX[".S"] == "asm"
assert SOURCE_KIND_FOR_SUFFIX[".c"] == "c"
assert SOURCE_KIND_FOR_SUFFIX[".cpp"] == "cxx"
@@ -1977,3 +1977,10 @@ def test_run_platformio_cli_invokes_heal(
with patch.object(toolchain, "heal_platformio_python_env") as mock_heal:
toolchain.run_platformio_cli("test")
mock_heal.assert_called_once()
def test_ccache_probe_spawns_with_close_fds_false() -> None:
"""The probe follows the repo-wide posix_spawn convention."""
with patch("subprocess.run") as mock_run:
assert toolchain._ccache_runs("/usr/bin/ccache") is True
assert mock_run.call_args.kwargs["close_fds"] is False