github/esphome
2
0
Fork 0
mirror of https://github.com/esphome/esphome.git synced 2026-08-27 08:28:30 +00:00
Code Issues Packages Projects Releases Wiki Activity

Decode the linker script strictly, fold the diagnostic note into the cache stamp, fix the comment attachment

Browse Source
This commit is contained in:
J. Nick Koston
2026-08-23 19:35:42 -05:00
parent 14a68e7b35
commit 983c5a1de9
2 changed files with 70 additions and 40 deletions
Download Patch File Download Diff File Expand all files Collapse all files
esphome/build_gen/arduino8266.py
+41 -17
View File
@@ -522,10 +522,9 @@ def _project_flags(
return compile_flags, link_flags, lib_dirs, libs
# Plain-form linker flags rejected by _project_flags: inert on a -c compile
# line, so the firmware would silently lack the requested link behavior
# Best-effort, not exhaustive: an unlisted link-only spelling still falls
# through to the compile line, with a warning from the shape check
# Recognized compile-flag shapes: the allow-list feeding the fall-through
# warning in _project_flags (an unlisted link-only spelling still reaches
# the compile line, but not silently)
_COMPILE_FLAG_PREFIXES = (
"-D",
"-I",
@@ -538,6 +537,9 @@ _COMPILE_FLAG_PREFIXES = (
"-std=",
"-include",
)
# Plain-form linker flags rejected by _project_flags: inert on a -c compile
# line, so the firmware would silently lack the requested link behavior.
# Best-effort, not exhaustive; see _COMPILE_FLAG_PREFIXES above.
_PLAIN_LINKER_FLAGS = (
"-u",
"-e",
@@ -632,6 +634,16 @@ def generate_ld_scripts(
+ f" {build_surgery.surgery_fingerprint()}"
)
stderr_note = ld_dir / f".{_COMMON_LD_NAME}.stderr"
def _note_digest() -> str:
# The note is an output like the script itself; folding its state
# into the stamp makes an externally removed or edited note a cache
# miss that re-runs -E and re-derives the diagnostic
if not stderr_note.is_file():
return "none"
return hashlib.sha256(stderr_note.read_bytes()).hexdigest()
def _cached_ld_is_valid() -> bool:
# Any damaged cache regenerates; never abort the build over it. The
# stamp records the sha256 of the content written, so an externally
@@ -639,41 +651,43 @@ def generate_ld_scripts(
try:
if not (output.is_file() and stamp.is_file()):
return False
inputs, sep, digest = stamp.read_text(encoding="utf-8").rpartition(
rest, sep, digest = stamp.read_text(encoding="utf-8").rpartition(
" content="
)
inputs, note_sep, note_digest = rest.rpartition(" note=")
return (
bool(sep)
and bool(note_sep)
and inputs == stamp_content
and note_digest == _note_digest()
and hashlib.sha256(output.read_bytes()).hexdigest() == digest
)
except (OSError, UnicodeDecodeError):
return False
stderr_note = ld_dir / f".{_COMMON_LD_NAME}.stderr"
if not _cached_ld_is_valid():
try:
result = subprocess.run(
cmd,
capture_output=True,
# Localized gcc diagnostics on a non-UTF-8 console must
# degrade, not UnicodeDecodeError the build
encoding="utf-8",
errors="replace",
check=False,
close_fds=False,
)
except OSError as err:
# A half-extracted or half-deleted toolchain cache reaches here
raise EsphomeError(f"Could not run {gcc}: {err}; {_CLEAN_HINT}") from err
# Localized gcc diagnostics on a non-UTF-8 console must degrade,
# not UnicodeDecodeError the build; the script itself (below) is
# decoded strictly instead, so a mangled byte can never be cached
stderr_text = result.stderr.decode("utf-8", errors="replace")
if result.returncode != 0:
raise EsphomeError(f"Generating the linker script failed:\n{result.stderr}")
raise EsphomeError(f"Generating the linker script failed:\n{stderr_text}")
note_persisted = True
if result.stderr.strip():
if stderr_text.strip():
# Preprocessor warnings on the success path must reach the user
# on this and every later cached build (see the re-emit below)
_LOGGER.warning("Linker-script preprocessor: %s", result.stderr.strip())
note_persisted = _write_note(stderr_note, result.stderr.strip(), warn=True)
_LOGGER.warning("Linker-script preprocessor: %s", stderr_text.strip())
note_persisted = _write_note(stderr_note, stderr_text.strip(), warn=True)
else:
try:
stderr_note.unlink(missing_ok=True)
@@ -688,12 +702,21 @@ def generate_ld_scripts(
_CLEAN_HINT,
)
note_persisted = False
if "SECTIONS" not in result.stdout:
try:
stdout_text = result.stdout.decode("utf-8")
except UnicodeDecodeError as err:
# -CC keeps header comments verbatim; a non-UTF-8 byte replaced
# with U+FFFD would be cached as valid for the build dir's life
raise EsphomeError(
f"Preprocessed linker script from {header} is not UTF-8: "
f"{err}; {_CLEAN_HINT}"
) from err
if "SECTIONS" not in stdout_text:
# A degenerate zero-exit run must not be stamped as a good cache
raise EsphomeError(
f"Generated linker script is missing its SECTIONS block; {_CLEAN_HINT}"
)
content = _apply_surgery(build_surgery.relocate_ratetable, result.stdout)
content = _apply_surgery(build_surgery.relocate_ratetable, stdout_text)
if CORE.testing_mode:
content = _apply_surgery(
build_surgery.apply_testing_memory_patches, content, ("iram1_0_seg",)
@@ -704,7 +727,8 @@ def generate_ld_scripts(
# diagnostic the lost note would have re-emitted
_write_note(
stamp,
f"{stamp_content} content={hashlib.sha256(content.encode('utf-8')).hexdigest()}",
f"{stamp_content} note={_note_digest()} "
f"content={hashlib.sha256(content.encode('utf-8')).hexdigest()}",
)
elif stderr_note.is_file():
# Re-emit cached preprocessor warnings on cache hits
tests/unit_tests/build_gen/test_arduino8266.py
+29 -23
View File
@@ -83,11 +83,15 @@ def _split_flags():
def _ok_result(stdout=None, stderr=""):
"""A successful preprocessor spawn (defaults to the common ld output)."""
"""A successful preprocessor spawn (defaults to the common ld output).
Streams are bytes, as the un-decoded subprocess.run delivers them.
"""
stdout = _COMMON_LD_H_OUTPUT if stdout is None else stdout
return MagicMock(
returncode=0,
stdout=_COMMON_LD_H_OUTPUT if stdout is None else stdout,
stderr=stderr,
stdout=stdout.encode() if isinstance(stdout, str) else stdout,
stderr=stderr.encode() if isinstance(stderr, str) else stderr,
)
@@ -383,7 +387,7 @@ def test_generate_ld_scripts_corrupt_cache_regenerates(tmp_path: Path) -> None:
def test_generate_ld_scripts_failure(tmp_path: Path) -> None:
paths = _make_framework(tmp_path)
result = MagicMock(returncode=1, stderr="nope")
result = MagicMock(returncode=1, stderr=b"nope")
with (
patch.object(arduino8266.subprocess, "run", return_value=result),
pytest.raises(EsphomeError, match="linker script failed"),
@@ -644,16 +648,14 @@ def test_generate_ld_scripts_surfaces_preprocessor_warnings(
"""Preprocessor stderr on a zero exit reaches the user; degenerate output is refused."""
paths = _make_framework(tmp_path)
_set_flags()
result = MagicMock(
returncode=0, stdout=_COMMON_LD_H_OUTPUT, stderr="warning: something"
)
result = _ok_result(stderr="warning: something")
with patch.object(arduino8266.subprocess, "run", return_value=result):
_run_generate_ld_scripts(paths)
assert "Linker-script preprocessor: warning: something" in caplog.text
# New flags invalidate the stamp so the degenerate run regenerates
_set_flags("-DVTABLES_IN_DRAM")
result = MagicMock(returncode=0, stdout="", stderr="")
result = _ok_result(stdout="")
with (
patch.object(arduino8266.subprocess, "run", return_value=result),
pytest.raises(EsphomeError, match="SECTIONS"),
@@ -668,9 +670,7 @@ def test_generate_ld_scripts_lost_warn_note_vetoes_the_stamp(
re-runs -E and re-derives the diagnostic instead of losing it."""
paths = _make_framework(tmp_path)
_set_flags()
result = MagicMock(
returncode=0, stdout=_COMMON_LD_H_OUTPUT, stderr="warning: something"
)
result = _ok_result(stderr="warning: something")
real_write_text = Path.write_text
def fail_note_writes(self: Path, text: str, encoding: str = "utf-8") -> int:
@@ -700,8 +700,8 @@ def test_generate_ld_scripts_unremovable_stale_note_vetoes_the_stamp(
obsolete diagnostic is not re-emitted on cache hits forever."""
paths = _make_framework(tmp_path)
_set_flags()
warn = MagicMock(returncode=0, stdout=_COMMON_LD_H_OUTPUT, stderr="warning: old")
clean = MagicMock(returncode=0, stdout=_COMMON_LD_H_OUTPUT, stderr="")
warn = _ok_result(stderr="warning: old")
clean = _ok_result()
with patch.object(arduino8266.subprocess, "run", return_value=warn):
_run_generate_ld_scripts(paths)
@@ -881,18 +881,26 @@ def test_generate_ld_scripts_corrupt_output_is_overwritten(tmp_path: Path) -> No
assert "SECTIONS" in output.read_text(encoding="utf-8")
def test_generate_ld_scripts_unreadable_note_still_warns(
tmp_path: Path, caplog: pytest.LogCaptureFixture
@pytest.mark.parametrize("damage", ["corrupt", "remove"])
def test_generate_ld_scripts_damaged_note_invalidates_cache(
tmp_path: Path, caplog: pytest.LogCaptureFixture, damage: str
) -> None:
"""A cached diagnostic that cannot be read must not vanish silently."""
"""A corrupted or externally removed diagnostic note is a cache miss:
-E re-runs and re-derives the warning instead of dropping it silently."""
paths = _make_framework(tmp_path)
result = _ok_result(stderr="warn!")
with patch.object(arduino8266.subprocess, "run", return_value=result):
ld_dir = _run_generate_ld_scripts(paths)
(ld_dir / ".local.eagle.app.v6.common.ld.stderr").write_bytes(b"\xff\xfe")
with patch.object(arduino8266.subprocess, "run", return_value=result):
note = ld_dir / ".local.eagle.app.v6.common.ld.stderr"
if damage == "corrupt":
note.write_bytes(b"\xff\xfe")
else:
note.unlink()
caplog.clear()
with patch.object(arduino8266.subprocess, "run", return_value=result) as mock_run:
_run_generate_ld_scripts(paths)
assert "could not be read" in caplog.text
assert mock_run.called
assert "Linker-script preprocessor: warn!" in caplog.text
@pytest.mark.parametrize("value", ["0x8000", "0xC000ul", "0x10UL"])
@@ -1017,7 +1025,7 @@ def test_generate_ld_scripts_surgery_failure_is_named(tmp_path: Path) -> None:
"""A moved rate-table anchor surfaces as a build error, not a traceback
or a silently unrelocated table."""
paths = _make_framework(tmp_path)
result = MagicMock(returncode=0, stdout="SECTIONS { no anchor here }", stderr="")
result = _ok_result(stdout="SECTIONS { no anchor here }")
with (
patch.object(arduino8266.subprocess, "run", return_value=result),
pytest.raises(EsphomeError, match="anchor not found"),
@@ -1085,9 +1093,7 @@ def test_generate_ld_scripts_reemits_cached_preprocessor_warning(
"""A preprocessor diagnostic survives cache hits instead of appearing
once and vanishing for the life of the build dir."""
paths = _make_framework(tmp_path)
result = MagicMock(
returncode=0, stdout=_COMMON_LD_H_OUTPUT, stderr="warning: something odd"
)
result = _ok_result(stderr="warning: something odd")
with patch.object(arduino8266.subprocess, "run", return_value=result):
_run_generate_ld_scripts(paths)
assert caplog.text.count("warning: something odd") == 1