diff --git a/esphome/build_gen/arduino8266.py b/esphome/build_gen/arduino8266.py index 0dabab57c2..fb884bf17b 100644 --- a/esphome/build_gen/arduino8266.py +++ b/esphome/build_gen/arduino8266.py @@ -522,10 +522,9 @@ def _project_flags( return compile_flags, link_flags, lib_dirs, libs -# Plain-form linker flags rejected by _project_flags: inert on a -c compile -# line, so the firmware would silently lack the requested link behavior -# Best-effort, not exhaustive: an unlisted link-only spelling still falls -# through to the compile line, with a warning from the shape check +# Recognized compile-flag shapes: the allow-list feeding the fall-through +# warning in _project_flags (an unlisted link-only spelling still reaches +# the compile line, but not silently) _COMPILE_FLAG_PREFIXES = ( "-D", "-I", @@ -538,6 +537,9 @@ _COMPILE_FLAG_PREFIXES = ( "-std=", "-include", ) +# Plain-form linker flags rejected by _project_flags: inert on a -c compile +# line, so the firmware would silently lack the requested link behavior. +# Best-effort, not exhaustive; see _COMPILE_FLAG_PREFIXES above. _PLAIN_LINKER_FLAGS = ( "-u", "-e", @@ -632,6 +634,16 @@ def generate_ld_scripts( + f" {build_surgery.surgery_fingerprint()}" ) + stderr_note = ld_dir / f".{_COMMON_LD_NAME}.stderr" + + def _note_digest() -> str: + # The note is an output like the script itself; folding its state + # into the stamp makes an externally removed or edited note a cache + # miss that re-runs -E and re-derives the diagnostic + if not stderr_note.is_file(): + return "none" + return hashlib.sha256(stderr_note.read_bytes()).hexdigest() + def _cached_ld_is_valid() -> bool: # Any damaged cache regenerates; never abort the build over it. The # stamp records the sha256 of the content written, so an externally @@ -639,41 +651,43 @@ def generate_ld_scripts( try: if not (output.is_file() and stamp.is_file()): return False - inputs, sep, digest = stamp.read_text(encoding="utf-8").rpartition( + rest, sep, digest = stamp.read_text(encoding="utf-8").rpartition( " content=" ) + inputs, note_sep, note_digest = rest.rpartition(" note=") return ( bool(sep) + and bool(note_sep) and inputs == stamp_content + and note_digest == _note_digest() and hashlib.sha256(output.read_bytes()).hexdigest() == digest ) except (OSError, UnicodeDecodeError): return False - stderr_note = ld_dir / f".{_COMMON_LD_NAME}.stderr" if not _cached_ld_is_valid(): try: result = subprocess.run( cmd, capture_output=True, - # Localized gcc diagnostics on a non-UTF-8 console must - # degrade, not UnicodeDecodeError the build - encoding="utf-8", - errors="replace", check=False, close_fds=False, ) except OSError as err: # A half-extracted or half-deleted toolchain cache reaches here raise EsphomeError(f"Could not run {gcc}: {err}; {_CLEAN_HINT}") from err + # Localized gcc diagnostics on a non-UTF-8 console must degrade, + # not UnicodeDecodeError the build; the script itself (below) is + # decoded strictly instead, so a mangled byte can never be cached + stderr_text = result.stderr.decode("utf-8", errors="replace") if result.returncode != 0: - raise EsphomeError(f"Generating the linker script failed:\n{result.stderr}") + raise EsphomeError(f"Generating the linker script failed:\n{stderr_text}") note_persisted = True - if result.stderr.strip(): + if stderr_text.strip(): # Preprocessor warnings on the success path must reach the user # on this and every later cached build (see the re-emit below) - _LOGGER.warning("Linker-script preprocessor: %s", result.stderr.strip()) - note_persisted = _write_note(stderr_note, result.stderr.strip(), warn=True) + _LOGGER.warning("Linker-script preprocessor: %s", stderr_text.strip()) + note_persisted = _write_note(stderr_note, stderr_text.strip(), warn=True) else: try: stderr_note.unlink(missing_ok=True) @@ -688,12 +702,21 @@ def generate_ld_scripts( _CLEAN_HINT, ) note_persisted = False - if "SECTIONS" not in result.stdout: + try: + stdout_text = result.stdout.decode("utf-8") + except UnicodeDecodeError as err: + # -CC keeps header comments verbatim; a non-UTF-8 byte replaced + # with U+FFFD would be cached as valid for the build dir's life + raise EsphomeError( + f"Preprocessed linker script from {header} is not UTF-8: " + f"{err}; {_CLEAN_HINT}" + ) from err + if "SECTIONS" not in stdout_text: # A degenerate zero-exit run must not be stamped as a good cache raise EsphomeError( f"Generated linker script is missing its SECTIONS block; {_CLEAN_HINT}" ) - content = _apply_surgery(build_surgery.relocate_ratetable, result.stdout) + content = _apply_surgery(build_surgery.relocate_ratetable, stdout_text) if CORE.testing_mode: content = _apply_surgery( build_surgery.apply_testing_memory_patches, content, ("iram1_0_seg",) @@ -704,7 +727,8 @@ def generate_ld_scripts( # diagnostic the lost note would have re-emitted _write_note( stamp, - f"{stamp_content} content={hashlib.sha256(content.encode('utf-8')).hexdigest()}", + f"{stamp_content} note={_note_digest()} " + f"content={hashlib.sha256(content.encode('utf-8')).hexdigest()}", ) elif stderr_note.is_file(): # Re-emit cached preprocessor warnings on cache hits diff --git a/tests/unit_tests/build_gen/test_arduino8266.py b/tests/unit_tests/build_gen/test_arduino8266.py index a394924892..733862057d 100644 --- a/tests/unit_tests/build_gen/test_arduino8266.py +++ b/tests/unit_tests/build_gen/test_arduino8266.py @@ -83,11 +83,15 @@ def _split_flags(): def _ok_result(stdout=None, stderr=""): - """A successful preprocessor spawn (defaults to the common ld output).""" + """A successful preprocessor spawn (defaults to the common ld output). + + Streams are bytes, as the un-decoded subprocess.run delivers them. + """ + stdout = _COMMON_LD_H_OUTPUT if stdout is None else stdout return MagicMock( returncode=0, - stdout=_COMMON_LD_H_OUTPUT if stdout is None else stdout, - stderr=stderr, + stdout=stdout.encode() if isinstance(stdout, str) else stdout, + stderr=stderr.encode() if isinstance(stderr, str) else stderr, ) @@ -383,7 +387,7 @@ def test_generate_ld_scripts_corrupt_cache_regenerates(tmp_path: Path) -> None: def test_generate_ld_scripts_failure(tmp_path: Path) -> None: paths = _make_framework(tmp_path) - result = MagicMock(returncode=1, stderr="nope") + result = MagicMock(returncode=1, stderr=b"nope") with ( patch.object(arduino8266.subprocess, "run", return_value=result), pytest.raises(EsphomeError, match="linker script failed"), @@ -644,16 +648,14 @@ def test_generate_ld_scripts_surfaces_preprocessor_warnings( """Preprocessor stderr on a zero exit reaches the user; degenerate output is refused.""" paths = _make_framework(tmp_path) _set_flags() - result = MagicMock( - returncode=0, stdout=_COMMON_LD_H_OUTPUT, stderr="warning: something" - ) + result = _ok_result(stderr="warning: something") with patch.object(arduino8266.subprocess, "run", return_value=result): _run_generate_ld_scripts(paths) assert "Linker-script preprocessor: warning: something" in caplog.text # New flags invalidate the stamp so the degenerate run regenerates _set_flags("-DVTABLES_IN_DRAM") - result = MagicMock(returncode=0, stdout="", stderr="") + result = _ok_result(stdout="") with ( patch.object(arduino8266.subprocess, "run", return_value=result), pytest.raises(EsphomeError, match="SECTIONS"), @@ -668,9 +670,7 @@ def test_generate_ld_scripts_lost_warn_note_vetoes_the_stamp( re-runs -E and re-derives the diagnostic instead of losing it.""" paths = _make_framework(tmp_path) _set_flags() - result = MagicMock( - returncode=0, stdout=_COMMON_LD_H_OUTPUT, stderr="warning: something" - ) + result = _ok_result(stderr="warning: something") real_write_text = Path.write_text def fail_note_writes(self: Path, text: str, encoding: str = "utf-8") -> int: @@ -700,8 +700,8 @@ def test_generate_ld_scripts_unremovable_stale_note_vetoes_the_stamp( obsolete diagnostic is not re-emitted on cache hits forever.""" paths = _make_framework(tmp_path) _set_flags() - warn = MagicMock(returncode=0, stdout=_COMMON_LD_H_OUTPUT, stderr="warning: old") - clean = MagicMock(returncode=0, stdout=_COMMON_LD_H_OUTPUT, stderr="") + warn = _ok_result(stderr="warning: old") + clean = _ok_result() with patch.object(arduino8266.subprocess, "run", return_value=warn): _run_generate_ld_scripts(paths) @@ -881,18 +881,26 @@ def test_generate_ld_scripts_corrupt_output_is_overwritten(tmp_path: Path) -> No assert "SECTIONS" in output.read_text(encoding="utf-8") -def test_generate_ld_scripts_unreadable_note_still_warns( - tmp_path: Path, caplog: pytest.LogCaptureFixture +@pytest.mark.parametrize("damage", ["corrupt", "remove"]) +def test_generate_ld_scripts_damaged_note_invalidates_cache( + tmp_path: Path, caplog: pytest.LogCaptureFixture, damage: str ) -> None: - """A cached diagnostic that cannot be read must not vanish silently.""" + """A corrupted or externally removed diagnostic note is a cache miss: + -E re-runs and re-derives the warning instead of dropping it silently.""" paths = _make_framework(tmp_path) result = _ok_result(stderr="warn!") with patch.object(arduino8266.subprocess, "run", return_value=result): ld_dir = _run_generate_ld_scripts(paths) - (ld_dir / ".local.eagle.app.v6.common.ld.stderr").write_bytes(b"\xff\xfe") - with patch.object(arduino8266.subprocess, "run", return_value=result): + note = ld_dir / ".local.eagle.app.v6.common.ld.stderr" + if damage == "corrupt": + note.write_bytes(b"\xff\xfe") + else: + note.unlink() + caplog.clear() + with patch.object(arduino8266.subprocess, "run", return_value=result) as mock_run: _run_generate_ld_scripts(paths) - assert "could not be read" in caplog.text + assert mock_run.called + assert "Linker-script preprocessor: warn!" in caplog.text @pytest.mark.parametrize("value", ["0x8000", "0xC000ul", "0x10UL"]) @@ -1017,7 +1025,7 @@ def test_generate_ld_scripts_surgery_failure_is_named(tmp_path: Path) -> None: """A moved rate-table anchor surfaces as a build error, not a traceback or a silently unrelocated table.""" paths = _make_framework(tmp_path) - result = MagicMock(returncode=0, stdout="SECTIONS { no anchor here }", stderr="") + result = _ok_result(stdout="SECTIONS { no anchor here }") with ( patch.object(arduino8266.subprocess, "run", return_value=result), pytest.raises(EsphomeError, match="anchor not found"), @@ -1085,9 +1093,7 @@ def test_generate_ld_scripts_reemits_cached_preprocessor_warning( """A preprocessor diagnostic survives cache hits instead of appearing once and vanishing for the life of the build dir.""" paths = _make_framework(tmp_path) - result = MagicMock( - returncode=0, stdout=_COMMON_LD_H_OUTPUT, stderr="warning: something odd" - ) + result = _ok_result(stderr="warning: something odd") with patch.object(arduino8266.subprocess, "run", return_value=result): _run_generate_ld_scripts(paths) assert caplog.text.count("warning: something odd") == 1