github/esphome
2
0
Fork 0
mirror of https://github.com/esphome/esphome.git synced 2026-08-22 22:26:21 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'esp8266-native-ninja-emission' into esp8266-arduino-toolchain

Browse Source
This commit is contained in:
J. Nick Koston
2026-08-22 12:02:36 -05:00
parent b597a4f02f 10a22d7d74
commit 583f47034f
2 changed files with 145 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files
esphome/build_gen/arduino8266.py
+67 -28
View File
@@ -14,11 +14,12 @@ from the build flags with the same precedence as the PlatformIO builder.
from __future__ import annotations
import contextlib
from dataclasses import dataclass
import hashlib
import logging
import os
from pathlib import Path
import re
import subprocess
import sys
from typing import TYPE_CHECKING
@@ -47,7 +48,7 @@ from esphome.framework_helpers import (
get_project_cxx_compile_flags,
strip_win_long_path_prefix,
)
from esphome.helpers import mkdir_p, write_file_if_changed
from esphome.helpers import mkdir_p, write_file, write_file_if_changed
from esphome.platformio.library import SOURCE_KIND_FOR_SUFFIX, lex_build_flags
if TYPE_CHECKING:
@@ -358,8 +359,13 @@ def _pio_option(key: str, default: str) -> str:
"""
value = CORE.platformio_options.get(key)
if isinstance(value, list):
value = value[-1] if value else None
return default if value is None else str(value)
value = value[-1] if value else ""
if value is None:
return default
value = str(value).strip()
if not value:
raise EsphomeError(f"platformio_options {key} is empty")
return value
def _defines_flags(
@@ -371,11 +377,16 @@ def _defines_flags(
defines embed ``\"``), so they must be emitted unquoted; wrapping
them in ``_shell_token`` would deliver literal backslashes to gcc.
"""
# Every supported board ships 80 MHz; board_build.f_cpu overrides
f_cpu = _pio_option("board_build.f_cpu", "80000000L")
if not re.fullmatch(r"\d+L?", f_cpu):
# The value lands unquoted on the compile line; reject by name
# instead of corrupting it
raise EsphomeError(f"Invalid board_build.f_cpu value {f_cpu!r}")
return [
f"-D{d}"
for d in (
# Every supported board ships 80 MHz; board_build.f_cpu overrides
f"F_CPU={_pio_option('board_build.f_cpu', '80000000L')}",
f"F_CPU={f_cpu}",
"__ets__",
"ICACHE_FLASH",
"_GNU_SOURCE",
@@ -386,7 +397,9 @@ def _defines_flags(
"LWIP_OPEN_SRC",
*config.knob_defines,
config.vtables,
*config.mmu_defines,
# User-supplied bodies re-quote like every other user token
# (a no-op for real MMU values)
*(_shell_token(d) for d in config.mmu_defines),
"ESP8266",
"ARDUINO_ARCH_ESP8266",
*board_defines,
@@ -407,9 +420,10 @@ def _project_flags(
) -> tuple[list[str], list[str], list[Path], list[str]]:
"""Split the ESPHome build flags into compile, linker, -L, and -l lists.
Plain-form linker flags (``-T``/``-u``/``-Xlinker``) raise: they would be
inert on the ``-c`` compile line. ``compile_flags``/``link_flags`` come
back shell-quoted; ``lib_dirs``/``libs`` are raw, quote at emission.
Plain-form linker flags (``_PLAIN_LINKER_FLAGS``/``_PLAIN_LINKER_PREFIXES``)
raise: they would be inert on the ``-c`` compile line.
``compile_flags``/``link_flags`` come back shell-quoted;
``lib_dirs``/``libs`` are raw, quote at emission.
"""
compile_flags: list[str] = []
link_flags: list[str] = []
@@ -437,11 +451,7 @@ def _project_flags(
continue
libs.append(tok[2:])
else:
if tok in ("-u", "-e", "-s", "-static", "-nostartfiles") or tok.startswith(
("-T", "-Xlinker")
):
# Inert on the -c compile line; the firmware would silently
# lack the requested link behavior
if tok in _PLAIN_LINKER_FLAGS or tok.startswith(_PLAIN_LINKER_PREFIXES):
raise EsphomeError(
f"Linker flag {tok} in build_flags is not routed to the "
"link line; use the -Wl, form"
@@ -450,6 +460,12 @@ def _project_flags(
return compile_flags, link_flags, lib_dirs, libs
# Plain-form linker flags rejected by _project_flags: inert on a -c compile
# line, so the firmware would silently lack the requested link behavior
_PLAIN_LINKER_FLAGS = ("-u", "-e", "-s", "-static", "-nostartfiles")
_PLAIN_LINKER_PREFIXES = ("-T", "-Xlinker")
def _collect_sources(root: Path, exclude: set[str] = frozenset()) -> list[Path]:
return sorted(
p
@@ -518,15 +534,20 @@ def generate_ld_scripts(
)
def _cached_ld_is_valid() -> bool:
# Any damaged cache regenerates; never abort the build over it
# Any damaged cache regenerates; never abort the build over it. The
# stamp records the sha256 of the content written, so an externally
# edited script regenerates too.
try:
if not (
output.is_file()
and stamp.is_file()
and stamp.read_text(encoding="utf-8") == stamp_content
):
if not (output.is_file() and stamp.is_file()):
return False
return "SECTIONS" in output.read_text(encoding="utf-8")
inputs, sep, digest = stamp.read_text(encoding="utf-8").rpartition(
" content="
)
return (
bool(sep)
and inputs == stamp_content
and hashlib.sha256(output.read_bytes()).hexdigest() == digest
)
except (OSError, UnicodeDecodeError):
return False
@@ -570,15 +591,30 @@ def generate_ld_scripts(
except RuntimeError as err:
# Same changed-linker-script failure class as the ratetable
raise EsphomeError(str(err)) from err
write_file_if_changed(output, content)
stamp.write_text(stamp_content, encoding="utf-8")
try:
write_file_if_changed(output, content)
except EsphomeError:
# A non-UTF-8/unreadable cached script fails the comparison
# read; regeneration must overwrite it, not abort
output.unlink(missing_ok=True)
write_file(output, content)
stamp.write_text(
f"{stamp_content} content={hashlib.sha256(content.encode('utf-8')).hexdigest()}",
encoding="utf-8",
)
elif stderr_note.is_file():
# Re-emit cached preprocessor warnings on cache hits; best-effort
with contextlib.suppress(OSError, UnicodeDecodeError):
# Re-emit cached preprocessor warnings on cache hits
try:
_LOGGER.warning(
"Linker-script preprocessor: %s",
stderr_note.read_text(encoding="utf-8"),
)
except (OSError, UnicodeDecodeError):
_LOGGER.warning(
"A cached linker-script preprocessor diagnostic exists at %s "
"but could not be read",
stderr_note,
)
if CORE.testing_mode:
# A patched copy of the flash ld in the build dir; resolved through
@@ -755,7 +791,8 @@ def write_project(paths: InstalledPaths, ccache: str | None) -> bool:
link_flags += project_link_flags
link_flags += [_shell_token(flag) for lib in libraries for flag in lib.link_flags]
flash_ld = _active_flash_ld_name(flash_ld_name)
link_flags += ["-T", flash_ld]
# A user-overridden script name re-quotes like every other user token
link_flags += ["-T", _shell_token(flash_ld)]
lib_dirs = [Path("ld"), sdk / "lib", sdk / "ld", sdk / "lib" / config.nonosdk]
lib_dirs += project_lib_dirs
@@ -814,7 +851,9 @@ def write_project(paths: InstalledPaths, ccache: str | None) -> bool:
" description = LINK $out",
"rule elf2bin",
# --flash_freq 40: every supported board's f_flash is 40 MHz;
# re-check on a platform bump
# re-check on a platform bump. --flash_size deliberately stays
# board-derived, as under PlatformIO (which reads
# upload.maximum_size, not the ldscript).
f" command = $python {_q(framework / 'tools' / 'elf2bin.py')} --eboot {_q(framework / 'bootloaders' / 'eboot' / 'eboot.elf')} --app $in --flash_mode {esp8266_data[KEY_FLASH_MODE]} --flash_freq 40 --flash_size {_flash_size_str(BOARDS[board][KEY_FLASH_SIZE])} --path {_q(toolchain_bin)} --out $out",
" description = BIN $out",
"rule copy",
tests/unit_tests/build_gen/test_arduino8266.py
+78
View File
@@ -1071,6 +1071,73 @@ def test_generate_ld_scripts_unreadable_stamp_regenerates(tmp_path: Path) -> Non
mock_run.assert_called_once()
def test_generate_ld_scripts_edited_output_regenerates(tmp_path: Path) -> None:
"""The stamp records the content hash, so an externally edited cached
script regenerates instead of linking untrusted content."""
paths = _make_framework(tmp_path)
result = MagicMock(returncode=0, stdout=_COMMON_LD_H_OUTPUT, stderr="")
with patch.object(arduino8266.subprocess, "run", return_value=result):
ld_dir = _run_generate_ld_scripts(paths)
output = ld_dir / "local.eagle.app.v6.common.ld"
output.write_text(output.read_text() + "\n/* tampered */\n")
with patch.object(arduino8266.subprocess, "run", return_value=result) as mock_run:
_run_generate_ld_scripts(paths)
mock_run.assert_called_once()
assert "tampered" not in output.read_text()
def test_generate_ld_scripts_corrupt_output_is_overwritten(tmp_path: Path) -> None:
"""A non-UTF-8 cached script must be overwritten by the regeneration,
not abort it (write_file_if_changed reads the old content)."""
paths = _make_framework(tmp_path)
result = MagicMock(returncode=0, stdout=_COMMON_LD_H_OUTPUT, stderr="")
with patch.object(arduino8266.subprocess, "run", return_value=result):
ld_dir = _run_generate_ld_scripts(paths)
output = ld_dir / "local.eagle.app.v6.common.ld"
output.write_bytes(b"\xff\xfe")
with patch.object(arduino8266.subprocess, "run", return_value=result):
_run_generate_ld_scripts(paths)
assert "SECTIONS" in output.read_text(encoding="utf-8")
def test_generate_ld_scripts_unreadable_note_still_warns(
tmp_path: Path, caplog: pytest.LogCaptureFixture
) -> None:
"""A cached diagnostic that cannot be read must not vanish silently."""
paths = _make_framework(tmp_path)
result = MagicMock(returncode=0, stdout=_COMMON_LD_H_OUTPUT, stderr="warn!")
with patch.object(arduino8266.subprocess, "run", return_value=result):
ld_dir = _run_generate_ld_scripts(paths)
(ld_dir / ".local.eagle.app.v6.common.ld.stderr").write_bytes(b"\xff\xfe")
with patch.object(arduino8266.subprocess, "run", return_value=result):
_run_generate_ld_scripts(paths)
assert "could not be read" in caplog.text
def test_pio_option_blank_value_raises() -> None:
"""An empty or blank platformio_options value is a config error, not a
silent fallback to the default."""
CORE.platformio_options = {"board_build.f_cpu": " "}
with pytest.raises(EsphomeError, match="board_build.f_cpu is empty"):
arduino8266._pio_option("board_build.f_cpu", "80000000L")
CORE.platformio_options = {"board_build.f_cpu": []}
with pytest.raises(EsphomeError, match="board_build.f_cpu is empty"):
arduino8266._pio_option("board_build.f_cpu", "80000000L")
def test_defines_flags_invalid_f_cpu_raises() -> None:
"""A non-numeric board_build.f_cpu is rejected by name; it would land
unquoted on the compile line."""
CORE.platformio_options = {"board_build.f_cpu": "160 MHz"}
with pytest.raises(EsphomeError, match="Invalid board_build.f_cpu"):
_defines_flags(
_resolve(),
"dout",
"nodemcuv2",
ESP8266_BOARD_BUILD["nodemcuv2"]["defines"],
)
def test_generate_ld_scripts_surgery_failure_is_named(tmp_path: Path) -> None:
"""A moved rate-table anchor surfaces as a build error, not a traceback
or a silently unrelocated table."""
@@ -1275,3 +1342,14 @@ def test_flash_ld_name_honors_ldscript_override(tmp_path: Path) -> None:
CORE.platformio_options = {"board_build.ldscript": "../evil.ld"}
with pytest.raises(EsphomeError, match="bare script name"):
arduino8266._flash_ld_name("nodemcuv2")
def test_write_project_quotes_spaced_ldscript_override(tmp_path: Path) -> None:
"""An overridden script name re-quotes on the link line like every other
user token (a space would otherwise split into two argv elements)."""
CORE.platformio_options = {"board_build.ldscript": "my script.ld"}
paths = _make_framework(tmp_path)
_set_flags()
content = _write_ninja(paths)
assert "'my script.ld'" in content or '"my script.ld"' in content
assert "-T my script.ld" not in content