github/esphome
2
0
Fork 0
mirror of https://github.com/esphome/esphome.git synced 2026-08-27 00:18:32 +00:00
Code Issues Packages Projects Releases Wiki Activity

[esp8266] Add linker-script surgery and board build metadata for the native toolchain (#18555)

Browse Source
This commit is contained in:
J. Nick Koston
2026-08-26 10:29:57 +12:00
committed by GitHub
parent cf5ec2d277
commit 150f75d8f6
6 changed files with 469 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files
esphome/components/esp8266/__init__.py
+28 -12
View File
@@ -35,7 +35,7 @@ from esphome.platformio.toolchain import copy_ccache_script
from esphome.storage_json import StorageJSON
from esphome.types import ConfigType
from .boards import BOARDS, ESP8266_LD_SCRIPTS
from .boards import BOARDS, ESP8266_LD_SCRIPTS, board_ld_script
from .const import (
CONF_EARLY_PIN_INIT,
CONF_ENABLE_SERIAL,
@@ -44,6 +44,7 @@ from .const import (
KEY_BOARD,
KEY_ESP8266,
KEY_FLASH_SIZE,
KEY_LDSCRIPT,
KEY_PIN_INITIAL_STATES,
KEY_SERIAL1_REQUIRED,
KEY_SERIAL_REQUIRED,
@@ -276,6 +277,31 @@ def check_rosetta() -> None:
)
def _choose_ld_script(board: str, ver: cv.Version) -> str | None:
"""The flash ld to pin for this board and core, or None for cores
without ld-script support."""
board_data = BOARDS[board]
ld_scripts = ESP8266_LD_SCRIPTS[board_data[KEY_FLASH_SIZE]]
if ver <= cv.Version(2, 3, 0):
# No ld script support
return None
if ver <= cv.Version(2, 4, 2):
# Old ld script path; the modern per-board override names do not
# exist in this core's SDK, so the override cannot be honored.
# Substituting the size default would move _FS_end and the
# preferences sector, wiping flash-backed state on flash.
if KEY_LDSCRIPT in board_data:
raise EsphomeError(
f"Board {board} requires its {board_data[KEY_LDSCRIPT]} "
f"flash layout, which Arduino core {ver} cannot honor; "
"use a core newer than 2.4.2"
)
return ld_scripts[0]
# A per-board override preserves a layout the board shipped with
# (see d1_wroom_02 in boards.py)
return board_ld_script(board_data)
@coroutine_with_priority(CoroPriority.PLATFORM)
async def to_code(config: ConfigType) -> None:
cg.add(esp8266_ns.setup_preferences())
@@ -397,17 +423,7 @@ async def to_code(config: ConfigType) -> None:
)
if config[CONF_BOARD] in BOARDS:
flash_size = BOARDS[config[CONF_BOARD]][KEY_FLASH_SIZE]
ld_scripts = ESP8266_LD_SCRIPTS[flash_size]
if ver <= cv.Version(2, 3, 0):
# No ld script support
ld_script = None
elif ver <= cv.Version(2, 4, 2):
# Old ld script path
ld_script = ld_scripts[0]
else:
ld_script = ld_scripts[1]
ld_script = _choose_ld_script(config[CONF_BOARD], ver)
if ld_script is not None:
cg.add_platformio_option("board_build.ldscript", ld_script)
esphome/components/esp8266/boards.py
+135 -1
View File
@@ -1,3 +1,5 @@
from .const import KEY_FLASH_SIZE, KEY_LDSCRIPT
FLASH_SIZE_1_MB = 2**20
FLASH_SIZE_512_KB = FLASH_SIZE_1_MB // 2
FLASH_SIZE_2_MB = 2 * FLASH_SIZE_1_MB
@@ -164,7 +166,8 @@ ESP8266_BOARD_PINS = {
}
"""
BOARDS generate with:
BOARDS generate with (preserve per-board KEY_LDSCRIPT overrides such as
d1_wroom_02; the recipe emits only name/flash_size):
git clone https://github.com/platformio/platform-espressif8266
for x in platform-espressif8266/boards/*.json; do
@@ -182,6 +185,19 @@ for x in platform-espressif8266/boards/*.json; do
done | sort
"""
def board_ld_script(board_data: dict) -> str:
"""The modern (core > 2.4.2) flash linker script for a board: its
shipped-layout override, else the size default (the no-FS layout).
Single source of truth for the PlatformIO pinning in __init__ and the
native generator's fallback, so the per-board rule cannot drift.
"""
return board_data.get(
KEY_LDSCRIPT, ESP8266_LD_SCRIPTS[board_data[KEY_FLASH_SIZE]][1]
)
BOARDS = {
"agruminolemon": {
"name": "Lifely Agrumino Lemon v4",
@@ -199,6 +215,15 @@ BOARDS = {
"name": "WeMos D1 mini Pro",
"flash_size": FLASH_SIZE_16_MB,
},
"d1_wroom_02": {
"name": "WeMos D1 ESP-WROOM-02",
"flash_size": FLASH_SIZE_2_MB,
# This board joined BOARDS after shipping with the manifest default
# (64 KB filesystem region); the flash-size default (2m.ld) would
# move _FS_end and with it the preferences sector, wiping existing
# devices' flash-backed state on update.
KEY_LDSCRIPT: "eagle.flash.2m64.ld",
},
"d1": {
"name": "WEMOS D1 R1",
"flash_size": FLASH_SIZE_4_MB,
@@ -360,3 +385,112 @@ BOARDS = {
"flash_size": FLASH_SIZE_4_MB,
},
}
# Per-board variant dir + identity defines from platform-espressif8266 4.x
# build.extra_flags; the shared -DESP8266/-DARDUINO_ARCH_ESP8266 are added
# by the generator.
#
# Regenerate ESP8266_BOARD_BUILD with (v4.2.1 is the platform version the
# native toolchain mirrors; regenerate against the tag when bumping it):
#
# git clone -b v4.2.1 https://github.com/platformio/platform-espressif8266
# python3 - <<'EOF'
# import json, glob, os
# for f in sorted(glob.glob("platform-espressif8266/boards/*.json")):
# b = json.load(open(f))["build"]
# extra = b["extra_flags"]
# extra = extra.split() if isinstance(extra, str) else extra
# defines = [
# e[2:] for e in extra if e not in ("-DESP8266", "-DARDUINO_ARCH_ESP8266")
# ]
# entries = ", ".join(f'"{d}"' for d in defines) + ("," if len(defines) == 1 else "")
# board = os.path.splitext(os.path.basename(f))[0]
# print(f' "{board}": {{"variant": "{b["variant"]}", "defines": ({entries})}},')
# EOF
ESP8266_BOARD_BUILD = {
"agruminolemon": {
"variant": "agruminolemonv4",
"defines": ("ARDUINO_ESP8266_AGRUMINO_LEMON_V4",),
},
"d1": {"variant": "d1", "defines": ("ARDUINO_ESP8266_WEMOS_D1R1",)},
"d1_mini": {"variant": "d1_mini", "defines": ("ARDUINO_ESP8266_WEMOS_D1MINI",)},
"d1_mini_lite": {
"variant": "d1_mini",
"defines": ("ARDUINO_ESP8266_WEMOS_D1MINILITE",),
},
"d1_mini_pro": {
"variant": "d1_mini",
"defines": ("ARDUINO_ESP8266_WEMOS_D1MINIPRO",),
},
"d1_wroom_02": {
"variant": "d1_mini",
"defines": ("ARDUINO_ESP8266_WEMOS_D1WROOM02",),
},
"eduinowifi": {
"variant": "eduinowifi",
"defines": ("ARDUINO_ESP8266_SCHIRMILABS_EDUINO_WIFI",),
},
"esp01": {"variant": "generic", "defines": ("ARDUINO_ESP8266_ESP01",)},
"esp01_1m": {"variant": "generic", "defines": ("ARDUINO_ESP8266_ESP01",)},
"esp07": {"variant": "generic", "defines": ("ARDUINO_ESP8266_ESP07",)},
"esp07s": {"variant": "nodemcu", "defines": ("ARDUINO_ESP8266_ESP07",)},
"esp12e": {"variant": "nodemcu", "defines": ("ARDUINO_ESP8266_ESP12",)},
"esp210": {"variant": "generic", "defines": ("ARDUINO_ESP8266_ESP210",)},
"esp8285": {"variant": "esp8285", "defines": ("ARDUINO_ESP8266_ESP01",)},
"esp_wroom_02": {
"variant": "nodemcu",
"defines": ("ARDUINO_ESP8266_ESP_WROOM_02",),
},
"espduino": {"variant": "ESPDuino", "defines": ("ARDUINO_ESP8266_ESP13",)},
"espectro": {"variant": "espectro", "defines": ("ARDUINO_ESP8266_ESPECTRO_CORE",)},
"espino": {"variant": "espino", "defines": ("ARDUINO_ESP8266_ESP12",)},
"espinotee": {"variant": "espinotee", "defines": ("ARDUINO_ESP8266_ESP13",)},
"espmxdevkit": {
"variant": "esp8285",
"defines": ("ARDUINO_ESP8266_ESP01", "LED_BUILTIN=16"),
},
"espresso_lite_v1": {
"variant": "espresso_lite_v1",
"defines": ("ARDUINO_ESP8266_ESPRESSO_LITE_V1",),
},
"espresso_lite_v2": {
"variant": "espresso_lite_v2",
"defines": ("ARDUINO_ESP8266_ESPRESSO_LITE_V2",),
},
"gen4iod": {"variant": "generic", "defines": ("ARDUINO_GEN4_IOD",)},
"heltec_wifi_kit_8": {
"variant": "wifi_kit_8",
"defines": ("ARDUINO_wifi_kit_8",),
},
"huzzah": {"variant": "adafruit", "defines": ("ARDUINO_ESP8266_ADAFRUIT_HUZZAH",)},
"inventone": {"variant": "inventone", "defines": ("ARDUINO_ESP8266_INVENT_ONE",)},
"modwifi": {"variant": "generic", "defines": ("ARDUINO_MOD_WIFI_ESP8266",)},
"nodemcu": {"variant": "nodemcu", "defines": ("ARDUINO_ESP8266_NODEMCU",)},
"nodemcuv2": {"variant": "nodemcu", "defines": ("ARDUINO_ESP8266_NODEMCU_ESP12E",)},
"oak": {"variant": "oak", "defines": ("ARDUINO_ESP8266_OAK",)},
"phoenix_v1": {
"variant": "phoenix_v1",
"defines": ("ARDUINO_ESP8266_PHOENIX_V1",),
},
"phoenix_v2": {
"variant": "phoenix_v2",
"defines": ("ARDUINO_ESP8266_PHOENIX_V2",),
},
"sonoff_basic": {"variant": "itead", "defines": ("ARDUINO_ESP8266_SONOFF_BASIC",)},
"sonoff_s20": {"variant": "itead", "defines": ("ARDUINO_ESP8266_SONOFF_S20",)},
"sonoff_sv": {"variant": "itead", "defines": ("ARDUINO_ESP8266_SONOFF_SV",)},
"sonoff_th": {"variant": "itead", "defines": ("ARDUINO_ESP8266_SONOFF_TH",)},
"sparkfunBlynk": {"variant": "thing", "defines": ("ARDUINO_ESP8266_THING",)},
"thing": {"variant": "thing", "defines": ("ARDUINO_ESP8266_THING",)},
"thingdev": {"variant": "thing", "defines": ("ARDUINO_ESP8266_THING_DEV",)},
"wifi_slot": {"variant": "wifi_slot", "defines": ("ARDUINO_AMPERKA_WIFI_SLOT",)},
"wifiduino": {"variant": "wifiduino", "defines": ("ARDUINO_WIFIDUINO_ESP8266",)},
"wifinfo": {"variant": "wifinfo", "defines": ("ARDUINO_WIFINFO",)},
"wio_link": {"variant": "wiolink", "defines": ("ARDUINO_ESP8266_WIO_LINK",)},
"wio_node": {"variant": "nodemcu", "defines": ("ARDUINO_ESP8266_ESP_WROOM_02",)},
"xinabox_cw01": {
"variant": "xinabox",
"defines": ("ARDUINO_ESP8266_XINABOX_CW01",),
},
}
esphome/components/esp8266/build_surgery.py
+123
View File
@@ -0,0 +1,123 @@
"""Linker-script surgery shared with the native (PlatformIO-free) toolchain.
These mirror the PlatformIO extra scripts in this directory
(``relocate_ratetable.py.script`` and ``testing_mode.py.script``), which run
inside SCons and must stay self-contained. The native build generator applies
the same patches to the linker scripts it generates, so the logic lives here
as plain functions. Keep both in sync when changing either.
``segment_length`` is native-toolchain-only and has no script twin.
"""
from __future__ import annotations
from collections.abc import Collection
import hashlib
import re
# Move the NONOS SDK wifi rate tables from flash to DRAM; see
# relocate_ratetable.py.script for the full background (NONOS SDK issue 320).
RATETABLE_RULE = "*libnet80211.a:ieee80211_phy.o(.irom.text .irom.text.*)"
_RATETABLE_COMMENT = (
"/* ESPHome: wifi rate tables must live in DRAM, see NONOS SDK issue 320 */"
)
# Match the whole line: "_data_start" is also a substring of the
# "_dport0_data_start" line in the earlier .dport0.data section
_RATETABLE_ANCHOR = re.compile(r"^\s*_data_start = ABSOLUTE\(\.\);", re.MULTILINE)
# Memory sizes for testing mode (allow larger builds for CI component grouping)
TESTING_IRAM_SIZE = "0x200000" # 2MB
TESTING_DRAM_SIZE = "0x200000" # 2MB
TESTING_FLASH_SIZE = "0x2000000" # 32MB
def relocate_ratetable(content: str) -> str:
"""Insert the rate-table DRAM rule into a generated common linker script."""
if RATETABLE_RULE in content:
return content
match = _RATETABLE_ANCHOR.search(content)
if match is None:
raise RuntimeError(
"'_data_start' anchor not found in the generated linker script; "
"cannot apply wifi rate table DRAM relocation "
"(has the Arduino core linker script changed?)"
)
insert_pos = match.end()
return (
content[:insert_pos]
+ f"\n {_RATETABLE_COMMENT}"
+ f"\n {RATETABLE_RULE}"
+ content[insert_pos:]
)
_TESTING_SEGMENT_SIZES = {
"iram1_0_seg": TESTING_IRAM_SIZE,
"dram0_0_seg": TESTING_DRAM_SIZE,
"irom0_0_seg": TESTING_FLASH_SIZE,
}
def _segment_line_re(segment_name: str) -> re.Pattern[str]:
"""The MEMORY line for one segment: ``<seg> : org = 0x..., len = 0x...``.
Anchored to the start of the line so a name never matches inside a
longer one (``ram0_0_seg`` must not read ``dram0_0_seg``). The size
group stops at the hex digits, leaving any ``ul`` suffix (from the
preprocessed ``MMU_IRAM_SIZE``) in place.
"""
return re.compile(
rf"(^[ \t]*{re.escape(segment_name)}"
r"\s*:\s*org\s*=\s*0x[0-9a-fA-F]+\s*,\s*len\s*=\s*)"
r"(0x[0-9a-fA-F]+)",
re.MULTILINE,
)
def apply_testing_memory_patches(content: str, segments: Collection[str]) -> str:
"""Enlarge the named memory segments so grouped CI test builds can link.
Each caller passes the segments its linker script defines: the
generated common ld carries ``iram1_0_seg``; the flash ld carries
``dram0_0_seg`` and ``irom0_0_seg``. A segment that fails to match
raises, since a silently kept real memory limit would fail grouped
builds far from the cause.
"""
for segment in _TESTING_SEGMENT_SIZES:
if segment not in segments and _segment_line_re(segment).search(content):
raise RuntimeError(
f"Testing-mode segment {segment} is present in the linker "
"script but was not selected for patching"
)
for segment in segments:
if segment not in _TESTING_SEGMENT_SIZES:
raise RuntimeError(f"Unknown testing-mode segment {segment!r}")
content, count = _segment_line_re(segment).subn(
rf"\g<1>{_TESTING_SEGMENT_SIZES[segment]}", content
)
if count == 0:
raise RuntimeError(
f"Testing-mode memory patch failed: segment {segment} "
"not found (has the Arduino core linker script changed?)"
)
return content
def segment_length(content: str, segment_name: str) -> int | None:
"""Read a memory segment's length from linker script content.
Returns None for an absent segment OR an unparsable line; callers must
treat None as "no usable budget" and warn (as the Flash summary does),
never as "no limit".
"""
match = _segment_line_re(segment_name).search(content)
return int(match.group(2), 16) if match else None
def surgery_fingerprint() -> str:
"""Hash of this module's source; linker-script caches include it so an
edit here invalidates them."""
import inspect
import sys
source = inspect.getsource(sys.modules[__name__])
return hashlib.sha256(source.encode()).hexdigest()
esphome/components/esp8266/const.py
+5
View File
@@ -15,6 +15,11 @@ CONF_ENABLE_SERIAL1 = "enable_serial1"
KEY_WAVEFORM_REQUIRED = "waveform_required"
KEY_SERIAL_REQUIRED = "serial_required"
KEY_SERIAL1_REQUIRED = "serial1_required"
# Set for the native (non-PlatformIO) toolchain's build generator
KEY_FLASH_MODE = "flash_mode"
KEY_SCANF_FLOAT = "scanf_float"
# Per-board flash-layout override consumed by board_ld_script()
KEY_LDSCRIPT = "ldscript"
# esp8266 namespace is already defined by arduino, manually prefix esphome
esp8266_ns = cg.global_ns.namespace("esphome").namespace("esp8266")
tests/unit_tests/components/esp8266/test_boards.py
+33
View File
@@ -0,0 +1,33 @@
"""Tests for the per-board linker-script rule."""
import pytest
from esphome.components.esp8266 import _choose_ld_script
from esphome.components.esp8266.boards import BOARDS, board_ld_script
import esphome.config_validation as cv
from esphome.core import EsphomeError
def test_d1_wroom_02_keeps_its_shipped_layout() -> None:
"""The override must survive a BOARDS regeneration or key typo: the
2m.ld default moves _FS_end and the preferences sector on deployed
devices."""
assert board_ld_script(BOARDS["d1_wroom_02"]) == "eagle.flash.2m64.ld"
def test_default_boards_use_the_flash_size_layout() -> None:
assert board_ld_script(BOARDS["d1_mini"]) == "eagle.flash.4m.ld"
assert board_ld_script(BOARDS["esp01_1m"]) == "eagle.flash.1m.ld"
def test_choose_ld_script_paths() -> None:
"""Old cores get the size default, overriding boards hard-error there
(a substituted layout would wipe flash-backed state), modern cores
honor the override."""
assert _choose_ld_script("nodemcuv2", cv.Version(2, 3, 0)) is None
assert _choose_ld_script("nodemcuv2", cv.Version(2, 4, 2)) == "eagle.flash.4m.ld"
assert _choose_ld_script("d1_wroom_02", cv.Version(2, 7, 4)) == (
"eagle.flash.2m64.ld"
)
with pytest.raises(EsphomeError, match="cannot honor"):
_choose_ld_script("d1_wroom_02", cv.Version(2, 4, 2))
tests/unit_tests/components/esp8266/test_build_surgery.py
+145
View File
@@ -0,0 +1,145 @@
"""Tests for the linker-script surgery shared with the native toolchain."""
from __future__ import annotations
import importlib.util
from pathlib import Path
import sys
import pytest
from esphome.components.esp8266 import build_surgery
from esphome.components.esp8266.boards import BOARDS, ESP8266_BOARD_BUILD
from esphome.components.esp8266.build_surgery import (
RATETABLE_RULE,
apply_testing_memory_patches,
relocate_ratetable,
segment_length,
)
_COMMON_LD_SNIPPET = """\
.dport0.data : ALIGN(4)
{
_dport0_data_start = ABSOLUTE(.);
} >dport0_0_seg :dport0_0_phdr
.data : ALIGN(4)
{
_data_start = ABSOLUTE(.);
*(.data)
} >dram0_0_seg :dram0_0_phdr
"""
# Shaped like the real SDK flash ld scripts: no iram1_0_seg (that lives in
# the generated common ld only)
_FLASH_LD_SNIPPET = """\
MEMORY
{
dport0_0_seg : org = 0x3FF00000, len = 0x10
dram0_0_seg : org = 0x3FFE8000, len = 0x14000
irom0_0_seg : org = 0x40201010, len = 0xfeff0
}
"""
# Shaped like the preprocessed common ld: MMU_IRAM_SIZE expands with a ul
# suffix the patcher must leave in place
_COMMON_LD_MEMORY_SNIPPET = """\
MEMORY
{
iram1_0_seg : org = 0x40100000, len = 0x8000ul
}
"""
def test_relocate_ratetable_inserts_after_data_start() -> None:
patched = relocate_ratetable(_COMMON_LD_SNIPPET)
assert RATETABLE_RULE in patched
# Inserted after the .data section's anchor, not the .dport0.data one
# (whose closing brace bounds the decoy block)
assert RATETABLE_RULE not in patched[: patched.index("} >dport0_0_seg")]
assert patched.index(RATETABLE_RULE) < patched.index("*(.data)")
# Idempotent on an already-patched script
assert relocate_ratetable(patched) == patched
def test_relocate_ratetable_requires_anchor() -> None:
with pytest.raises(RuntimeError, match="_data_start"):
relocate_ratetable("SECTIONS { }")
def test_testing_memory_patches_enlarge_segments() -> None:
patched = apply_testing_memory_patches(
_FLASH_LD_SNIPPET, ("dram0_0_seg", "irom0_0_seg")
)
assert segment_length(patched, "dram0_0_seg") == 0x200000
assert segment_length(patched, "irom0_0_seg") == 0x2000000
# Untouched segments keep their sizes
assert segment_length(patched, "dport0_0_seg") == 0x10
def test_testing_memory_patches_keep_ul_suffix() -> None:
"""The common ld's preprocessed sizes carry a ul suffix; the patch must
replace only the hex digits, as testing_mode.py.script does."""
patched = apply_testing_memory_patches(_COMMON_LD_MEMORY_SNIPPET, ("iram1_0_seg",))
assert "len = 0x200000ul" in patched
assert segment_length(patched, "iram1_0_seg") == 0x200000
def test_segment_length_requires_whole_name() -> None:
"""A name must match its own line, never inside a longer segment name."""
assert segment_length(_FLASH_LD_SNIPPET, "ram0_0_seg") is None
def test_testing_memory_patches_unknown_segment_raises() -> None:
with pytest.raises(RuntimeError, match="Unknown testing-mode segment"):
apply_testing_memory_patches("MEMORY { }", ("bogus_seg",))
def test_segment_length() -> None:
assert segment_length(_FLASH_LD_SNIPPET, "irom0_0_seg") == 0xFEFF0
assert segment_length(_FLASH_LD_SNIPPET, "missing_seg") is None
def test_testing_memory_patches_missing_segment_raises() -> None:
"""A named segment the patch could not find raises instead of silently
keeping the real memory limits."""
with pytest.raises(RuntimeError, match="dram0_0_seg"):
apply_testing_memory_patches("MEMORY { }", ("dram0_0_seg",))
def test_board_build_covers_every_board() -> None:
"""Every supported board has native build metadata (the table may carry
extras that BOARDS does not expose)."""
assert set(BOARDS) <= set(ESP8266_BOARD_BUILD)
def test_surgery_fingerprint_is_stable_and_sensitive(tmp_path) -> None:
"""The properties the linker-script cache depends on: the fingerprint is
stable across calls and changes when the module's source changes."""
first = build_surgery.surgery_fingerprint()
assert first == build_surgery.surgery_fingerprint()
assert len(first) == 64
int(first, 16) # sha256 hex digest
# A modified copy of the module must fingerprint differently
copy = tmp_path / "build_surgery_variant.py"
copy.write_text(
Path(build_surgery.__file__).read_text(encoding="utf-8")
+ "\nEXTRA_BEHAVIORAL_INPUT = 1\n",
encoding="utf-8",
)
spec = importlib.util.spec_from_file_location("build_surgery_variant", copy)
variant = importlib.util.module_from_spec(spec)
sys.modules[spec.name] = variant
try:
spec.loader.exec_module(variant)
assert variant.surgery_fingerprint() != first
finally:
del sys.modules[spec.name]
def test_testing_memory_patches_present_but_unselected_raises() -> None:
"""A known segment left off the caller's list must fail, not silently
keep its real memory limit."""
with pytest.raises(RuntimeError, match="not selected"):
apply_testing_memory_patches(_FLASH_LD_SNIPPET, ("dram0_0_seg",))