[esp8266] Add native toolchain that builds the Arduino core without PlatformIO

This commit is contained in:
J. Nick Koston
2026-08-19 22:31:46 -05:00
parent 29404a782c
commit 373bee8aca
25 changed files with 2053 additions and 106 deletions
+39
View File
@@ -101,6 +101,8 @@ jobs:
device-builder: ${{ steps.determine.outputs.device-builder }}
esp32-platformio: ${{ steps.determine.outputs.esp32-platformio }}
esp32-platformio-components: ${{ steps.determine.outputs.esp32-platformio-components }}
esp8266-native: ${{ steps.determine.outputs.esp8266-native }}
esp8266-native-components: ${{ steps.determine.outputs.esp8266-native-components }}
changed-components: ${{ steps.determine.outputs.changed-components }}
changed-components-with-tests: ${{ steps.determine.outputs.changed-components-with-tests }}
directly-changed-components-with-tests: ${{ steps.determine.outputs.directly-changed-components-with-tests }}
@@ -155,6 +157,8 @@ jobs:
echo "device-builder=$(echo "$output" | jq -r '.device_builder')" >> $GITHUB_OUTPUT
echo "esp32-platformio=$(echo "$output" | jq -r '.esp32_platformio')" >> $GITHUB_OUTPUT
echo "esp32-platformio-components=$(echo "$output" | jq -r '.esp32_platformio_components')" >> $GITHUB_OUTPUT
echo "esp8266-native=$(echo "$output" | jq -r '.esp8266_native')" >> $GITHUB_OUTPUT
echo "esp8266-native-components=$(echo "$output" | jq -r '.esp8266_native_components')" >> $GITHUB_OUTPUT
echo "changed-components=$(echo "$output" | jq -c '.changed_components')" >> $GITHUB_OUTPUT
echo "changed-components-with-tests=$(echo "$output" | jq -c '.changed_components_with_tests')" >> $GITHUB_OUTPUT
echo "directly-changed-components-with-tests=$(echo "$output" | jq -c '.directly_changed_components_with_tests')" >> $GITHUB_OUTPUT
@@ -1125,6 +1129,40 @@ jobs:
# Arduino framework via PlatformIO (only components with an esp32-ard test are built):
python3 script/test_build_components.py -e compile -t esp32-ard -c "$TEST_COMPONENTS" -f --toolchain platformio
test-esp8266-native:
name: Test esp8266 components with the native toolchain
runs-on: ubuntu-24.04
needs:
- common
- determine-jobs
if: github.event_name == 'pull_request' && needs.determine-jobs.outputs.esp8266-native == 'true'
env:
# Comma-joined subset of the native-ESP8266 representative component list,
# computed by script/determine-jobs.py (esp8266_native_components_to_test).
# Single source of truth -- the full list lives in
# script/determine-jobs.py::ESP8266_NATIVE_TEST_COMPONENTS.
TEST_COMPONENTS: ${{ needs.determine-jobs.outputs.esp8266-native-components }}
steps:
- name: Check out code from GitHub
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
- name: Restore Python
uses: ./.github/actions/restore-python
with:
python-version: ${{ env.DEFAULT_PYTHON }}
cache-key: ${{ needs.common.outputs.cache-key }}
- name: Run native toolchain compile test
run: |
. venv/bin/activate
echo "Testing components: $TEST_COMPONENTS"
echo ""
# ESP8266 Arduino built directly (no PlatformIO); compile validates
# config first, so a separate config pass is redundant.
python3 script/test_build_components.py -e compile -t esp8266-ard -c "$TEST_COMPONENTS" -f --toolchain arduino
device-builder:
name: Test downstream esphome/device-builder
runs-on: ubuntu-24.04
@@ -1502,6 +1540,7 @@ jobs:
- clang-tidy-esp32-variants
- test-build-components-split
- test-esp32-platformio
- test-esp8266-native
- device-builder
- memory-impact-target-branch
- memory-impact-pr-branch
+8
View File
@@ -813,6 +813,10 @@ def write_cpp_file() -> int:
from esphome.build_gen import espidf
espidf.write_project()
elif CORE.using_toolchain_arduino:
# The ninja project is generated at compile time by
# esphome.arduino8266.toolchain (it needs the downloaded framework).
pass
else:
from esphome.build_gen import platformio
@@ -964,6 +968,10 @@ def upload_using_esptool(
flash_images = [
FlashImage(path=toolchain.get_factory_firmware_path(), offset="0x0")
]
elif CORE.using_toolchain_arduino:
# The native backend writes PlatformIO-compatible output paths, so the
# shared property already points at the right file.
flash_images = [FlashImage(path=CORE.firmware_bin, offset="0x0")]
else:
from esphome.platformio import toolchain
+9
View File
@@ -0,0 +1,9 @@
"""Native (PlatformIO-free) build support for the ESP8266 Arduino core.
This package downloads the Arduino ESP8266 core and the xtensa-lx106
toolchain, generates a ninja build for them plus the ESPHome sources, and
drives the build directly — the ESP8266 equivalent of ``esphome.espidf``.
Deliberately importable without the esp8266 component to avoid circular
imports; the component wires these modules in via lazy imports.
"""
+36
View File
@@ -0,0 +1,36 @@
"""Tiny cross-platform build steps invoked from the generated ninja file.
Plain script (not ``python -m``): it runs from ninja with whatever Python
started esphome and must not depend on the package being importable.
Subcommands:
ar <ar-binary> <archive> <rspfile> remove stale archive, then ``ar rc``
copy <src> <dst> copy a file
"""
from pathlib import Path
import shutil
import subprocess
import sys
def main() -> int:
mode = sys.argv[1]
if mode == "ar":
ar, archive, rspfile = sys.argv[2:5]
# Remove first: ``ar rc`` replaces members but never drops ones whose
# source was removed from the build, which would leak stale objects.
Path(archive).unlink(missing_ok=True)
return subprocess.run(
[ar, "rc", archive, f"@{rspfile}"], check=False
).returncode
if mode == "copy":
src, dst = sys.argv[2:4]
shutil.copyfile(src, dst)
return 0
print(f"unknown build_tool mode: {mode}", file=sys.stderr)
return 1
if __name__ == "__main__":
sys.exit(main())
+173
View File
@@ -0,0 +1,173 @@
"""Arduino ESP8266 backend for the shared PlatformIO library converter.
Turns the libraries registered via ``cg.add_library()`` into build inputs for
the ninja generator. Bare names that exist under the framework's bundled
``libraries/`` directory (ESP8266WiFi, Wire, SPI, ...) are read straight from
the framework tree; everything else goes through the shared
resolution/download pipeline in ``esphome.platformio.library``.
Mirrors PlatformIO's ``lib_ldf_mode=off`` behavior: each library builds into
its own static archive and every library's include dir joins one global
include path.
"""
from __future__ import annotations
from dataclasses import dataclass, field
import logging
from pathlib import Path
import shlex
from esphome.core import CORE, Library
from esphome.espidf.extra_script import apply_extra_script
from esphome.platformio.library import (
DEFAULT_BUILD_INCLUDE_DIR,
DEFAULT_BUILD_SRC_FILTER,
SRC_FILE_EXTENSIONS,
ConvertedLibrary,
InvalidLibrary,
LibraryBackend,
check_library_data,
collect_filtered_files,
convert_libraries,
ensure_list,
normalize_dependencies,
parse_library_properties,
)
_LOGGER = logging.getLogger(__name__)
ESP8266_PLATFORM = "espressif8266"
@dataclass
class ArduinoLibrary:
"""One resolved library, ready for the ninja generator."""
name: str
sources: list[Path] = field(default_factory=list)
include_dirs: list[Path] = field(default_factory=list)
# Extra compile flags private to this library's own sources
flags: list[str] = field(default_factory=list)
# Link inputs the library contributes (-L dirs / -l libs, e.g. from
# precompiled vendor blobs)
link_dirs: list[Path] = field(default_factory=list)
link_libs: list[str] = field(default_factory=list)
def _library_info(name: str, read_path: Path, data: dict) -> ArduinoLibrary:
"""Resolve one library's sources, include dirs, and flags (PIO semantics)."""
build = data.get("build", {})
src_dir = build.get("srcDir")
if not src_dir:
for d in ("src", "Src", "."):
if (read_path / d).is_dir():
src_dir = d
break
src_filter = ensure_list(build.get("srcFilter", DEFAULT_BUILD_SRC_FILTER))
# PlatformIO shell-lexes each build.flags entry
raw_flags = [
token
for entry in ensure_list(build.get("flags", []))
for token in shlex.split(entry)
]
lib = ArduinoLibrary(name=name)
it = iter(raw_flags)
include_flags: list[str] = []
for tok in it:
if tok in ("-I", "-L", "-l"):
tok += next(it, "")
if tok.startswith("-I"):
include_flags.append(tok[2:])
elif tok.startswith("-L"):
lib.link_dirs.append((read_path / tok[2:]).resolve())
elif tok.startswith("-l"):
lib.link_libs.append(tok[2:])
else:
lib.flags.append(tok)
include_dir = build.get("includeDir", DEFAULT_BUILD_INCLUDE_DIR)
for d in [include_dir, src_dir, *include_flags]:
if d and (path := (read_path / d)).is_dir():
lib.include_dirs.append(path.resolve())
if src_dir:
lib.sources = sorted(
Path(f).resolve()
for f in collect_filtered_files(read_path / src_dir, src_filter)
if Path(f).suffix in SRC_FILE_EXTENSIONS
)
return lib
def _bundled_library(framework_path: Path, name: str) -> ArduinoLibrary:
"""A library bundled with the Arduino core, read from the framework tree."""
lib_dir = framework_path / "libraries" / name
manifest = lib_dir / "library.properties"
data = parse_library_properties(manifest) if manifest.is_file() else {}
return _library_info(name, lib_dir, {"name": name, **data})
def resolve_libraries(framework_path: Path) -> list[ArduinoLibrary]:
"""Resolve every ``cg.add_library()`` entry into an :class:`ArduinoLibrary`."""
bundled: list[ArduinoLibrary] = []
external: list[Library] = []
for library in CORE.platformio_libraries.values():
if library.repository or not library.name or "/" in library.name:
external.append(library)
elif (framework_path / "libraries" / library.name).is_dir():
bundled.append(_bundled_library(framework_path, library.name))
else:
# A bare name that is not a bundled library ("Updater" from the
# ota component) has nothing to build; PlatformIO's LDF-off mode
# ignores it the same way.
_LOGGER.debug("Skipping unknown bundled library %s", library.name)
converted: list[ArduinoLibrary] = []
bundled_names = {lib.name for lib in bundled}
def _add_bundled_dependencies(component: ConvertedLibrary) -> None:
# A version-less bare-name dependency ("Hash" in ESPAsyncWebServer)
# is a core-bundled library; the shared converter skips it because
# it cannot be resolved from the registry.
for dep in normalize_dependencies(component.data.get("dependencies")):
name = dep.get("name")
if (
not name
or dep.get("owner")
or "version" in dep
or name in bundled_names
or not (framework_path / "libraries" / name).is_dir()
):
continue
try:
check_library_data(dep, ESP8266_PLATFORM, "arduino")
except InvalidLibrary:
continue
bundled_names.add(name)
bundled.append(_bundled_library(framework_path, name))
def _emit(component: ConvertedLibrary) -> None:
apply_extra_script(component, "esp8266")
converted.append(
_library_info(
component.get_require_name(), component.source_dir, component.data
)
)
_add_bundled_dependencies(component)
if external:
convert_libraries(
external,
LibraryBackend(
platform=ESP8266_PLATFORM,
framework="arduino",
emit=_emit,
cache_key="arduino8266",
),
)
return bundled + converted
+270
View File
@@ -0,0 +1,270 @@
"""Download and install the Arduino ESP8266 core, toolchain, and ninja.
Artifacts land in a machine-global cache (shared across projects, like the
ESP-IDF install in ``esphome.espidf.framework``):
<cache>/arduino8266/frameworks/<version>/ framework-arduinoespressif8266
<cache>/arduino8266/toolchains/<version>/ toolchain-xtensa (gcc 10.3)
<cache>/arduino8266/tools/ninja/ ninja (only if not on PATH)
Sources default to the PlatformIO registry (the exact packages the PlatformIO
toolchain has always used, so the bits are identical); the
``ESPHOME_ARDUINO8266_*_MIRRORS`` environment variables override the URLs with
``{VERSION}`` / ``{SYSTEM}`` substitution.
"""
from __future__ import annotations
import functools
import logging
import os
from pathlib import Path
import platform
import shutil
import stat
import tempfile
import platformdirs
import esphome.config_validation as cv
from esphome.core import CORE, EsphomeError
from esphome.framework_helpers import (
archive_extract_all,
download_from_mirrors,
rmdir,
str_to_lst_of_str,
)
from esphome.helpers import get_bool_env, get_str_env
_LOGGER = logging.getLogger(__name__)
FRAMEWORK_PACKAGE = "framework-arduinoespressif8266"
TOOLCHAIN_PACKAGE = "toolchain-xtensa"
# gcc 10.3, the toolchain Arduino core 3.x builds with. The compile flags in
# the build generator are tuned to it; treat version changes as a full
# reinstall (the install dir is keyed on the version).
TOOLCHAIN_VERSION = "2.100300.220621"
NINJA_VERSION = "1.12.1"
_REGISTRY_URL = (
"https://api.registry.platformio.org/v3/packages/platformio/tool/{package}"
)
ESPHOME_ARDUINO8266_FRAMEWORK_MIRRORS = str_to_lst_of_str(
os.environ.get("ESPHOME_ARDUINO8266_FRAMEWORK_MIRRORS", "")
)
ESPHOME_ARDUINO8266_TOOLCHAIN_MIRRORS = str_to_lst_of_str(
os.environ.get("ESPHOME_ARDUINO8266_TOOLCHAIN_MIRRORS", "")
)
ESPHOME_ARDUINO8266_NINJA_MIRRORS = str_to_lst_of_str(
os.environ.get(
"ESPHOME_ARDUINO8266_NINJA_MIRRORS",
"https://github.com/ninja-build/ninja/releases/download/v{VERSION}/{ARCHIVE}",
)
)
def get_arduino8266_tools_path() -> Path:
# Treat an empty/whitespace prefix as unset: Path("") resolves to the CWD,
# which clean-all would then delete.
if prefix := get_str_env("ESPHOME_ARDUINO8266_PREFIX", "").strip():
path = Path(prefix).expanduser()
else:
# Machine-global so all projects share one install; see
# espidf.framework.get_idf_tools_path for the location rationale.
path = (
Path(platformdirs.user_cache_dir("esphome", appauthor=False))
/ "arduino8266"
)
return path.resolve()
def framework_package_version(ver: cv.Version) -> str:
"""Map an Arduino core version (e.g. 3.1.2) to its package version.
Same encoding as the PlatformIO package registry uses for core 3.x
releases (3.1.2 -> 3.30102.0). The native toolchain only supports core
>= 3.1.0, so the 1.x/2.x encodings never apply here.
"""
return f"3.{ver.major}{ver.minor:02d}{ver.patch:02d}.0"
def get_framework_path(package_version: str) -> Path:
return get_arduino8266_tools_path() / "frameworks" / package_version
def get_toolchain_path() -> Path:
return get_arduino8266_tools_path() / "toolchains" / TOOLCHAIN_VERSION
def _pio_system() -> str:
"""The PlatformIO registry system tag for the current host.
Hand-rolled instead of ``platformio.util.get_systype()`` so this backend
never imports the PlatformIO package. The windows-arm64 and darwin-arm64
mappings are deliberate: the toolchain packages ship x86_64 binaries for
those hosts (Rosetta / x86 emulation).
"""
sysname = platform.system().lower()
machine = platform.machine().lower()
if sysname == "darwin":
return "darwin_arm64" if machine == "arm64" else "darwin_x86_64"
if sysname == "windows":
return "windows_amd64" if machine in ("amd64", "arm64") else "windows_x86"
if machine in ("arm64", "aarch64"):
return "linux_aarch64"
if machine in ("i686", "i386", "x86"):
return "linux_i686"
if machine.startswith("arm"):
return f"linux_{machine}"
return "linux_x86_64"
def _registry_download_url(package: str, version: str) -> str:
"""Resolve a package's download URL for this host via the PIO registry."""
import requests
url = _REGISTRY_URL.format(package=package)
resp = requests.get(url, timeout=30)
resp.raise_for_status()
data = resp.json()
system = _pio_system()
for ver in data.get("versions", []):
if ver.get("name") != version:
continue
for file in ver.get("files", []):
systems = file.get("system") or "*"
if systems == "*" or system in systems:
return file["download_url"]
raise EsphomeError(f"No {package} {version} build for this platform ({system})")
raise EsphomeError(f"{package} {version} not found in the package registry")
def _install_package(
name: str,
version: str,
dest: Path,
mirrors: list[str],
) -> None:
"""Download and extract one package if not already installed."""
marker = dest / ".esphome_extracted"
if marker.is_file():
return
rmdir(dest, msg=f"Clean up incomplete {name} install")
with tempfile.NamedTemporaryFile() as tmp:
_LOGGER.info("Downloading %s %s ...", name, version)
if mirrors:
download_from_mirrors(
mirrors, {"VERSION": version, "SYSTEM": _pio_system()}, tmp.file
)
else:
download_from_mirrors([_registry_download_url(name, version)], {}, tmp.file)
_LOGGER.info("Extracting %s ...", name)
archive_extract_all(tmp.file, dest, progress_header="Extracting")
marker.touch()
def _ninja_archive_name() -> str:
sysname = platform.system().lower()
machine = platform.machine().lower()
if sysname == "darwin":
return "ninja-mac.zip"
if sysname == "windows":
return "ninja-winarm64.zip" if machine == "arm64" else "ninja-win.zip"
if machine in ("arm64", "aarch64"):
return "ninja-linux-aarch64.zip"
return "ninja-linux.zip"
def _check_ninja_install() -> Path:
"""Return a usable ninja binary, downloading one if none is on PATH."""
if ninja := shutil.which("ninja"):
return Path(ninja)
ninja_dir = get_arduino8266_tools_path() / "tools" / "ninja" / NINJA_VERSION
binary = ninja_dir / ("ninja.exe" if os.name == "nt" else "ninja")
if binary.is_file():
return binary
rmdir(ninja_dir, msg="Clean up incomplete ninja install")
with tempfile.NamedTemporaryFile() as tmp:
_LOGGER.info("Downloading ninja %s ...", NINJA_VERSION)
download_from_mirrors(
ESPHOME_ARDUINO8266_NINJA_MIRRORS,
{"VERSION": NINJA_VERSION, "ARCHIVE": _ninja_archive_name()},
tmp.file,
)
archive_extract_all(tmp.file, ninja_dir)
if not binary.is_file():
raise EsphomeError(f"ninja binary missing after extraction in {ninja_dir}")
binary.chmod(binary.stat().st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)
return binary
def check_and_install(framework_version: cv.Version) -> dict[str, Path]:
"""Ensure framework, toolchain, and ninja are installed; return their paths."""
package_version = framework_package_version(framework_version)
framework_path = get_framework_path(package_version)
_install_package(
FRAMEWORK_PACKAGE,
package_version,
framework_path,
ESPHOME_ARDUINO8266_FRAMEWORK_MIRRORS,
)
toolchain_path = get_toolchain_path()
_install_package(
TOOLCHAIN_PACKAGE,
TOOLCHAIN_VERSION,
toolchain_path,
ESPHOME_ARDUINO8266_TOOLCHAIN_MIRRORS,
)
return {
"framework_path": framework_path,
"toolchain_path": toolchain_path,
"ninja_path": _check_ninja_install(),
}
def get_build_env(toolchain_path: Path) -> dict[str, str]:
env = os.environ.copy()
env["PATH"] = str(toolchain_path / "bin") + os.pathsep + env.get("PATH", "")
env.update(ccache_env())
return env
@functools.cache
def ccache_path() -> str | None:
"""The ccache binary to prefix compiles with, or None when disabled.
Same opt-out convention as the PlatformIO path: on by default when the
binary is on PATH, ``ESPHOME_CCACHE_ENABLE=0`` disables it.
"""
from esphome.platformio.toolchain import _ccache_runs, _strip_win_long_path_prefix
if "ESPHOME_CCACHE_ENABLE" in os.environ and not get_bool_env(
"ESPHOME_CCACHE_ENABLE"
):
return None
ccache = shutil.which("ccache")
if ccache is None:
return None
ccache = _strip_win_long_path_prefix(ccache)
return ccache if _ccache_runs(ccache) else None
def ccache_env() -> dict[str, str]:
"""Return ccache settings for the build subprocess (not os.environ).
Mirrors ``espidf.framework._ccache_env``: cache under the machine-global
tools dir, depend mode (gcc emits depfiles via -MMD), and CCACHE_BASEDIR
scoped to the build dir so devices share framework cache entries. Values
the user already set in the environment are respected.
"""
if ccache_path() is None:
return {}
defaults = {
"CCACHE_DIR": str(get_arduino8266_tools_path() / "ccache"),
"CCACHE_NOHASHDIR": "true",
"CCACHE_DEPEND": "1",
"CCACHE_BASEDIR": str(Path(CORE.build_path).resolve()),
}
return {k: v for k, v in defaults.items() if k not in os.environ}
+146
View File
@@ -0,0 +1,146 @@
"""Native Arduino ESP8266 build driver (the PlatformIO ``run`` equivalent)."""
from __future__ import annotations
import logging
from pathlib import Path
import re
import subprocess
from esphome.arduino8266 import framework
from esphome.const import (
CONF_COMPILE_PROCESS_LIMIT,
CONF_ESPHOME,
KEY_CORE,
KEY_FRAMEWORK_VERSION,
)
from esphome.core import CORE
from esphome.helpers import write_file_if_changed
from esphome.types import ConfigType
_LOGGER = logging.getLogger(__name__)
# ESP8266 user RAM (matches upload.maximum_ram_size in every board manifest)
_MAX_RAM_SIZE = 81920
_RAM_SECTIONS = (".data", ".rodata", ".bss")
_FLASH_SECTIONS = (".irom0.text", ".text", ".text1", ".data", ".rodata")
def get_build_dir() -> Path:
return CORE.relative_pioenvs_path(CORE.name)
def get_elf_path() -> Path:
return get_build_dir() / "firmware.elf"
def get_addr2line_path() -> Path:
return framework.get_toolchain_path() / "bin" / "xtensa-lx106-elf-addr2line"
def run_compile(config: ConfigType, verbose: bool) -> int:
from esphome.build_gen import arduino8266 as build_gen
paths = framework.check_and_install(CORE.data[KEY_CORE][KEY_FRAMEWORK_VERSION])
build_gen.write_project(paths)
build_dir = get_build_dir()
env = framework.get_build_env(paths["toolchain_path"])
cmd = [str(paths["ninja_path"]), "-C", str(build_dir)]
if verbose:
cmd.append("-v")
if jobs := config[CONF_ESPHOME].get(CONF_COMPILE_PROCESS_LIMIT):
cmd += ["-j", str(jobs)]
_LOGGER.debug("Running: %s", " ".join(cmd))
rc = subprocess.run(cmd, env=env, check=False, close_fds=False).returncode
if rc != 0:
return rc
_write_compile_commands(paths["ninja_path"], build_dir, env)
_print_size_summary(build_dir, paths["toolchain_path"])
get_idedata()
return 0
def _write_compile_commands(
ninja_path: Path, build_dir: Path, env: dict[str, str]
) -> None:
result = subprocess.run(
[str(ninja_path), "-C", str(build_dir), "-t", "compdb", "cc", "cxx", "asm"],
env=env,
capture_output=True,
text=True,
check=False,
close_fds=False,
)
if result.returncode == 0:
# write_file_if_changed keeps the mtime stable on no-op builds so the
# idedata cache in get_idedata() stays valid.
write_file_if_changed(build_dir / "compile_commands.json", result.stdout)
else:
_LOGGER.warning("Could not generate compile_commands.json: %s", result.stderr)
def _parse_app_size(build_dir: Path) -> int | None:
"""Read the app flash budget (irom0_0_seg length) from the linker script."""
from esphome.build_gen.arduino8266 import get_flash_ld_path
appsize_re = re.compile(r"irom0_0_seg\s*:.+len\s*=\s*(0x[\da-f]+)", re.IGNORECASE)
try:
ld_text = get_flash_ld_path(build_dir).read_text(encoding="utf-8")
except OSError:
return None
for line in ld_text.splitlines():
if match := appsize_re.search(line):
return int(match.group(1), 16)
return None
def _print_size_summary(build_dir: Path, toolchain_path: Path) -> None:
"""Print the PlatformIO-shaped RAM/Flash lines.
The exact shape (including the bar) is parsed by
``script/ci_memory_impact_extract.py``; ``format_bar`` matches it.
"""
from esphome.espidf.size_summary import format_bar
size_tool = toolchain_path / "bin" / "xtensa-lx106-elf-size"
result = subprocess.run(
[str(size_tool), "-A", "-d", str(get_elf_path())],
capture_output=True,
text=True,
check=False,
close_fds=False,
)
if result.returncode != 0:
return
sections: dict[str, int] = {}
for line in result.stdout.splitlines():
parts = line.split()
if len(parts) >= 2 and parts[0].startswith("."):
try:
sections[parts[0]] = int(parts[1])
except ValueError:
continue
ram = sum(sections.get(s, 0) for s in _RAM_SECTIONS)
flash = sum(sections.get(s, 0) for s in _FLASH_SECTIONS)
print(f"RAM: {format_bar(ram, _MAX_RAM_SIZE)}")
if app_size := _parse_app_size(build_dir):
print(f"Flash: {format_bar(flash, app_size)}")
def get_idedata() -> dict | None:
"""Derive idedata from the build's compile_commands.json.
Same contract as ``espidf.toolchain.get_idedata``: the fields IDE
integrations, clang-tidy, and the memory analyzer expect.
"""
from esphome.espidf.idedata import load_or_build_idedata
return load_or_build_idedata(
get_build_dir() / "compile_commands.json",
get_elf_path(),
CORE.relative_internal_path("idedata", f"{CORE.name}.json"),
)
+576
View File
@@ -0,0 +1,576 @@
"""Native ninja build generator for the ESP8266 Arduino core.
Transliterates the PlatformIO build spec for the Arduino ESP8266 framework
(``framework-arduinoespressif8266/tools/platformio-build.py`` plus
``platform-espressif8266/builder/main.py``) into a ``build.ninja`` under
``.pioenvs/<name>/``. The flag sets, defines, link line, linker-script
generation, and ``elf2bin`` invocation deliberately match what PlatformIO
produces so the binaries stay near-identical between the two toolchains.
The ``PIO_FRAMEWORK_ARDUINO_*`` knob defines (lwIP variant, NONOS SDK
version, MMU layout, exceptions, waveform phase) keep working: they are read
from the build flags with the same precedence as the PlatformIO builder.
"""
from __future__ import annotations
from dataclasses import dataclass, field
import os
from pathlib import Path
import re
import subprocess
import sys
from esphome.components.esp8266.boards import (
BOARDS,
ESP8266_BOARD_BUILD,
ESP8266_LD_SCRIPTS,
)
from esphome.components.esp8266.build_surgery import (
apply_testing_memory_patches,
relocate_ratetable,
)
from esphome.components.esp8266.const import (
KEY_BOARD,
KEY_ESP8266,
KEY_FLASH_MODE,
KEY_FLASH_SIZE,
KEY_SCANF_FLOAT,
)
from esphome.const import KEY_CORE, KEY_FRAMEWORK_VERSION
from esphome.core import CORE, EsphomeError
from esphome.framework_helpers import get_project_cxx_compile_flags
from esphome.helpers import mkdir_p, write_file_if_changed
_RULE_FOR_SUFFIX = {".c": "cc", ".cpp": "cxx", ".S": "asm"}
# Always excluded from the core build: ESPHome uses its own native OTA
# backend, so the Arduino Updater (and its 228-byte global) never links.
_CORE_EXCLUDE_ALWAYS = {"Updater.cpp"}
# Excluded when no component called require_waveform(); waveform_stubs.cpp
# supplies the stopWaveform()/_stopPWM() stubs digitalWrite needs.
_CORE_EXCLUDE_WAVEFORM = {
"core_esp8266_waveform_pwm.cpp",
"core_esp8266_waveform_phase.cpp",
}
# From platformio-build.py, in its order of precedence (first is the default).
_NONOSDK_VERSIONS = (
("SDK22x_190703", "NONOSDK22x_190703"),
("SDK221", "NONOSDK221"),
("SDK22x_190313", "NONOSDK22x_190313"),
("SDK22x_191024", "NONOSDK22x_191024"),
("SDK22x_191105", "NONOSDK22x_191105"),
("SDK22x_191122", "NONOSDK22x_191122"),
("SDK305", "NONOSDK305"),
)
# knob define -> (TCP_MSS, LWIP_FEATURES, LWIP_IPV6, library name)
_LWIP_VARIANTS = (
("PIO_FRAMEWORK_ARDUINO_LWIP2_IPV6_LOW_MEMORY", (536, 1, 1, "lwip6-536-feat")),
(
"PIO_FRAMEWORK_ARDUINO_LWIP2_IPV6_HIGHER_BANDWIDTH",
(1460, 1, 1, "lwip6-1460-feat"),
),
("PIO_FRAMEWORK_ARDUINO_LWIP2_HIGHER_BANDWIDTH", (1460, 1, 0, "lwip2-1460-feat")),
("PIO_FRAMEWORK_ARDUINO_LWIP2_LOW_MEMORY_LOW_FLASH", (536, 0, 0, "lwip2-536")),
(
"PIO_FRAMEWORK_ARDUINO_LWIP2_HIGHER_BANDWIDTH_LOW_FLASH",
(1460, 0, 0, "lwip2-1460"),
),
)
_LWIP_DEFAULT = (536, 1, 0, "lwip2-536-feat")
_ASFLAGS = ["-mlongcalls", "-mtext-section-literals"]
_CFLAGS = [
"-std=gnu17",
"-Wpointer-arith",
"-Wno-implicit-function-declaration",
"-Wl,-EL",
"-fno-inline-functions",
"-nostdlib",
]
_CCFLAGS = [
"-Os",
"-mlongcalls",
"-mtext-section-literals",
"-falign-functions=4",
"-U__STRICT_ANSI__",
"-ffunction-sections",
"-fdata-sections",
"-Wall",
"-Werror=return-type",
"-free",
"-fipa-pta",
]
_LINKFLAGS = [
"-Os",
"-nostdlib",
"-Wl,--no-check-sections",
"-Wl,-static",
"-Wl,--gc-sections",
"-Wl,-wrap,system_restart_local",
"-Wl,-wrap,spi_flash_read",
"-u",
"app_entry",
"-u",
"_printf_float",
"-u",
"_DebugExceptionVector",
"-u",
"_DoubleExceptionVector",
"-u",
"_KernelExceptionVector",
"-u",
"_NMIExceptionVector",
"-u",
"_UserExceptionVector",
]
_SYSTEM_LIBS_PRE_LWIP = ["hal", "phy", "pp", "net80211"]
_SYSTEM_LIBS_POST_LWIP = [
"wpa",
"crypto",
"main",
"wps",
"bearssl",
"espnow",
"smartconfig",
"airkiss",
"wpa2",
]
@dataclass
class _BuildConfig:
"""Knob-derived build configuration (PIO_FRAMEWORK_ARDUINO_* defines)."""
nonosdk: str
lwip_lib: str
exceptions: bool
vtables: str
fp_in_irom: bool
knob_defines: list[str] = field(default_factory=list)
mmu_defines: list[str] = field(default_factory=list)
def _flag_defines() -> dict[str, str]:
"""Map define name -> full ``NAME[=VALUE]`` for every -D build flag."""
defines: dict[str, str] = {}
for flag in CORE.build_flags:
if flag.startswith("-D"):
body = flag[2:]
defines[body.split("=", 1)[0]] = body
return defines
def _resolve_build_config(defines: dict[str, str]) -> _BuildConfig:
nonosdk = _NONOSDK_VERSIONS[0][1]
for name, define in _NONOSDK_VERSIONS:
if f"PIO_FRAMEWORK_ARDUINO_ESPRESSIF_{name}" in defines:
nonosdk = define
tcp_mss, features, ipv6, lwip_lib = _LWIP_DEFAULT
for knob, variant in _LWIP_VARIANTS:
if knob in defines:
tcp_mss, features, ipv6, lwip_lib = variant
break
knob_defines = [
f"{nonosdk}=1",
f"TCP_MSS={tcp_mss}",
f"LWIP_FEATURES={features}",
f"LWIP_IPV6={ipv6}",
]
if "PIO_FRAMEWORK_ARDUINO_WAVEFORM_LOCKED_PHASE" in defines:
knob_defines.append("WAVEFORM_LOCKED_PHASE=1")
vtables = next(
(name for name in defines if name.startswith("VTABLES_IN_")),
"VTABLES_IN_FLASH",
)
if "PIO_FRAMEWORK_ARDUINO_MMU_CACHE16_IRAM48" in defines:
mmu = ["MMU_IRAM_SIZE=0xC000", "MMU_ICACHE_SIZE=0x4000"]
elif "PIO_FRAMEWORK_ARDUINO_MMU_CACHE16_IRAM48_SECHEAP_SHARED" in defines:
mmu = ["MMU_IRAM_SIZE=0xC000", "MMU_ICACHE_SIZE=0x4000", "MMU_IRAM_HEAP"]
elif "PIO_FRAMEWORK_ARDUINO_MMU_CACHE16_IRAM32_SECHEAP_NOTSHARED" in defines:
mmu = [
"MMU_IRAM_SIZE=0x8000",
"MMU_ICACHE_SIZE=0x4000",
"MMU_SEC_HEAP_SIZE=0x4000",
"MMU_SEC_HEAP=0x40108000",
]
elif "PIO_FRAMEWORK_ARDUINO_MMU_EXTERNAL_128K" in defines:
mmu = [
"MMU_IRAM_SIZE=0x8000",
"MMU_ICACHE_SIZE=0x8000",
"MMU_EXTERNAL_HEAP=128",
]
elif "PIO_FRAMEWORK_ARDUINO_MMU_EXTERNAL_1024K" in defines:
mmu = [
"MMU_IRAM_SIZE=0x8000",
"MMU_ICACHE_SIZE=0x8000",
"MMU_EXTERNAL_HEAP=256",
]
elif "PIO_FRAMEWORK_ARDUINO_MMU_CUSTOM" in defines:
if "MMU_IRAM_SIZE" not in defines or "MMU_ICACHE_SIZE" not in defines:
raise EsphomeError(
"PIO_FRAMEWORK_ARDUINO_MMU_CUSTOM requires MMU_IRAM_SIZE and "
"MMU_ICACHE_SIZE build flags"
)
mmu = [body for name, body in defines.items() if name.startswith("MMU_")]
else:
mmu = ["MMU_IRAM_SIZE=0x8000", "MMU_ICACHE_SIZE=0x8000"]
return _BuildConfig(
nonosdk=nonosdk,
lwip_lib=lwip_lib,
exceptions="PIO_FRAMEWORK_ARDUINO_ENABLE_EXCEPTIONS" in defines,
vtables=vtables,
fp_in_irom="FP_IN_IROM" in defines,
knob_defines=knob_defines,
mmu_defines=mmu,
)
def _flash_ld_name(board: str) -> str:
return ESP8266_LD_SCRIPTS[BOARDS[board][KEY_FLASH_SIZE]][1]
def _e(value) -> str:
"""Escape a path or token for a ninja file."""
return str(value).replace("$", "$$").replace(":", "$:").replace(" ", "$ ")
def _q(value) -> str:
"""Quote a path for use inside a ninja command line (shell/CreateProcess)."""
return f'"{value}"'
def _defines_flags(
config: _BuildConfig, flash_mode: str, board: str, board_defines: tuple[str, ...]
) -> list[str]:
return [
f"-D{d}"
for d in (
"F_CPU=80000000L",
"__ets__",
"ICACHE_FLASH",
"_GNU_SOURCE",
"ARDUINO=10805",
f'ARDUINO_BOARD=\\"PLATFORMIO_{board.upper()}\\"',
f'ARDUINO_BOARD_ID=\\"{board}\\"',
f"FLASHMODE_{flash_mode.upper()}",
"LWIP_OPEN_SRC",
*config.knob_defines,
config.vtables,
*config.mmu_defines,
"ESP8266",
"ARDUINO_ARCH_ESP8266",
*board_defines,
)
]
def _project_flags() -> tuple[list[str], list[str]]:
"""Split the ESPHome build flags into (compile, linker) lists.
Unlike ``framework_helpers.get_project_compile_flags`` this keeps every
non-linker flag, matching how PlatformIO passes ``build_flags`` to the
compiler verbatim, and applies ``build_unflags``.
"""
unflags = set(CORE.build_unflags)
flags = [f for f in sorted(CORE.build_flags) if f not in unflags]
compile_flags = [f for f in flags if not f.startswith("-Wl,")]
link_flags = [f for f in flags if f.startswith("-Wl,")]
return compile_flags, link_flags
def _collect_sources(root: Path, exclude: set[str] = frozenset()) -> list[Path]:
return sorted(
p
for p in root.rglob("*")
if p.suffix in _RULE_FOR_SUFFIX and p.name not in exclude
)
def generate_ld_scripts(
paths: dict[str, Path], config: _BuildConfig, flash_ld_name: str
) -> None:
"""Generate the common linker script (and testing-mode flash ld copy).
Runs the same preprocessor invocation as the PlatformIO builder over
``eagle.app.v6.common.ld.h``, then applies ESPHome's surgeries: the wifi
rate-table DRAM relocation, and enlarged memory segments in testing mode.
"""
framework = paths["framework_path"]
gcc = paths["toolchain_path"] / "bin" / "xtensa-lx106-elf-gcc"
ld_dir = CORE.relative_pioenvs_path(CORE.name, "ld")
mkdir_p(ld_dir)
cmd = [str(gcc), "-CC", "-E", "-P", f"-D{config.vtables}"]
cmd += [f"-D{d}" for d in config.mmu_defines]
if config.fp_in_irom:
cmd.append("-DFP_IN_IROM")
cmd += [
str(framework / "tools" / "sdk" / "ld" / "eagle.app.v6.common.ld.h"),
"-o",
"-",
]
# The inputs are the command line (defines + framework version, which is
# baked into the paths) plus testing mode; skip the preprocessor spawn on
# incremental builds when nothing changed.
output = ld_dir / "local.eagle.app.v6.common.ld"
stamp = ld_dir / ".local.eagle.app.v6.common.ld.stamp"
stamp_content = " ".join(cmd) + f" testing={CORE.testing_mode}"
if not (
output.is_file()
and stamp.is_file()
and stamp.read_text(encoding="utf-8") == stamp_content
):
result = subprocess.run(
cmd, capture_output=True, text=True, check=False, close_fds=False
)
if result.returncode != 0:
raise EsphomeError(f"Generating the linker script failed:\n{result.stderr}")
content = relocate_ratetable(result.stdout)
if CORE.testing_mode:
content = apply_testing_memory_patches(content)
write_file_if_changed(output, content)
stamp.write_text(stamp_content, encoding="utf-8")
if CORE.testing_mode:
# A patched copy of the flash ld in the build dir; resolved through
# the same -L path as the SDK original it shadows.
flash_ld = framework / "tools" / "sdk" / "ld" / flash_ld_name
write_file_if_changed(
ld_dir / f"testing_{flash_ld_name}",
apply_testing_memory_patches(flash_ld.read_text(encoding="utf-8")),
)
def _ninja_compile_edges(
lines: list[str],
sources: list[Path],
root: Path,
group: str,
flags: str = "",
) -> list[str]:
"""Emit compile edges for ``sources``; return the object paths."""
objects = []
for src in sources:
rel = src.relative_to(root).as_posix()
obj = f"obj/{group}/{rel}.o"
lines.append(f"build {_e(obj)}: {_RULE_FOR_SUFFIX[src.suffix]} {_e(src)}")
if flags:
lines.append(f" flags = {flags}")
objects.append(obj)
return objects
def _common_parent(paths: list[Path]) -> Path:
return Path(os.path.commonpath([str(p.parent) for p in paths]))
def write_project(paths: dict[str, Path]) -> None:
"""Write the ninja build for the current configuration."""
from esphome.arduino8266.component import resolve_libraries
from esphome.arduino8266.framework import ccache_path
framework = paths["framework_path"]
toolchain_bin = paths["toolchain_path"] / "bin"
build_dir = CORE.relative_pioenvs_path(CORE.name)
mkdir_p(build_dir)
flag_defines = _flag_defines()
config = _resolve_build_config(flag_defines)
esp8266_data = CORE.data[KEY_ESP8266]
# Board support was validated at config time (_validate_native_toolchain).
board = esp8266_data[KEY_BOARD]
board_build = ESP8266_BOARD_BUILD[board]
flash_ld_name = _flash_ld_name(board)
generate_ld_scripts(paths, config, flash_ld_name)
sdk = framework / "tools" / "sdk"
core_dir = framework / "cores" / "esp8266"
variant_dir = framework / "variants" / board_build["variant"]
src_dir = CORE.relative_src_path()
libraries = resolve_libraries(framework)
include_dirs = [
src_dir,
sdk / "include",
core_dir,
paths["toolchain_path"] / "include",
sdk / "lwip2" / "include",
variant_dir,
]
include_dirs = [d for d in include_dirs if d.is_dir()]
for lib in libraries:
include_dirs += lib.include_dirs
project_compile_flags, project_link_flags = _project_flags()
defines = _defines_flags(
config, esp8266_data[KEY_FLASH_MODE], board, board_build["defines"]
)
includes = [f"-I{_q(d)}" for d in include_dirs]
common = _CCFLAGS + defines + includes + project_compile_flags
cflags = _CFLAGS + common
cpp_standard = CORE.cpp_standard or "gnu++17"
cxxflags = (
["-fno-rtti", f"-std={cpp_standard}"]
+ ["-fexceptions" if config.exceptions else "-fno-exceptions"]
+ common
+ get_project_cxx_compile_flags()
)
asflags = _ASFLAGS + defines + includes + project_compile_flags
link_flags = list(_LINKFLAGS)
if esp8266_data.get(KEY_SCANF_FLOAT):
link_flags += ["-u", "_scanf_float"]
link_flags += project_link_flags
flash_ld = f"testing_{flash_ld_name}" if CORE.testing_mode else flash_ld_name
link_flags += ["-T", flash_ld]
lib_dirs = [Path("ld"), sdk / "lib", sdk / "ld", sdk / "lib" / config.nonosdk]
for lib in libraries:
lib_dirs += lib.link_dirs
system_libs = (
_SYSTEM_LIBS_PRE_LWIP
+ [config.lwip_lib]
+ _SYSTEM_LIBS_POST_LWIP
+ [lib_name for lib in libraries for lib_name in lib.link_libs]
+ ["stdc++-exc" if config.exceptions else "stdc++", "m", "c", "gcc"]
)
build_tool = Path(__file__).parent.parent / "arduino8266" / "build_tool.py"
ccache = ccache_path()
lines = [
"# Auto-generated by ESPHome",
"ninja_required_version = 1.5",
f"cc = {_q(toolchain_bin / 'xtensa-lx106-elf-gcc')}",
f"cxx = {_q(toolchain_bin / 'xtensa-lx106-elf-g++')}",
f"python = {_q(sys.executable)}",
f"buildtool = {_q(build_tool)}",
f"ccache = {_q(ccache) if ccache else ''}",
"",
"rule cc",
" command = $ccache $cc -MMD -MF $out.d $cflags $flags -c $in -o $out",
" depfile = $out.d",
" deps = gcc",
" description = CC $out",
"rule cxx",
" command = $ccache $cxx -MMD -MF $out.d $cxxflags $flags -c $in -o $out",
" depfile = $out.d",
" deps = gcc",
" description = CXX $out",
"rule asm",
" command = $ccache $cc -MMD -MF $out.d -x assembler-with-cpp $asflags $flags -c $in -o $out",
" depfile = $out.d",
" deps = gcc",
" description = AS $out",
"rule ar",
f" command = $python $buildtool ar {_q(toolchain_bin / 'xtensa-lx106-elf-ar')} $out $out.rsp",
" rspfile = $out.rsp",
" rspfile_content = $in_newline",
" description = AR $out",
"rule link",
" command = $cxx -o $out $linkflags @$out.rsp $libdirflags -Wl,--start-group $archives $libflags -Wl,--end-group",
" rspfile = $out.rsp",
" rspfile_content = $in_newline",
" description = LINK $out",
"rule elf2bin",
f" command = $python {_q(framework / 'tools' / 'elf2bin.py')} --eboot {_q(framework / 'bootloaders' / 'eboot' / 'eboot.elf')} --app $in --flash_mode {esp8266_data[KEY_FLASH_MODE]} --flash_freq 40 --flash_size {_flash_size_str(flash_ld_name)} --path {_q(toolchain_bin)} --out $out",
" description = BIN $out",
"rule copy",
" command = $python $buildtool copy $in $out",
" description = COPY $out",
"",
f"cflags = {' '.join(cflags)}",
f"cxxflags = {' '.join(cxxflags)}",
f"asflags = {' '.join(asflags)}",
f"linkflags = {' '.join(link_flags)}",
f"libdirflags = {' '.join(f'-L{_q(d)}' for d in lib_dirs)}",
f"libflags = {' '.join(f'-l{lib}' for lib in system_libs)}",
"",
]
core_exclude = set(_CORE_EXCLUDE_ALWAYS)
if "USE_ESP8266_WAVEFORM_STUBS" in flag_defines:
core_exclude |= _CORE_EXCLUDE_WAVEFORM
archives = []
variant_sources = _collect_sources(variant_dir) if variant_dir.is_dir() else []
if variant_sources:
objs = _ninja_compile_edges(lines, variant_sources, variant_dir, "variant")
lines.append(f"build libFrameworkArduinoVariant.a: ar {' '.join(objs)}")
archives.append("libFrameworkArduinoVariant.a")
core_objs = _ninja_compile_edges(
lines, _collect_sources(core_dir, core_exclude), core_dir, "core"
)
lines.append(f"build libFrameworkArduino.a: ar {' '.join(core_objs)}")
archives.append("libFrameworkArduino.a")
for lib in libraries:
if not lib.sources:
continue
lib_root = _common_parent(lib.sources)
objs = _ninja_compile_edges(
lines,
lib.sources,
lib_root,
f"lib/{lib.name}",
flags=" ".join(lib.flags),
)
archive = f"lib{lib.name}.a"
lines.append(f"build {_e(archive)}: ar {' '.join(objs)}")
archives.append(archive)
src_extra = f"-include {_q(src_dir / 'esphome' / 'components' / 'esp8266' / 'throw_stubs.h')}"
src_objs = _ninja_compile_edges(
lines, _collect_sources(src_dir), src_dir, "src", flags=src_extra
)
ld_deps = ["ld/local.eagle.app.v6.common.ld"]
if CORE.testing_mode:
ld_deps.append(f"ld/{flash_ld}")
lines.append(
f"build firmware.elf: link {' '.join(src_objs)} | "
f"{' '.join(_e(a) for a in archives)} {' '.join(_e(d) for d in ld_deps)}"
)
lines.append(f" archives = {' '.join(archives)}")
lines.append("build firmware.bin: elf2bin firmware.elf")
lines.append("build firmware.factory.bin: copy firmware.bin")
lines.append("build firmware.ota.bin: copy firmware.bin")
lines.append("default firmware.factory.bin firmware.ota.bin")
lines.append("")
write_file_if_changed(build_dir / "build.ninja", "\n".join(lines))
def get_flash_ld_path(build_dir: Path) -> Path:
"""The flash linker script the link actually uses (for size reporting)."""
from esphome.arduino8266.framework import (
framework_package_version,
get_framework_path,
)
name = _flash_ld_name(CORE.data[KEY_ESP8266][KEY_BOARD])
if CORE.testing_mode:
return build_dir / "ld" / f"testing_{name}"
version = framework_package_version(CORE.data[KEY_CORE][KEY_FRAMEWORK_VERSION])
return get_framework_path(version) / "tools" / "sdk" / "ld" / name
def _flash_size_str(flash_ld_name: str) -> str:
"""Flash size for elf2bin, derived from the ld script name (PIO logic)."""
match = re.search(r"\.flash\.(\d+)([mk])", flash_ld_name)
if not match:
raise EsphomeError(f"Cannot parse flash size from {flash_ld_name}")
return f"{match.group(1)}{match.group(2).upper()}"
+131 -35
View File
@@ -13,6 +13,7 @@ from esphome.const import (
CONF_FRAMEWORK,
CONF_PLATFORM_VERSION,
CONF_SOURCE,
CONF_TOOLCHAIN,
CONF_VERSION,
KEY_CORE,
KEY_FRAMEWORK_VERSION,
@@ -20,6 +21,7 @@ from esphome.const import (
KEY_TARGET_PLATFORM,
PLATFORM_ESP8266,
ThreadModel,
Toolchain,
)
from esphome.core import (
CORE,
@@ -33,7 +35,7 @@ from esphome.helpers import IS_MACOS, copy_file_if_changed
from esphome.platformio.toolchain import copy_ccache_script
from esphome.types import ConfigType
from .boards import BOARDS, ESP8266_LD_SCRIPTS
from .boards import BOARDS, ESP8266_BOARD_BUILD, ESP8266_LD_SCRIPTS
from .const import (
CONF_EARLY_PIN_INIT,
CONF_ENABLE_SERIAL,
@@ -41,8 +43,10 @@ from .const import (
CONF_RESTORE_FROM_FLASH,
KEY_BOARD,
KEY_ESP8266,
KEY_FLASH_MODE,
KEY_FLASH_SIZE,
KEY_PIN_INITIAL_STATES,
KEY_SCANF_FLOAT,
KEY_SERIAL1_REQUIRED,
KEY_SERIAL_REQUIRED,
KEY_WAVEFORM_REQUIRED,
@@ -96,12 +100,57 @@ def set_core_data(config):
config[CONF_FRAMEWORK][CONF_VERSION]
)
CORE.data[KEY_ESP8266][KEY_BOARD] = config[CONF_BOARD]
CORE.data[KEY_ESP8266][KEY_FLASH_MODE] = config[CONF_BOARD_FLASH_MODE]
CORE.data[KEY_ESP8266][KEY_PIN_INITIAL_STATES] = [
PinInitialState() for _ in range(16)
]
return config
def _validate_toolchain(value: str) -> Toolchain:
return Toolchain(
cv.one_of(Toolchain.PLATFORMIO, Toolchain.ARDUINO, lower=True)(value)
)
def _resolve_toolchain(config: ConfigType) -> ConfigType:
# Resolve toolchain: CLI (already on CORE.toolchain) > YAML > default.
if CORE.toolchain is None:
CORE.toolchain = config.get(CONF_TOOLCHAIN, Toolchain.PLATFORMIO)
if CORE.toolchain not in (Toolchain.PLATFORMIO, Toolchain.ARDUINO):
raise cv.Invalid(
f"Unsupported toolchain '{CORE.toolchain.value}' for ESP8266. "
"Supported toolchains are 'platformio' and 'arduino'."
)
return config
def _validate_native_toolchain(config: ConfigType) -> ConfigType:
"""Constraints of the native (non-PlatformIO) Arduino toolchain."""
if not CORE.using_toolchain_arduino:
return config
conf = config[CONF_FRAMEWORK]
version = cv.Version.parse(conf[CONF_VERSION])
if version < cv.Version(3, 1, 0):
raise cv.Invalid(
"'toolchain: arduino' requires framework version 3.1.0 or newer"
)
if conf[CONF_SOURCE] != _format_framework_arduino_version(version):
raise cv.Invalid(
"'toolchain: arduino' does not support a custom framework source; "
"use 'toolchain: platformio'"
)
if (
config[CONF_BOARD] not in BOARDS
or config[CONF_BOARD] not in ESP8266_BOARD_BUILD
):
raise cv.Invalid(
f"Board '{config[CONF_BOARD]}' is not supported by "
"'toolchain: arduino'; use 'toolchain: platformio'"
)
return config
def get_download_types(storage_json):
"""Binary-download entries for a built ESP8266 firmware.
@@ -134,7 +183,11 @@ def _format_framework_arduino_version(ver: cv.Version) -> str:
return f"~1.{ver.major}{ver.minor:02d}{ver.patch:02d}.0"
if ver <= cv.Version(2, 6, 2):
return f"~2.{ver.major}{ver.minor:02d}{ver.patch:02d}.0"
return f"~3.{ver.major}{ver.minor:02d}{ver.patch:02d}.0"
# Same encoding the native toolchain uses for its package download, so a
# custom-source check against this value cannot drift from what it fetches.
from esphome.arduino8266.framework import framework_package_version
return f"~{framework_package_version(ver)}"
# NOTE: Keep this in mind when updating the recommended version:
@@ -242,8 +295,13 @@ CONFIG_SCHEMA = cv.All(
cv.Optional(CONF_ENABLE_SERIAL1): cv.boolean,
cv.Optional(CONF_ENABLE_FULL_PRINTF, default=False): cv.boolean,
cv.Optional(CONF_ENABLE_SCANF_FLOAT): cv.boolean,
cv.Optional(
CONF_TOOLCHAIN, visibility=cv.Visibility.ADVANCED
): _validate_toolchain,
}
),
_resolve_toolchain,
_validate_native_toolchain,
set_core_data,
)
@@ -276,12 +334,13 @@ def check_rosetta() -> None:
@coroutine_with_priority(CoroPriority.PLATFORM)
async def to_code(config):
use_platformio = CORE.using_toolchain_platformio
cg.add(esp8266_ns.setup_preferences())
cg.add_platformio_option("lib_ldf_mode", "off")
cg.add_platformio_option("lib_compat_mode", "strict")
cg.add_platformio_option("board", config[CONF_BOARD])
if use_platformio:
cg.add_platformio_option("lib_ldf_mode", "off")
cg.add_platformio_option("lib_compat_mode", "strict")
cg.add_platformio_option("board", config[CONF_BOARD])
cg.add_build_flag("-DUSE_ESP8266")
cg.set_cpp_standard("gnu++20")
cg.add_define("ESPHOME_BOARD", config[CONF_BOARD])
@@ -297,28 +356,33 @@ async def to_code(config):
"enabling scanf float support (~8KB flash)"
)
extra_scripts = [
"pre:ccache.py",
"pre:testing_mode.py",
"pre:exclude_updater.py",
"pre:exclude_waveform.py",
"pre:relocate_ratetable.py",
]
if not enable_scanf_float:
extra_scripts.append("pre:remove_float_scanf.py")
extra_scripts.append("post:post_build.py")
cg.add_platformio_option("extra_scripts", extra_scripts)
# The native toolchain reads this decision from CORE.data instead of the
# remove_float_scanf extra script.
CORE.data[KEY_ESP8266][KEY_SCANF_FLOAT] = bool(enable_scanf_float)
if use_platformio:
extra_scripts = [
"pre:ccache.py",
"pre:testing_mode.py",
"pre:exclude_updater.py",
"pre:exclude_waveform.py",
"pre:relocate_ratetable.py",
]
if not enable_scanf_float:
extra_scripts.append("pre:remove_float_scanf.py")
extra_scripts.append("post:post_build.py")
cg.add_platformio_option("extra_scripts", extra_scripts)
conf = config[CONF_FRAMEWORK]
cg.add_platformio_option("framework", "arduino")
cg.add_build_flag("-DUSE_ARDUINO")
cg.add_build_flag("-DUSE_ESP8266_FRAMEWORK_ARDUINO")
cg.add_build_flag("-Wno-nonnull-compare")
cg.add_platformio_option("platform", conf[CONF_PLATFORM_VERSION])
cg.add_platformio_option(
"platform_packages",
[f"platformio/framework-arduinoespressif8266@{conf[CONF_SOURCE]}"],
)
if use_platformio:
cg.add_platformio_option("framework", "arduino")
cg.add_platformio_option("platform", conf[CONF_PLATFORM_VERSION])
cg.add_platformio_option(
"platform_packages",
[f"platformio/framework-arduinoespressif8266@{conf[CONF_SOURCE]}"],
)
# Default for platformio is LWIP2_LOW_MEMORY with:
# - MSS=536
@@ -356,10 +420,12 @@ async def to_code(config):
# Force-include inline std::__throw_* overrides so GCC dead-strips the unused
# libstdc++ error message strings (e.g. "basic_string::_M_create") from DRAM.
# See throw_stubs.h for details. Must be prepended before <string>, so this
# uses build_src_flags with -include.
cg.add_platformio_option(
"build_src_flags", "-include esphome/components/esp8266/throw_stubs.h"
)
# uses build_src_flags with -include. The native toolchain's build
# generator adds the equivalent flag itself.
if use_platformio:
cg.add_platformio_option(
"build_src_flags", "-include esphome/components/esp8266/throw_stubs.h"
)
# In testing mode, fake larger memory to allow linking grouped component tests
# Real ESP8266 hardware only has 32KB IRAM and ~80KB RAM, but for CI testing
@@ -386,7 +452,10 @@ async def to_code(config):
# implementation in the Arduino ESP8266 core.
cg.add_build_flag("-Wl,--wrap=millis")
cg.add_platformio_option("board_build.flash_mode", config[CONF_BOARD_FLASH_MODE])
if use_platformio:
cg.add_platformio_option(
"board_build.flash_mode", config[CONF_BOARD_FLASH_MODE]
)
ver: cv.Version = CORE.data[KEY_CORE][KEY_FRAMEWORK_VERSION]
cg.add_define(
@@ -394,7 +463,7 @@ async def to_code(config):
cg.RawExpression(f"VERSION_CODE({ver.major}, {ver.minor}, {ver.patch})"),
)
if config[CONF_BOARD] in BOARDS:
if use_platformio and config[CONF_BOARD] in BOARDS:
flash_size = BOARDS[config[CONF_BOARD]][KEY_FLASH_SIZE]
ld_scripts = ESP8266_LD_SCRIPTS[flash_size]
@@ -446,8 +515,24 @@ async def finalize_serial_config() -> None:
cg.add_build_flag("-DNO_GLOBAL_SERIAL1")
# Called by __main__.compile_program; returning False falls through to the
# PlatformIO toolchain.
def run_compile(args, config: ConfigType) -> bool:
if CORE.using_toolchain_platformio:
return False
from esphome.arduino8266 import toolchain
if toolchain.run_compile(config, CORE.verbose) != 0:
raise EsphomeError("ESP8266 native build failed")
return True
# Called by writer.py
def copy_files() -> None:
if not CORE.using_toolchain_platformio:
# The extra scripts are PlatformIO/SCons-only; the native toolchain
# applies their logic in the build generator instead.
return
dir = Path(__file__).parent
for script in (
"post_build",
@@ -505,13 +590,24 @@ ESP8266_EXCEPTION_CODES = {
def _decode_pc(config, addr):
from esphome.platformio import toolchain
if CORE.using_toolchain_arduino:
from esphome.arduino8266 import toolchain as native_toolchain
idedata = toolchain.get_idedata(config)
if not idedata.addr2line_path or not idedata.firmware_elf_path:
_LOGGER.debug("decode_pc no addr2line")
return
command = [idedata.addr2line_path, "-pfiaC", "-e", idedata.firmware_elf_path, addr]
addr2line = native_toolchain.get_addr2line_path()
elf = native_toolchain.get_elf_path()
if not addr2line.is_file() or not elf.is_file():
_LOGGER.debug("decode_pc no addr2line")
return
addr2line, elf = str(addr2line), str(elf)
else:
from esphome.platformio import toolchain
idedata = toolchain.get_idedata(config)
if not idedata.addr2line_path or not idedata.firmware_elf_path:
_LOGGER.debug("decode_pc no addr2line")
return
addr2line, elf = idedata.addr2line_path, idedata.firmware_elf_path
command = [addr2line, "-pfiaC", "-e", elf, addr]
try:
translation = subprocess.check_output(command, close_fds=False).decode().strip()
except Exception: # noqa: BLE001 # pylint: disable=broad-except
+93
View File
@@ -360,3 +360,96 @@ BOARDS = {
"flash_size": FLASH_SIZE_4_MB,
},
}
# Per-board Arduino core build metadata for the native (PlatformIO-free)
# toolchain: the variant directory (supplies pins_arduino.h) and the
# board-identity defines the PlatformIO builder passes via build.extra_flags.
# -DESP8266 and -DARDUINO_ARCH_ESP8266 are shared by every board and added by
# the generator; only the per-board defines are listed here.
# Generated from platform-espressif8266 boards/*.json (see BOARDS note above).
ESP8266_BOARD_BUILD = {
"agruminolemon": {
"variant": "agruminolemonv4",
"defines": ("ARDUINO_ESP8266_AGRUMINO_LEMON_V4",),
},
"d1": {"variant": "d1", "defines": ("ARDUINO_ESP8266_WEMOS_D1R1",)},
"d1_mini": {"variant": "d1_mini", "defines": ("ARDUINO_ESP8266_WEMOS_D1MINI",)},
"d1_mini_lite": {
"variant": "d1_mini",
"defines": ("ARDUINO_ESP8266_WEMOS_D1MINILITE",),
},
"d1_mini_pro": {
"variant": "d1_mini",
"defines": ("ARDUINO_ESP8266_WEMOS_D1MINIPRO",),
},
"d1_wroom_02": {
"variant": "d1_mini",
"defines": ("ARDUINO_ESP8266_WEMOS_D1WROOM02",),
},
"eduinowifi": {
"variant": "eduinowifi",
"defines": ("ARDUINO_ESP8266_SCHIRMILABS_EDUINO_WIFI",),
},
"esp01": {"variant": "generic", "defines": ("ARDUINO_ESP8266_ESP01",)},
"esp01_1m": {"variant": "generic", "defines": ("ARDUINO_ESP8266_ESP01",)},
"esp07": {"variant": "generic", "defines": ("ARDUINO_ESP8266_ESP07",)},
"esp07s": {"variant": "nodemcu", "defines": ("ARDUINO_ESP8266_ESP07",)},
"esp12e": {"variant": "nodemcu", "defines": ("ARDUINO_ESP8266_ESP12",)},
"esp210": {"variant": "generic", "defines": ("ARDUINO_ESP8266_ESP210",)},
"esp8285": {"variant": "esp8285", "defines": ("ARDUINO_ESP8266_ESP01",)},
"esp_wroom_02": {
"variant": "nodemcu",
"defines": ("ARDUINO_ESP8266_ESP_WROOM_02",),
},
"espduino": {"variant": "ESPDuino", "defines": ("ARDUINO_ESP8266_ESP13",)},
"espectro": {"variant": "espectro", "defines": ("ARDUINO_ESP8266_ESPECTRO_CORE",)},
"espino": {"variant": "espino", "defines": ("ARDUINO_ESP8266_ESP12",)},
"espinotee": {"variant": "espinotee", "defines": ("ARDUINO_ESP8266_ESP13",)},
"espmxdevkit": {
"variant": "esp8285",
"defines": ("ARDUINO_ESP8266_ESP01", "LED_BUILTIN=16"),
},
"espresso_lite_v1": {
"variant": "espresso_lite_v1",
"defines": ("ARDUINO_ESP8266_ESPRESSO_LITE_V1",),
},
"espresso_lite_v2": {
"variant": "espresso_lite_v2",
"defines": ("ARDUINO_ESP8266_ESPRESSO_LITE_V2",),
},
"gen4iod": {"variant": "generic", "defines": ("ARDUINO_GEN4_IOD",)},
"heltec_wifi_kit_8": {
"variant": "wifi_kit_8",
"defines": ("ARDUINO_wifi_kit_8",),
},
"huzzah": {"variant": "adafruit", "defines": ("ARDUINO_ESP8266_ADAFRUIT_HUZZAH",)},
"inventone": {"variant": "inventone", "defines": ("ARDUINO_ESP8266_INVENT_ONE",)},
"modwifi": {"variant": "generic", "defines": ("ARDUINO_MOD_WIFI_ESP8266",)},
"nodemcu": {"variant": "nodemcu", "defines": ("ARDUINO_ESP8266_NODEMCU",)},
"nodemcuv2": {"variant": "nodemcu", "defines": ("ARDUINO_ESP8266_NODEMCU_ESP12E",)},
"oak": {"variant": "oak", "defines": ("ARDUINO_ESP8266_OAK",)},
"phoenix_v1": {
"variant": "phoenix_v1",
"defines": ("ARDUINO_ESP8266_PHOENIX_V1",),
},
"phoenix_v2": {
"variant": "phoenix_v2",
"defines": ("ARDUINO_ESP8266_PHOENIX_V2",),
},
"sonoff_basic": {"variant": "itead", "defines": ("ARDUINO_ESP8266_SONOFF_BASIC",)},
"sonoff_s20": {"variant": "itead", "defines": ("ARDUINO_ESP8266_SONOFF_S20",)},
"sonoff_sv": {"variant": "itead", "defines": ("ARDUINO_ESP8266_SONOFF_SV",)},
"sonoff_th": {"variant": "itead", "defines": ("ARDUINO_ESP8266_SONOFF_TH",)},
"sparkfunBlynk": {"variant": "thing", "defines": ("ARDUINO_ESP8266_THING",)},
"thing": {"variant": "thing", "defines": ("ARDUINO_ESP8266_THING",)},
"thingdev": {"variant": "thing", "defines": ("ARDUINO_ESP8266_THING_DEV",)},
"wifi_slot": {"variant": "wifi_slot", "defines": ("ARDUINO_AMPERKA_WIFI_SLOT",)},
"wifiduino": {"variant": "wifiduino", "defines": ("ARDUINO_WIFIDUINO_ESP8266",)},
"wifinfo": {"variant": "wifinfo", "defines": ("ARDUINO_WIFINFO",)},
"wio_link": {"variant": "wiolink", "defines": ("ARDUINO_ESP8266_WIO_LINK",)},
"wio_node": {"variant": "nodemcu", "defines": ("ARDUINO_ESP8266_ESP_WROOM_02",)},
"xinabox_cw01": {
"variant": "xinabox",
"defines": ("ARDUINO_ESP8266_XINABOX_CW01",),
},
}
@@ -0,0 +1,59 @@
"""Linker-script surgery shared with the native (PlatformIO-free) toolchain.
These mirror the PlatformIO extra scripts in this directory
(``relocate_ratetable.py.script`` and ``testing_mode.py.script``), which run
inside SCons and must stay self-contained. The native build generator applies
the same patches to the linker scripts it generates, so the logic lives here
as plain functions. Keep both in sync when changing either.
"""
from __future__ import annotations
import re
# Move the NONOS SDK wifi rate tables from flash to DRAM; see
# relocate_ratetable.py.script for the full background (NONOS SDK issue 320).
RATETABLE_RULE = "*libnet80211.a:ieee80211_phy.o(.irom.text .irom.text.*)"
# Match the whole line: "_data_start" is also a substring of the
# "_dport0_data_start" line in the earlier .dport0.data section
_RATETABLE_ANCHOR = re.compile(r"^\s*_data_start = ABSOLUTE\(\.\);", re.MULTILINE)
# Memory sizes for testing mode (allow larger builds for CI component grouping)
TESTING_IRAM_SIZE = "0x200000" # 2MB
TESTING_DRAM_SIZE = "0x200000" # 2MB
TESTING_FLASH_SIZE = "0x2000000" # 32MB
def relocate_ratetable(content: str) -> str:
"""Insert the rate-table DRAM rule into a generated common linker script."""
if RATETABLE_RULE in content:
return content
match = _RATETABLE_ANCHOR.search(content)
if match is None:
raise RuntimeError(
"'_data_start' anchor not found in the generated linker script; "
"cannot apply wifi rate table DRAM relocation "
"(has the Arduino core linker script changed?)"
)
insert_pos = match.end()
return (
content[:insert_pos]
+ "\n /* ESPHome: wifi rate tables must live in DRAM, see NONOS SDK issue 320 */"
+ f"\n {RATETABLE_RULE}"
+ content[insert_pos:]
)
def _patch_segment_size(content: str, segment_name: str, new_size: str) -> str:
pattern = (
rf"({segment_name}\s*:\s*org\s*=\s*0x[0-9a-fA-F]+\s*,\s*len\s*=\s*)"
r"0x[0-9a-fA-F]+"
)
return re.sub(pattern, rf"\g<1>{new_size}", content)
def apply_testing_memory_patches(content: str) -> str:
"""Enlarge IRAM/DRAM/flash segments so grouped CI test builds can link."""
content = _patch_segment_size(content, "iram1_0_seg", TESTING_IRAM_SIZE)
content = _patch_segment_size(content, "dram0_0_seg", TESTING_DRAM_SIZE)
return _patch_segment_size(content, "irom0_0_seg", TESTING_FLASH_SIZE)
+3
View File
@@ -15,6 +15,9 @@ CONF_ENABLE_SERIAL1 = "enable_serial1"
KEY_WAVEFORM_REQUIRED = "waveform_required"
KEY_SERIAL_REQUIRED = "serial_required"
KEY_SERIAL1_REQUIRED = "serial1_required"
# Set for the native (non-PlatformIO) toolchain's build generator
KEY_FLASH_MODE = "flash_mode"
KEY_SCANF_FLOAT = "scanf_float"
# esp8266 namespace is already defined by arduino, manually prefix esphome
esp8266_ns = cg.global_ns.namespace("esphome").namespace("esp8266")
+2
View File
@@ -21,6 +21,8 @@ class Toolchain(StrEnum):
PLATFORMIO = "platformio"
ESP_IDF = "esp-idf"
SDK_NRF = "sdk-nrf"
# ESP8266: the Arduino core built directly (no PlatformIO)
ARDUINO = "arduino"
class Platform(StrEnum):
+4
View File
@@ -979,6 +979,10 @@ class EsphomeCore:
def using_toolchain_sdk_nrf(self):
return self.toolchain == Toolchain.SDK_NRF
@property
def using_toolchain_arduino(self):
return self.toolchain == Toolchain.ARDUINO
@property
def using_zephyr(self):
return self.target_framework == "zephyr"
+2 -26
View File
@@ -41,34 +41,10 @@ def _idf_framework() -> str:
def _apply_extra_script(component: IDFComponent) -> None:
"""Run a PIO ``extraScript`` and fold its captured env vars into
``component.data["build"]["flags"]`` so the existing -L/-l/-D
extraction in ``generate_cmakelists_txt`` picks them up."""
extra_script = component.data.get("build", {}).get("extraScript")
if not extra_script:
return
# Resolve and confine to the library's source dir so a malicious
# library.json can't escape (e.g. ``"extraScript": "../../etc/passwd"``).
source_path = component.source_dir
library_root = source_path.resolve()
script_path = (source_path / extra_script).resolve()
if not script_path.is_relative_to(library_root) or not script_path.is_file():
return
from esphome.components.esp32 import get_esp32_variant
from esphome.espidf.extra_script import captured_as_build_flags, run_extra_script
from esphome.espidf.extra_script import apply_extra_script
idf_target = variant_to_idf_target(get_esp32_variant())
result = run_extra_script(
script_path, library_dir=source_path, idf_target=idf_target
)
extra_flags = captured_as_build_flags(result, library_dir=source_path)
if not extra_flags:
return
flags = component.data.setdefault("build", {}).setdefault("flags", [])
if isinstance(flags, str):
flags = [flags]
flags.extend(extra_flags)
component.data["build"]["flags"] = flags
apply_extra_script(component, lambda: variant_to_idf_target(get_esp32_variant()))
def generate_cmakelists_txt(component: IDFComponent) -> str:
+41
View File
@@ -28,13 +28,54 @@ Caveats
from __future__ import annotations
from collections.abc import Callable
from dataclasses import dataclass, field
import logging
import os
from pathlib import Path
from typing import TYPE_CHECKING
if TYPE_CHECKING:
from esphome.platformio.library import ConvertedLibrary
_LOGGER = logging.getLogger(__name__)
def apply_extra_script(
component: ConvertedLibrary, idf_target: str | Callable[[], str]
) -> None:
"""Run a library's PIO ``extraScript`` and fold its captured env vars into
``component.data["build"]["flags"]`` so the backend's -L/-l/-D extraction
picks them up. Shared by the ESP-IDF and ESP8266 Arduino backends.
``idf_target`` may be a callable so a backend whose target lookup needs
build state (the esp32 variant) resolves it only when a script will run.
"""
extra_script = component.data.get("build", {}).get("extraScript")
if not extra_script:
return
# Resolve and confine to the library's source dir so a malicious
# library.json can't escape (e.g. ``"extraScript": "../../etc/passwd"``).
source_path = component.source_dir
library_root = source_path.resolve()
script_path = (source_path / extra_script).resolve()
if not script_path.is_relative_to(library_root) or not script_path.is_file():
return
if callable(idf_target):
idf_target = idf_target()
result = run_extra_script(
script_path, library_dir=source_path, idf_target=idf_target
)
extra_flags = captured_as_build_flags(result, library_dir=source_path)
if not extra_flags:
return
flags = component.data.setdefault("build", {}).setdefault("flags", [])
if isinstance(flags, str):
flags = [flags]
flags.extend(extra_flags)
component.data["build"]["flags"] = flags
# Keys we know how to translate back into ESPHome's build-flag pipeline.
# Other env.Append kwargs are recorded but ignored downstream.
_CAPTURED_KEYS = frozenset({"LIBPATH", "LIBS", "CPPDEFINES", "LINKFLAGS", "CPPFLAGS"})
+35
View File
@@ -136,6 +136,9 @@ def _parse_entry(entry: dict) -> tuple[str, list[str], list[str], list[str]]:
raw = os.path.normpath(directory / raw)
return raw.replace("\\", "/")
# A ccache-wrapped command ("ccache g++ ...") names the compiler second.
if Path(tokens[0]).stem == "ccache":
tokens = tokens[1:]
# token0 is the compiler path; the rest of the command already uses forward
# slashes on Windows, so normalize it too for a consistent idedata file.
cxx_path = tokens[0].replace("\\", "/")
@@ -219,6 +222,38 @@ def _cc_path_from_cxx(cxx_path: str) -> str:
return f"{stem}{suffix}"
def load_or_build_idedata(
compile_commands: Path, elf_path: Path, cache: Path
) -> dict | None:
"""Return idedata for a compile_commands.json build, cached on mtime.
Shared by the native ESP-IDF and ESP8266 Arduino toolchains. Returns None
when the compile DB doesn't exist yet (nothing was built).
"""
if not compile_commands.is_file():
_LOGGER.debug("No %s yet; skipping idedata generation", compile_commands)
return None
if cache.is_file() and cache.stat().st_mtime >= compile_commands.stat().st_mtime:
try:
cached = json.loads(cache.read_text(encoding="utf-8"))
except ValueError:
pass
else:
# Caches written before cc_path was emitted stay newer than
# compile_commands.json forever, so rebuild them on the field rather
# than on the timestamp. Check the type too: a corrupted cache can
# still be valid JSON, and "in" would match a substring of a string.
if isinstance(cached, dict) and "cc_path" in cached:
return cached
data = idedata_from_build(compile_commands)
data["prog_path"] = str(elf_path)
cache.parent.mkdir(parents=True, exist_ok=True)
cache.write_text(json.dumps(data, indent=2) + "\n", encoding="utf-8")
return data
def idedata_from_build(compile_commands: Path) -> dict:
"""Parse compile_commands.json into the idedata fields consumers expect.
+3 -3
View File
@@ -67,7 +67,7 @@ def _find_app_partition_size(partitions_csv: Path) -> int:
raise ValueError(f"No app+factory or app+ota_0 partition in {partitions_csv}")
def _format_bar(used: int, total: int) -> str:
def format_bar(used: int, total: int) -> str:
"""Match PlatformIO's ``_format_availale_bytes`` (pioupload.py) exactly."""
pct_raw = used / total if total else 0
blocks = 10
@@ -99,7 +99,7 @@ def print_summary(size_json: Path, partitions_csv: Path | None) -> None:
ram_used = ram_region.get("used")
ram_total = ram_region.get("size")
if ram_total and ram_used is not None:
print(f"RAM: {_format_bar(ram_used, ram_total)}")
print(f"RAM: {format_bar(ram_used, ram_total)}")
image_size = data.get("image_size")
if image_size is None or partitions_csv is None:
@@ -109,4 +109,4 @@ def print_summary(size_json: Path, partitions_csv: Path | None) -> None:
except ValueError as e:
_LOGGER.debug("Skipping Flash summary: %s", e)
return
print(f"Flash: {_format_bar(image_size, app_size)}")
print(f"Flash: {format_bar(image_size, app_size)}")
+6 -25
View File
@@ -504,32 +504,13 @@ def get_idedata() -> dict | None:
idedata fields IDE integrations and clang-tidy expect, cached alongside the
PlatformIO idedata path. Returns None if the compile DB doesn't exist yet.
"""
from esphome.espidf.idedata import idedata_from_build
from esphome.espidf.idedata import load_or_build_idedata
compile_commands = CORE.relative_build_path("build", "compile_commands.json")
if not compile_commands.is_file():
_LOGGER.debug("No %s yet; skipping idedata generation", compile_commands)
return None
cache = CORE.relative_internal_path("idedata", f"{CORE.name}.json")
if cache.is_file() and cache.stat().st_mtime >= compile_commands.stat().st_mtime:
try:
cached = json.loads(cache.read_text(encoding="utf-8"))
except ValueError:
pass
else:
# Caches written before cc_path was emitted stay newer than
# compile_commands.json forever, so rebuild them on the field rather
# than on the timestamp. Check the type too: a corrupted cache can
# still be valid JSON, and "in" would match a substring of a string.
if isinstance(cached, dict) and "cc_path" in cached:
return cached
data = idedata_from_build(compile_commands)
data["prog_path"] = str(get_elf_path())
cache.parent.mkdir(parents=True, exist_ok=True)
cache.write_text(json.dumps(data, indent=2) + "\n", encoding="utf-8")
return data
return load_or_build_idedata(
CORE.relative_build_path("build", "compile_commands.json"),
get_elf_path(),
CORE.relative_internal_path("idedata", f"{CORE.name}.json"),
)
def create_factory_bin() -> bool:
+4 -4
View File
@@ -469,7 +469,7 @@ def _parse_library_json(library_json_path: PathType):
return json.load(fp)
def _parse_library_properties(library_properties_path: PathType):
def parse_library_properties(library_properties_path: PathType):
"""
Parse a key-value platformio .properties style file into a dictionary.
@@ -553,7 +553,7 @@ def _resolve_registry_version(
return owner, name, best["name"], pkgfile["download_url"]
def _normalize_dependencies(dependencies: Any) -> list[dict]:
def normalize_dependencies(dependencies: Any) -> list[dict]:
"""Normalize a library manifest's ``dependencies`` to a list of dicts.
PIO's library.json accepts both the list-of-dicts form and the shorthand
@@ -840,7 +840,7 @@ def convert_libraries(
if has_json:
component.data = _parse_library_json(library_json_path)
elif has_properties:
component.data = _parse_library_properties(library_properties_path)
component.data = parse_library_properties(library_properties_path)
else:
# For a local library a missing manifest is user input, so raise
# EsphomeError (clean CLI message) like the missing-directory case;
@@ -869,7 +869,7 @@ def convert_libraries(
# Requirements changed (we got past the short-circuit above), so
# (re)walk this component's dependencies.
node.edges = set()
for dependency in _normalize_dependencies(component.data.get("dependencies")):
for dependency in normalize_dependencies(component.data.get("dependencies")):
if "name" not in dependency or "version" not in dependency:
continue
try:
+7 -1
View File
@@ -660,11 +660,17 @@ def clean_all(configuration: list[str]):
# that live outside it.
import platformdirs
from esphome.arduino8266.framework import get_arduino8266_tools_path
from esphome.components.nrf52.framework import get_sdk_nrf_tools_path
from esphome.espidf.framework import get_idf_tools_path
cache_root = Path(platformdirs.user_cache_dir("esphome", appauthor=False)).resolve()
for install_path in (cache_root, get_idf_tools_path(), get_sdk_nrf_tools_path()):
for install_path in (
cache_root,
get_idf_tools_path(),
get_sdk_nrf_tools_path(),
get_arduino8266_tools_path(),
):
if install_path.is_dir():
_LOGGER.info("Deleting %s", install_path)
rmtree(install_path)
+69
View File
@@ -610,6 +610,69 @@ def should_run_esp32_platformio(branch: str | None = None) -> bool:
return bool(esp32_platformio_components_to_test(branch))
# Components tested by the native (PlatformIO-free) ESP8266 Arduino toolchain
# compile-test job. The regular component matrix builds esp8266 with the
# default platformio toolchain; this list is the `--toolchain arduino` smoke
# test, chosen to exercise the core, the bundled libraries (ESP8266WiFi,
# ESP8266mDNS, Wire, SPI, DNSServer, Hash), the converted registry libraries
# (ESPAsyncTCP/WebServer, AsyncMqttClient, NeoPixelBus), and the waveform path.
ESP8266_NATIVE_TEST_COMPONENTS = frozenset(
{
"esp8266",
"api",
"web_server",
"captive_portal",
"mqtt",
"esp8266_pwm",
"neopixelbus",
"bme280_i2c",
"uart",
}
)
# Infrastructure whose changes always trigger the native ESP8266 compile test.
ESP8266_NATIVE_TRIGGER_PATH_PREFIXES = ("esphome/arduino8266/",)
ESP8266_NATIVE_TRIGGER_FILES = frozenset(
{
"esphome/build_gen/arduino8266.py",
"esphome/components/esp8266/build_surgery.py",
"esphome/components/esp8266/boards.py",
"script/test_build_components.py",
".github/workflows/ci.yml",
}
)
def _esp8266_native_path_or_file_trigger(files: list[str]) -> bool:
"""Whether any changed file is native-ESP8266 infrastructure / harness."""
for file in files:
if file in ESP8266_NATIVE_TRIGGER_FILES:
return True
if any(
file.startswith(prefix) for prefix in ESP8266_NATIVE_TRIGGER_PATH_PREFIXES
):
return True
return False
def esp8266_native_components_to_test(branch: str | None = None) -> list[str]:
"""Subset of ``ESP8266_NATIVE_TEST_COMPONENTS`` the job needs to compile.
Same narrowing logic as ``esp32_platformio_components_to_test``: the full
list on core or infrastructure changes, otherwise the intersection with
the changed-component dependency closure (empty list skips the job).
"""
files = changed_files(branch)
if core_changed(files) or _esp8266_native_path_or_file_trigger(files):
return sorted(ESP8266_NATIVE_TEST_COMPONENTS)
component_files = [f for f in files if filter_component_and_test_files(f)]
changed = get_components_with_dependencies(component_files, True)
return sorted(ESP8266_NATIVE_TEST_COMPONENTS & set(changed))
def determine_cpp_unit_tests(
branch: str | None = None,
) -> tuple[bool, list[str]]:
@@ -1205,6 +1268,8 @@ def main() -> None:
run_device_builder = True
esp32_platformio_components = sorted(ESP32_PLATFORMIO_TEST_COMPONENTS)
run_esp32_platformio = True
esp8266_native_components = sorted(ESP8266_NATIVE_TEST_COMPONENTS)
run_esp8266_native = True
else:
integration_run_all, integration_test_files = determine_integration_tests(
args.branch
@@ -1216,6 +1281,8 @@ def main() -> None:
run_device_builder = should_run_device_builder(args.branch)
esp32_platformio_components = esp32_platformio_components_to_test(args.branch)
run_esp32_platformio = bool(esp32_platformio_components)
esp8266_native_components = esp8266_native_components_to_test(args.branch)
run_esp8266_native = bool(esp8266_native_components)
run_integration, integration_test_buckets = _compute_integration_test_buckets(
integration_run_all, integration_test_files
)
@@ -1410,6 +1477,8 @@ def main() -> None:
"device_builder": run_device_builder,
"esp32_platformio": run_esp32_platformio,
"esp32_platformio_components": ",".join(esp32_platformio_components),
"esp8266_native": run_esp8266_native,
"esp8266_native_components": ",".join(esp8266_native_components),
"changed_components": changed_components,
"changed_components_with_tests": changed_components_with_tests,
"directly_changed_components_with_tests": list(directly_changed_with_tests),
@@ -0,0 +1,261 @@
"""Drift tests for the native ESP8266 Arduino build generator.
These pin the build spec transliterated from the PlatformIO builder
(framework-arduinoespressif8266/tools/platformio-build.py and
platform-espressif8266/builder/main.py) so a change on either side of the
toolchain seam is caught: the knob-define precedence, the define/flag sets,
the link line, and the core source exclusions must keep matching what the
PlatformIO toolchain produces for the same configuration.
"""
from __future__ import annotations
from pathlib import Path
from unittest.mock import patch
import pytest
from esphome.components.esp8266.boards import BOARDS, ESP8266_BOARD_BUILD
from esphome.components.esp8266.const import (
KEY_BOARD,
KEY_ESP8266,
KEY_FLASH_MODE,
KEY_SCANF_FLOAT,
)
import esphome.config_validation as cv
from esphome.const import KEY_CORE, KEY_FRAMEWORK_VERSION
from esphome.core import CORE, EsphomeError
@pytest.fixture(autouse=True)
def _setup_core(tmp_path: Path) -> None:
CORE.name = "test8266"
CORE.build_path = tmp_path
CORE.testing_mode = False
CORE.cpp_standard = "gnu++20"
CORE.data[KEY_CORE] = {KEY_FRAMEWORK_VERSION: cv.Version(3, 1, 2)}
CORE.data[KEY_ESP8266] = {
KEY_BOARD: "nodemcuv2",
KEY_FLASH_MODE: "dout",
KEY_SCANF_FLOAT: False,
}
def _set_flags(*flags: str) -> None:
CORE.build_flags = set(flags)
def test_board_build_covers_every_board() -> None:
"""Every supported board must have variant/define metadata."""
assert set(BOARDS) <= set(ESP8266_BOARD_BUILD)
def test_build_config_defaults() -> None:
from esphome.build_gen.arduino8266 import _flag_defines, _resolve_build_config
_set_flags()
config = _resolve_build_config(_flag_defines())
assert config.nonosdk == "NONOSDK22x_190703"
assert config.lwip_lib == "lwip2-536-feat"
assert not config.exceptions
assert config.vtables == "VTABLES_IN_FLASH"
assert config.knob_defines == [
"NONOSDK22x_190703=1",
"TCP_MSS=536",
"LWIP_FEATURES=1",
"LWIP_IPV6=0",
]
assert config.mmu_defines == ["MMU_IRAM_SIZE=0x8000", "MMU_ICACHE_SIZE=0x8000"]
def test_build_config_esphome_lwip_knob() -> None:
"""The lwIP variant ESPHome selects maps to the same defines and library
as the PlatformIO builder."""
from esphome.build_gen.arduino8266 import _flag_defines, _resolve_build_config
_set_flags("-DPIO_FRAMEWORK_ARDUINO_LWIP2_HIGHER_BANDWIDTH_LOW_FLASH")
config = _resolve_build_config(_flag_defines())
assert config.lwip_lib == "lwip2-1460"
assert "TCP_MSS=1460" in config.knob_defines
assert "LWIP_FEATURES=0" in config.knob_defines
assert "LWIP_IPV6=0" in config.knob_defines
def test_build_config_knobs() -> None:
from esphome.build_gen.arduino8266 import _flag_defines, _resolve_build_config
_set_flags(
"-DPIO_FRAMEWORK_ARDUINO_ESPRESSIF_SDK305",
"-DPIO_FRAMEWORK_ARDUINO_ENABLE_EXCEPTIONS",
"-DPIO_FRAMEWORK_ARDUINO_MMU_CACHE16_IRAM48",
"-DVTABLES_IN_DRAM",
)
config = _resolve_build_config(_flag_defines())
assert config.nonosdk == "NONOSDK305"
assert config.exceptions
assert config.vtables == "VTABLES_IN_DRAM"
assert config.mmu_defines == ["MMU_IRAM_SIZE=0xC000", "MMU_ICACHE_SIZE=0x4000"]
def test_build_config_mmu_custom_requires_sizes() -> None:
from esphome.build_gen.arduino8266 import _flag_defines, _resolve_build_config
_set_flags("-DPIO_FRAMEWORK_ARDUINO_MMU_CUSTOM")
with pytest.raises(EsphomeError, match="MMU_IRAM_SIZE"):
_resolve_build_config(_flag_defines())
_set_flags(
"-DPIO_FRAMEWORK_ARDUINO_MMU_CUSTOM",
"-DMMU_IRAM_SIZE=0xC000",
"-DMMU_ICACHE_SIZE=0x4000",
)
config = _resolve_build_config(_flag_defines())
assert sorted(config.mmu_defines) == [
"MMU_ICACHE_SIZE=0x4000",
"MMU_IRAM_SIZE=0xC000",
]
def test_defines_match_platformio_builder() -> None:
"""The exact define set the PlatformIO builder passes for nodemcuv2/dout."""
from esphome.build_gen.arduino8266 import (
_defines_flags,
_flag_defines,
_resolve_build_config,
)
_set_flags("-DPIO_FRAMEWORK_ARDUINO_LWIP2_HIGHER_BANDWIDTH_LOW_FLASH")
assert _defines_flags(
_resolve_build_config(_flag_defines()),
"dout",
"nodemcuv2",
ESP8266_BOARD_BUILD["nodemcuv2"]["defines"],
) == [
"-DF_CPU=80000000L",
"-D__ets__",
"-DICACHE_FLASH",
"-D_GNU_SOURCE",
"-DARDUINO=10805",
'-DARDUINO_BOARD=\\"PLATFORMIO_NODEMCUV2\\"',
'-DARDUINO_BOARD_ID=\\"nodemcuv2\\"',
"-DFLASHMODE_DOUT",
"-DLWIP_OPEN_SRC",
"-DNONOSDK22x_190703=1",
"-DTCP_MSS=1460",
"-DLWIP_FEATURES=0",
"-DLWIP_IPV6=0",
"-DVTABLES_IN_FLASH",
"-DMMU_IRAM_SIZE=0x8000",
"-DMMU_ICACHE_SIZE=0x8000",
"-DESP8266",
"-DARDUINO_ARCH_ESP8266",
"-DARDUINO_ESP8266_NODEMCU_ESP12E",
]
def _make_framework(tmp_path: Path) -> dict[str, Path]:
framework = tmp_path / "framework"
core = framework / "cores" / "esp8266"
core.mkdir(parents=True)
for name in (
"core_esp8266_main.cpp",
"Updater.cpp",
"core_esp8266_waveform_pwm.cpp",
"core_esp8266_waveform_phase.cpp",
"cont.S",
"abi.c",
):
(core / name).write_text("")
(framework / "variants" / "nodemcu").mkdir(parents=True)
for sub in ("include", "ld", "lwip2/include", "lib"):
(framework / "tools" / "sdk" / sub).mkdir(parents=True)
(framework / "libraries").mkdir()
toolchain = tmp_path / "toolchain"
(toolchain / "bin").mkdir(parents=True)
return {
"framework_path": framework,
"toolchain_path": toolchain,
"ninja_path": Path("ninja"),
}
def _write_ninja(paths: dict[str, Path]) -> str:
from esphome.build_gen import arduino8266
src = CORE.relative_src_path()
(src / "esphome" / "components" / "esp8266").mkdir(parents=True, exist_ok=True)
(src / "main.cpp").write_text("")
(src / "esphome" / "vendor.c").write_text("")
with (
patch.object(arduino8266, "generate_ld_scripts"),
patch("esphome.arduino8266.framework.ccache_path", return_value=None),
):
arduino8266.write_project(paths)
return (CORE.relative_pioenvs_path(CORE.name) / "build.ninja").read_text()
def test_write_project_link_line_and_exclusions(tmp_path: Path) -> None:
paths = _make_framework(tmp_path)
_set_flags(
"-DPIO_FRAMEWORK_ARDUINO_LWIP2_HIGHER_BANDWIDTH_LOW_FLASH",
"-DUSE_ESP8266_WAVEFORM_STUBS",
"-Wl,--wrap=millis",
"-Wl,--wrap=printf",
"-Wno-nonnull-compare",
)
content = _write_ninja(paths)
# Base link flags from the PlatformIO builder
for flag in (
"-Wl,--no-check-sections",
"-Wl,-static",
"-Wl,--gc-sections",
"-Wl,-wrap,system_restart_local",
"-Wl,-wrap,spi_flash_read",
"-u app_entry",
"-u _printf_float",
"-u _DebugExceptionVector",
"-u _DoubleExceptionVector",
"-u _KernelExceptionVector",
"-u _NMIExceptionVector",
"-u _UserExceptionVector",
):
assert flag in content
# ESPHome's link flags and the board linker script
assert "-Wl,--wrap=millis" in content
assert "-Wl,--wrap=printf" in content
assert "-T eagle.flash.4m.ld" in content
# scanf float disabled: the forced-link flag must not appear
assert "_scanf_float" not in content
# System libraries with the selected lwIP variant, in the builder's order
assert (
"-lhal -lphy -lpp -lnet80211 -llwip2-1460 -lwpa -lcrypto -lmain -lwps "
"-lbearssl -lespnow -lsmartconfig -lairkiss -lwpa2 -lstdc++ -lm -lc -lgcc"
in content
)
# Core exclusions: native OTA backend and waveform stubs
assert "Updater.cpp" not in content
assert "core_esp8266_waveform_pwm.cpp" not in content
assert "core_esp8266_waveform_phase.cpp" not in content
assert "core_esp8266_main.cpp.o" in content
# Assembly and C sources compile through their own rules
assert "cont.S.o: asm" in content
assert "abi.c.o: cc" in content
# throw_stubs is force-included for ESPHome sources only
src_lines = [line for line in content.splitlines() if "obj/src/" in line]
assert any("main.cpp.o: cxx" in line for line in src_lines)
assert content.count("throw_stubs.h") == len(
[line for line in content.splitlines() if line.startswith(" flags = ")]
)
def test_write_project_scanf_float_and_waveform_kept(tmp_path: Path) -> None:
paths = _make_framework(tmp_path)
CORE.data[KEY_ESP8266][KEY_SCANF_FLOAT] = True
_set_flags("-DPIO_FRAMEWORK_ARDUINO_LWIP2_HIGHER_BANDWIDTH_LOW_FLASH")
content = _write_ninja(paths)
assert "-u _scanf_float" in content
# Waveform not stubbed out: both implementations stay in the archive
assert "core_esp8266_waveform_pwm.cpp.o" in content
assert "core_esp8266_waveform_phase.cpp.o" in content
@@ -0,0 +1,64 @@
"""Tests for the linker-script surgery shared with the native toolchain."""
from __future__ import annotations
import pytest
from esphome.components.esp8266.build_surgery import (
RATETABLE_RULE,
apply_testing_memory_patches,
relocate_ratetable,
)
_COMMON_LD_SNIPPET = """\
.dport0.data : ALIGN(4)
{
_dport0_data_start = ABSOLUTE(.);
} >dport0_0_seg :dport0_0_phdr
.data : ALIGN(4)
{
_data_start = ABSOLUTE(.);
*(.data)
} >dram0_0_seg :dram0_0_phdr
"""
_FLASH_LD_SNIPPET = """\
MEMORY
{
dport0_0_seg : org = 0x3FF00000, len = 0x10
dram0_0_seg : org = 0x3FFE8000, len = 0x14000
iram1_0_seg : org = 0x40100000, len = 0x8000
irom0_0_seg : org = 0x40201010, len = 0xfeff0
}
"""
def test_relocate_ratetable_inserts_after_data_start() -> None:
patched = relocate_ratetable(_COMMON_LD_SNIPPET)
assert RATETABLE_RULE in patched
# Inserted after the .data section's anchor, not the .dport0.data one
assert patched.index("_data_start = ABSOLUTE(.);") < patched.index(RATETABLE_RULE)
assert patched.index(RATETABLE_RULE) < patched.index("*(.data)")
# Idempotent on an already-patched script
assert relocate_ratetable(patched) == patched
def test_relocate_ratetable_requires_anchor() -> None:
with pytest.raises(RuntimeError, match="_data_start"):
relocate_ratetable("SECTIONS { }")
def test_testing_memory_patches_enlarge_segments() -> None:
patched = apply_testing_memory_patches(_FLASH_LD_SNIPPET)
assert (
"iram1_0_seg : org = 0x40100000, len = 0x200000"
in patched
)
assert (
"dram0_0_seg : org = 0x3FFE8000, len = 0x200000"
in patched
)
assert (
"irom0_0_seg : org = 0x40201010, len = 0x2000000"
in patched
)
+12 -12
View File
@@ -26,11 +26,11 @@ from esphome.platformio.library import (
GitSource,
URLSource,
_node_key,
_normalize_dependencies,
_parse_library_json,
_parse_library_properties,
_resolve_registry_version,
collect_filtered_files,
normalize_dependencies,
parse_library_properties,
split_list_by_condition,
)
@@ -499,7 +499,7 @@ def test_parse_library_json(tmp_path):
assert result["name"] == "test"
def test_parse_library_properties(tmp_path):
def testparse_library_properties(tmp_path):
f = tmp_path / "library.properties"
f.write_text(
"""
@@ -510,7 +510,7 @@ empty=
"""
)
result = _parse_library_properties(f)
result = parse_library_properties(f)
assert result["name"] == "Test"
assert result["version"] == "1.0"
@@ -679,23 +679,23 @@ def test_node_key_registry_bare_name():
assert (key, kind, locator) == ("bar", "registry", (None, "bar"))
def test_normalize_dependencies_none():
assert _normalize_dependencies(None) == []
def testnormalize_dependencies_none():
assert normalize_dependencies(None) == []
def test_normalize_dependencies_list_form():
def testnormalize_dependencies_list_form():
deps = [{"name": "foo", "version": "1.0"}]
assert _normalize_dependencies(deps) == [{"name": "foo", "version": "1.0"}]
assert normalize_dependencies(deps) == [{"name": "foo", "version": "1.0"}]
def test_normalize_dependencies_dict_form():
out = _normalize_dependencies({"nanopb/Nanopb": "^0.4.91", "BareName": "1.2.3"})
def testnormalize_dependencies_dict_form():
out = normalize_dependencies({"nanopb/Nanopb": "^0.4.91", "BareName": "1.2.3"})
assert {"name": "Nanopb", "owner": "nanopb", "version": "^0.4.91"} in out
assert {"name": "BareName", "owner": None, "version": "1.2.3"} in out
def test_normalize_dependencies_dict_form_nested_spec():
out = _normalize_dependencies(
def testnormalize_dependencies_dict_form_nested_spec():
out = normalize_dependencies(
{"nanopb/Nanopb": {"version": "^0.4.91", "platforms": "espidf"}}
)
assert out == [