diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 35148de0c0..96018fd037 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -101,6 +101,8 @@ jobs: device-builder: ${{ steps.determine.outputs.device-builder }} esp32-platformio: ${{ steps.determine.outputs.esp32-platformio }} esp32-platformio-components: ${{ steps.determine.outputs.esp32-platformio-components }} + esp8266-native: ${{ steps.determine.outputs.esp8266-native }} + esp8266-native-components: ${{ steps.determine.outputs.esp8266-native-components }} changed-components: ${{ steps.determine.outputs.changed-components }} changed-components-with-tests: ${{ steps.determine.outputs.changed-components-with-tests }} directly-changed-components-with-tests: ${{ steps.determine.outputs.directly-changed-components-with-tests }} @@ -155,6 +157,8 @@ jobs: echo "device-builder=$(echo "$output" | jq -r '.device_builder')" >> $GITHUB_OUTPUT echo "esp32-platformio=$(echo "$output" | jq -r '.esp32_platformio')" >> $GITHUB_OUTPUT echo "esp32-platformio-components=$(echo "$output" | jq -r '.esp32_platformio_components')" >> $GITHUB_OUTPUT + echo "esp8266-native=$(echo "$output" | jq -r '.esp8266_native')" >> $GITHUB_OUTPUT + echo "esp8266-native-components=$(echo "$output" | jq -r '.esp8266_native_components')" >> $GITHUB_OUTPUT echo "changed-components=$(echo "$output" | jq -c '.changed_components')" >> $GITHUB_OUTPUT echo "changed-components-with-tests=$(echo "$output" | jq -c '.changed_components_with_tests')" >> $GITHUB_OUTPUT echo "directly-changed-components-with-tests=$(echo "$output" | jq -c '.directly_changed_components_with_tests')" >> $GITHUB_OUTPUT @@ -1125,6 +1129,40 @@ jobs: # Arduino framework via PlatformIO (only components with an esp32-ard test are built): python3 script/test_build_components.py -e compile -t esp32-ard -c "$TEST_COMPONENTS" -f --toolchain platformio + test-esp8266-native: + name: Test esp8266 components with the native toolchain + runs-on: ubuntu-24.04 + needs: + - common + - determine-jobs + if: github.event_name == 'pull_request' && needs.determine-jobs.outputs.esp8266-native == 'true' + env: + # Comma-joined subset of the native-ESP8266 representative component list, + # computed by script/determine-jobs.py (esp8266_native_components_to_test). + # Single source of truth -- the full list lives in + # script/determine-jobs.py::ESP8266_NATIVE_TEST_COMPONENTS. + TEST_COMPONENTS: ${{ needs.determine-jobs.outputs.esp8266-native-components }} + steps: + - name: Check out code from GitHub + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Restore Python + uses: ./.github/actions/restore-python + with: + python-version: ${{ env.DEFAULT_PYTHON }} + cache-key: ${{ needs.common.outputs.cache-key }} + + - name: Run native toolchain compile test + run: | + . venv/bin/activate + + echo "Testing components: $TEST_COMPONENTS" + echo "" + + # ESP8266 Arduino built directly (no PlatformIO); compile validates + # config first, so a separate config pass is redundant. + python3 script/test_build_components.py -e compile -t esp8266-ard -c "$TEST_COMPONENTS" -f --toolchain arduino + device-builder: name: Test downstream esphome/device-builder runs-on: ubuntu-24.04 @@ -1502,6 +1540,7 @@ jobs: - clang-tidy-esp32-variants - test-build-components-split - test-esp32-platformio + - test-esp8266-native - device-builder - memory-impact-target-branch - memory-impact-pr-branch diff --git a/esphome/__main__.py b/esphome/__main__.py index c1e05d2ea7..fd22925c78 100644 --- a/esphome/__main__.py +++ b/esphome/__main__.py @@ -813,6 +813,10 @@ def write_cpp_file() -> int: from esphome.build_gen import espidf espidf.write_project() + elif CORE.using_toolchain_arduino: + # The ninja project is generated at compile time by + # esphome.arduino8266.toolchain (it needs the downloaded framework). + pass else: from esphome.build_gen import platformio @@ -964,6 +968,10 @@ def upload_using_esptool( flash_images = [ FlashImage(path=toolchain.get_factory_firmware_path(), offset="0x0") ] + elif CORE.using_toolchain_arduino: + # The native backend writes PlatformIO-compatible output paths, so the + # shared property already points at the right file. + flash_images = [FlashImage(path=CORE.firmware_bin, offset="0x0")] else: from esphome.platformio import toolchain diff --git a/esphome/arduino8266/__init__.py b/esphome/arduino8266/__init__.py new file mode 100644 index 0000000000..8f403a8553 --- /dev/null +++ b/esphome/arduino8266/__init__.py @@ -0,0 +1,9 @@ +"""Native (PlatformIO-free) build support for the ESP8266 Arduino core. + +This package downloads the Arduino ESP8266 core and the xtensa-lx106 +toolchain, generates a ninja build for them plus the ESPHome sources, and +drives the build directly — the ESP8266 equivalent of ``esphome.espidf``. + +Deliberately importable without the esp8266 component to avoid circular +imports; the component wires these modules in via lazy imports. +""" diff --git a/esphome/arduino8266/build_tool.py b/esphome/arduino8266/build_tool.py new file mode 100644 index 0000000000..bf08662e3e --- /dev/null +++ b/esphome/arduino8266/build_tool.py @@ -0,0 +1,36 @@ +"""Tiny cross-platform build steps invoked from the generated ninja file. + +Plain script (not ``python -m``): it runs from ninja with whatever Python +started esphome and must not depend on the package being importable. + +Subcommands: + ar remove stale archive, then ``ar rc`` + copy copy a file +""" + +from pathlib import Path +import shutil +import subprocess +import sys + + +def main() -> int: + mode = sys.argv[1] + if mode == "ar": + ar, archive, rspfile = sys.argv[2:5] + # Remove first: ``ar rc`` replaces members but never drops ones whose + # source was removed from the build, which would leak stale objects. + Path(archive).unlink(missing_ok=True) + return subprocess.run( + [ar, "rc", archive, f"@{rspfile}"], check=False + ).returncode + if mode == "copy": + src, dst = sys.argv[2:4] + shutil.copyfile(src, dst) + return 0 + print(f"unknown build_tool mode: {mode}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/esphome/arduino8266/component.py b/esphome/arduino8266/component.py new file mode 100644 index 0000000000..e08eafd96c --- /dev/null +++ b/esphome/arduino8266/component.py @@ -0,0 +1,173 @@ +"""Arduino ESP8266 backend for the shared PlatformIO library converter. + +Turns the libraries registered via ``cg.add_library()`` into build inputs for +the ninja generator. Bare names that exist under the framework's bundled +``libraries/`` directory (ESP8266WiFi, Wire, SPI, ...) are read straight from +the framework tree; everything else goes through the shared +resolution/download pipeline in ``esphome.platformio.library``. + +Mirrors PlatformIO's ``lib_ldf_mode=off`` behavior: each library builds into +its own static archive and every library's include dir joins one global +include path. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +import logging +from pathlib import Path +import shlex + +from esphome.core import CORE, Library +from esphome.espidf.extra_script import apply_extra_script +from esphome.platformio.library import ( + DEFAULT_BUILD_INCLUDE_DIR, + DEFAULT_BUILD_SRC_FILTER, + SRC_FILE_EXTENSIONS, + ConvertedLibrary, + InvalidLibrary, + LibraryBackend, + check_library_data, + collect_filtered_files, + convert_libraries, + ensure_list, + normalize_dependencies, + parse_library_properties, +) + +_LOGGER = logging.getLogger(__name__) + +ESP8266_PLATFORM = "espressif8266" + + +@dataclass +class ArduinoLibrary: + """One resolved library, ready for the ninja generator.""" + + name: str + sources: list[Path] = field(default_factory=list) + include_dirs: list[Path] = field(default_factory=list) + # Extra compile flags private to this library's own sources + flags: list[str] = field(default_factory=list) + # Link inputs the library contributes (-L dirs / -l libs, e.g. from + # precompiled vendor blobs) + link_dirs: list[Path] = field(default_factory=list) + link_libs: list[str] = field(default_factory=list) + + +def _library_info(name: str, read_path: Path, data: dict) -> ArduinoLibrary: + """Resolve one library's sources, include dirs, and flags (PIO semantics).""" + build = data.get("build", {}) + + src_dir = build.get("srcDir") + if not src_dir: + for d in ("src", "Src", "."): + if (read_path / d).is_dir(): + src_dir = d + break + + src_filter = ensure_list(build.get("srcFilter", DEFAULT_BUILD_SRC_FILTER)) + # PlatformIO shell-lexes each build.flags entry + raw_flags = [ + token + for entry in ensure_list(build.get("flags", [])) + for token in shlex.split(entry) + ] + + lib = ArduinoLibrary(name=name) + it = iter(raw_flags) + include_flags: list[str] = [] + for tok in it: + if tok in ("-I", "-L", "-l"): + tok += next(it, "") + if tok.startswith("-I"): + include_flags.append(tok[2:]) + elif tok.startswith("-L"): + lib.link_dirs.append((read_path / tok[2:]).resolve()) + elif tok.startswith("-l"): + lib.link_libs.append(tok[2:]) + else: + lib.flags.append(tok) + + include_dir = build.get("includeDir", DEFAULT_BUILD_INCLUDE_DIR) + for d in [include_dir, src_dir, *include_flags]: + if d and (path := (read_path / d)).is_dir(): + lib.include_dirs.append(path.resolve()) + + if src_dir: + lib.sources = sorted( + Path(f).resolve() + for f in collect_filtered_files(read_path / src_dir, src_filter) + if Path(f).suffix in SRC_FILE_EXTENSIONS + ) + return lib + + +def _bundled_library(framework_path: Path, name: str) -> ArduinoLibrary: + """A library bundled with the Arduino core, read from the framework tree.""" + lib_dir = framework_path / "libraries" / name + manifest = lib_dir / "library.properties" + data = parse_library_properties(manifest) if manifest.is_file() else {} + return _library_info(name, lib_dir, {"name": name, **data}) + + +def resolve_libraries(framework_path: Path) -> list[ArduinoLibrary]: + """Resolve every ``cg.add_library()`` entry into an :class:`ArduinoLibrary`.""" + bundled: list[ArduinoLibrary] = [] + external: list[Library] = [] + for library in CORE.platformio_libraries.values(): + if library.repository or not library.name or "/" in library.name: + external.append(library) + elif (framework_path / "libraries" / library.name).is_dir(): + bundled.append(_bundled_library(framework_path, library.name)) + else: + # A bare name that is not a bundled library ("Updater" from the + # ota component) has nothing to build; PlatformIO's LDF-off mode + # ignores it the same way. + _LOGGER.debug("Skipping unknown bundled library %s", library.name) + + converted: list[ArduinoLibrary] = [] + bundled_names = {lib.name for lib in bundled} + + def _add_bundled_dependencies(component: ConvertedLibrary) -> None: + # A version-less bare-name dependency ("Hash" in ESPAsyncWebServer) + # is a core-bundled library; the shared converter skips it because + # it cannot be resolved from the registry. + for dep in normalize_dependencies(component.data.get("dependencies")): + name = dep.get("name") + if ( + not name + or dep.get("owner") + or "version" in dep + or name in bundled_names + or not (framework_path / "libraries" / name).is_dir() + ): + continue + try: + check_library_data(dep, ESP8266_PLATFORM, "arduino") + except InvalidLibrary: + continue + bundled_names.add(name) + bundled.append(_bundled_library(framework_path, name)) + + def _emit(component: ConvertedLibrary) -> None: + apply_extra_script(component, "esp8266") + converted.append( + _library_info( + component.get_require_name(), component.source_dir, component.data + ) + ) + _add_bundled_dependencies(component) + + if external: + convert_libraries( + external, + LibraryBackend( + platform=ESP8266_PLATFORM, + framework="arduino", + emit=_emit, + cache_key="arduino8266", + ), + ) + + return bundled + converted diff --git a/esphome/arduino8266/framework.py b/esphome/arduino8266/framework.py new file mode 100644 index 0000000000..2ae14d5f89 --- /dev/null +++ b/esphome/arduino8266/framework.py @@ -0,0 +1,270 @@ +"""Download and install the Arduino ESP8266 core, toolchain, and ninja. + +Artifacts land in a machine-global cache (shared across projects, like the +ESP-IDF install in ``esphome.espidf.framework``): + + /arduino8266/frameworks// framework-arduinoespressif8266 + /arduino8266/toolchains// toolchain-xtensa (gcc 10.3) + /arduino8266/tools/ninja/ ninja (only if not on PATH) + +Sources default to the PlatformIO registry (the exact packages the PlatformIO +toolchain has always used, so the bits are identical); the +``ESPHOME_ARDUINO8266_*_MIRRORS`` environment variables override the URLs with +``{VERSION}`` / ``{SYSTEM}`` substitution. +""" + +from __future__ import annotations + +import functools +import logging +import os +from pathlib import Path +import platform +import shutil +import stat +import tempfile + +import platformdirs + +import esphome.config_validation as cv +from esphome.core import CORE, EsphomeError +from esphome.framework_helpers import ( + archive_extract_all, + download_from_mirrors, + rmdir, + str_to_lst_of_str, +) +from esphome.helpers import get_bool_env, get_str_env + +_LOGGER = logging.getLogger(__name__) + +FRAMEWORK_PACKAGE = "framework-arduinoespressif8266" +TOOLCHAIN_PACKAGE = "toolchain-xtensa" +# gcc 10.3, the toolchain Arduino core 3.x builds with. The compile flags in +# the build generator are tuned to it; treat version changes as a full +# reinstall (the install dir is keyed on the version). +TOOLCHAIN_VERSION = "2.100300.220621" + +NINJA_VERSION = "1.12.1" + +_REGISTRY_URL = ( + "https://api.registry.platformio.org/v3/packages/platformio/tool/{package}" +) + +ESPHOME_ARDUINO8266_FRAMEWORK_MIRRORS = str_to_lst_of_str( + os.environ.get("ESPHOME_ARDUINO8266_FRAMEWORK_MIRRORS", "") +) +ESPHOME_ARDUINO8266_TOOLCHAIN_MIRRORS = str_to_lst_of_str( + os.environ.get("ESPHOME_ARDUINO8266_TOOLCHAIN_MIRRORS", "") +) +ESPHOME_ARDUINO8266_NINJA_MIRRORS = str_to_lst_of_str( + os.environ.get( + "ESPHOME_ARDUINO8266_NINJA_MIRRORS", + "https://github.com/ninja-build/ninja/releases/download/v{VERSION}/{ARCHIVE}", + ) +) + + +def get_arduino8266_tools_path() -> Path: + # Treat an empty/whitespace prefix as unset: Path("") resolves to the CWD, + # which clean-all would then delete. + if prefix := get_str_env("ESPHOME_ARDUINO8266_PREFIX", "").strip(): + path = Path(prefix).expanduser() + else: + # Machine-global so all projects share one install; see + # espidf.framework.get_idf_tools_path for the location rationale. + path = ( + Path(platformdirs.user_cache_dir("esphome", appauthor=False)) + / "arduino8266" + ) + return path.resolve() + + +def framework_package_version(ver: cv.Version) -> str: + """Map an Arduino core version (e.g. 3.1.2) to its package version. + + Same encoding as the PlatformIO package registry uses for core 3.x + releases (3.1.2 -> 3.30102.0). The native toolchain only supports core + >= 3.1.0, so the 1.x/2.x encodings never apply here. + """ + return f"3.{ver.major}{ver.minor:02d}{ver.patch:02d}.0" + + +def get_framework_path(package_version: str) -> Path: + return get_arduino8266_tools_path() / "frameworks" / package_version + + +def get_toolchain_path() -> Path: + return get_arduino8266_tools_path() / "toolchains" / TOOLCHAIN_VERSION + + +def _pio_system() -> str: + """The PlatformIO registry system tag for the current host. + + Hand-rolled instead of ``platformio.util.get_systype()`` so this backend + never imports the PlatformIO package. The windows-arm64 and darwin-arm64 + mappings are deliberate: the toolchain packages ship x86_64 binaries for + those hosts (Rosetta / x86 emulation). + """ + sysname = platform.system().lower() + machine = platform.machine().lower() + if sysname == "darwin": + return "darwin_arm64" if machine == "arm64" else "darwin_x86_64" + if sysname == "windows": + return "windows_amd64" if machine in ("amd64", "arm64") else "windows_x86" + if machine in ("arm64", "aarch64"): + return "linux_aarch64" + if machine in ("i686", "i386", "x86"): + return "linux_i686" + if machine.startswith("arm"): + return f"linux_{machine}" + return "linux_x86_64" + + +def _registry_download_url(package: str, version: str) -> str: + """Resolve a package's download URL for this host via the PIO registry.""" + import requests + + url = _REGISTRY_URL.format(package=package) + resp = requests.get(url, timeout=30) + resp.raise_for_status() + data = resp.json() + system = _pio_system() + for ver in data.get("versions", []): + if ver.get("name") != version: + continue + for file in ver.get("files", []): + systems = file.get("system") or "*" + if systems == "*" or system in systems: + return file["download_url"] + raise EsphomeError(f"No {package} {version} build for this platform ({system})") + raise EsphomeError(f"{package} {version} not found in the package registry") + + +def _install_package( + name: str, + version: str, + dest: Path, + mirrors: list[str], +) -> None: + """Download and extract one package if not already installed.""" + marker = dest / ".esphome_extracted" + if marker.is_file(): + return + rmdir(dest, msg=f"Clean up incomplete {name} install") + with tempfile.NamedTemporaryFile() as tmp: + _LOGGER.info("Downloading %s %s ...", name, version) + if mirrors: + download_from_mirrors( + mirrors, {"VERSION": version, "SYSTEM": _pio_system()}, tmp.file + ) + else: + download_from_mirrors([_registry_download_url(name, version)], {}, tmp.file) + _LOGGER.info("Extracting %s ...", name) + archive_extract_all(tmp.file, dest, progress_header="Extracting") + marker.touch() + + +def _ninja_archive_name() -> str: + sysname = platform.system().lower() + machine = platform.machine().lower() + if sysname == "darwin": + return "ninja-mac.zip" + if sysname == "windows": + return "ninja-winarm64.zip" if machine == "arm64" else "ninja-win.zip" + if machine in ("arm64", "aarch64"): + return "ninja-linux-aarch64.zip" + return "ninja-linux.zip" + + +def _check_ninja_install() -> Path: + """Return a usable ninja binary, downloading one if none is on PATH.""" + if ninja := shutil.which("ninja"): + return Path(ninja) + ninja_dir = get_arduino8266_tools_path() / "tools" / "ninja" / NINJA_VERSION + binary = ninja_dir / ("ninja.exe" if os.name == "nt" else "ninja") + if binary.is_file(): + return binary + rmdir(ninja_dir, msg="Clean up incomplete ninja install") + with tempfile.NamedTemporaryFile() as tmp: + _LOGGER.info("Downloading ninja %s ...", NINJA_VERSION) + download_from_mirrors( + ESPHOME_ARDUINO8266_NINJA_MIRRORS, + {"VERSION": NINJA_VERSION, "ARCHIVE": _ninja_archive_name()}, + tmp.file, + ) + archive_extract_all(tmp.file, ninja_dir) + if not binary.is_file(): + raise EsphomeError(f"ninja binary missing after extraction in {ninja_dir}") + binary.chmod(binary.stat().st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH) + return binary + + +def check_and_install(framework_version: cv.Version) -> dict[str, Path]: + """Ensure framework, toolchain, and ninja are installed; return their paths.""" + package_version = framework_package_version(framework_version) + framework_path = get_framework_path(package_version) + _install_package( + FRAMEWORK_PACKAGE, + package_version, + framework_path, + ESPHOME_ARDUINO8266_FRAMEWORK_MIRRORS, + ) + toolchain_path = get_toolchain_path() + _install_package( + TOOLCHAIN_PACKAGE, + TOOLCHAIN_VERSION, + toolchain_path, + ESPHOME_ARDUINO8266_TOOLCHAIN_MIRRORS, + ) + return { + "framework_path": framework_path, + "toolchain_path": toolchain_path, + "ninja_path": _check_ninja_install(), + } + + +def get_build_env(toolchain_path: Path) -> dict[str, str]: + env = os.environ.copy() + env["PATH"] = str(toolchain_path / "bin") + os.pathsep + env.get("PATH", "") + env.update(ccache_env()) + return env + + +@functools.cache +def ccache_path() -> str | None: + """The ccache binary to prefix compiles with, or None when disabled. + + Same opt-out convention as the PlatformIO path: on by default when the + binary is on PATH, ``ESPHOME_CCACHE_ENABLE=0`` disables it. + """ + from esphome.platformio.toolchain import _ccache_runs, _strip_win_long_path_prefix + + if "ESPHOME_CCACHE_ENABLE" in os.environ and not get_bool_env( + "ESPHOME_CCACHE_ENABLE" + ): + return None + ccache = shutil.which("ccache") + if ccache is None: + return None + ccache = _strip_win_long_path_prefix(ccache) + return ccache if _ccache_runs(ccache) else None + + +def ccache_env() -> dict[str, str]: + """Return ccache settings for the build subprocess (not os.environ). + + Mirrors ``espidf.framework._ccache_env``: cache under the machine-global + tools dir, depend mode (gcc emits depfiles via -MMD), and CCACHE_BASEDIR + scoped to the build dir so devices share framework cache entries. Values + the user already set in the environment are respected. + """ + if ccache_path() is None: + return {} + defaults = { + "CCACHE_DIR": str(get_arduino8266_tools_path() / "ccache"), + "CCACHE_NOHASHDIR": "true", + "CCACHE_DEPEND": "1", + "CCACHE_BASEDIR": str(Path(CORE.build_path).resolve()), + } + return {k: v for k, v in defaults.items() if k not in os.environ} diff --git a/esphome/arduino8266/toolchain.py b/esphome/arduino8266/toolchain.py new file mode 100644 index 0000000000..a434a53501 --- /dev/null +++ b/esphome/arduino8266/toolchain.py @@ -0,0 +1,146 @@ +"""Native Arduino ESP8266 build driver (the PlatformIO ``run`` equivalent).""" + +from __future__ import annotations + +import logging +from pathlib import Path +import re +import subprocess + +from esphome.arduino8266 import framework +from esphome.const import ( + CONF_COMPILE_PROCESS_LIMIT, + CONF_ESPHOME, + KEY_CORE, + KEY_FRAMEWORK_VERSION, +) +from esphome.core import CORE +from esphome.helpers import write_file_if_changed +from esphome.types import ConfigType + +_LOGGER = logging.getLogger(__name__) + +# ESP8266 user RAM (matches upload.maximum_ram_size in every board manifest) +_MAX_RAM_SIZE = 81920 + +_RAM_SECTIONS = (".data", ".rodata", ".bss") +_FLASH_SECTIONS = (".irom0.text", ".text", ".text1", ".data", ".rodata") + + +def get_build_dir() -> Path: + return CORE.relative_pioenvs_path(CORE.name) + + +def get_elf_path() -> Path: + return get_build_dir() / "firmware.elf" + + +def get_addr2line_path() -> Path: + return framework.get_toolchain_path() / "bin" / "xtensa-lx106-elf-addr2line" + + +def run_compile(config: ConfigType, verbose: bool) -> int: + from esphome.build_gen import arduino8266 as build_gen + + paths = framework.check_and_install(CORE.data[KEY_CORE][KEY_FRAMEWORK_VERSION]) + build_gen.write_project(paths) + + build_dir = get_build_dir() + env = framework.get_build_env(paths["toolchain_path"]) + cmd = [str(paths["ninja_path"]), "-C", str(build_dir)] + if verbose: + cmd.append("-v") + if jobs := config[CONF_ESPHOME].get(CONF_COMPILE_PROCESS_LIMIT): + cmd += ["-j", str(jobs)] + + _LOGGER.debug("Running: %s", " ".join(cmd)) + rc = subprocess.run(cmd, env=env, check=False, close_fds=False).returncode + if rc != 0: + return rc + + _write_compile_commands(paths["ninja_path"], build_dir, env) + _print_size_summary(build_dir, paths["toolchain_path"]) + get_idedata() + return 0 + + +def _write_compile_commands( + ninja_path: Path, build_dir: Path, env: dict[str, str] +) -> None: + result = subprocess.run( + [str(ninja_path), "-C", str(build_dir), "-t", "compdb", "cc", "cxx", "asm"], + env=env, + capture_output=True, + text=True, + check=False, + close_fds=False, + ) + if result.returncode == 0: + # write_file_if_changed keeps the mtime stable on no-op builds so the + # idedata cache in get_idedata() stays valid. + write_file_if_changed(build_dir / "compile_commands.json", result.stdout) + else: + _LOGGER.warning("Could not generate compile_commands.json: %s", result.stderr) + + +def _parse_app_size(build_dir: Path) -> int | None: + """Read the app flash budget (irom0_0_seg length) from the linker script.""" + from esphome.build_gen.arduino8266 import get_flash_ld_path + + appsize_re = re.compile(r"irom0_0_seg\s*:.+len\s*=\s*(0x[\da-f]+)", re.IGNORECASE) + try: + ld_text = get_flash_ld_path(build_dir).read_text(encoding="utf-8") + except OSError: + return None + for line in ld_text.splitlines(): + if match := appsize_re.search(line): + return int(match.group(1), 16) + return None + + +def _print_size_summary(build_dir: Path, toolchain_path: Path) -> None: + """Print the PlatformIO-shaped RAM/Flash lines. + + The exact shape (including the bar) is parsed by + ``script/ci_memory_impact_extract.py``; ``format_bar`` matches it. + """ + from esphome.espidf.size_summary import format_bar + + size_tool = toolchain_path / "bin" / "xtensa-lx106-elf-size" + result = subprocess.run( + [str(size_tool), "-A", "-d", str(get_elf_path())], + capture_output=True, + text=True, + check=False, + close_fds=False, + ) + if result.returncode != 0: + return + sections: dict[str, int] = {} + for line in result.stdout.splitlines(): + parts = line.split() + if len(parts) >= 2 and parts[0].startswith("."): + try: + sections[parts[0]] = int(parts[1]) + except ValueError: + continue + ram = sum(sections.get(s, 0) for s in _RAM_SECTIONS) + flash = sum(sections.get(s, 0) for s in _FLASH_SECTIONS) + print(f"RAM: {format_bar(ram, _MAX_RAM_SIZE)}") + if app_size := _parse_app_size(build_dir): + print(f"Flash: {format_bar(flash, app_size)}") + + +def get_idedata() -> dict | None: + """Derive idedata from the build's compile_commands.json. + + Same contract as ``espidf.toolchain.get_idedata``: the fields IDE + integrations, clang-tidy, and the memory analyzer expect. + """ + from esphome.espidf.idedata import load_or_build_idedata + + return load_or_build_idedata( + get_build_dir() / "compile_commands.json", + get_elf_path(), + CORE.relative_internal_path("idedata", f"{CORE.name}.json"), + ) diff --git a/esphome/build_gen/arduino8266.py b/esphome/build_gen/arduino8266.py new file mode 100644 index 0000000000..1429e51c40 --- /dev/null +++ b/esphome/build_gen/arduino8266.py @@ -0,0 +1,576 @@ +"""Native ninja build generator for the ESP8266 Arduino core. + +Transliterates the PlatformIO build spec for the Arduino ESP8266 framework +(``framework-arduinoespressif8266/tools/platformio-build.py`` plus +``platform-espressif8266/builder/main.py``) into a ``build.ninja`` under +``.pioenvs//``. The flag sets, defines, link line, linker-script +generation, and ``elf2bin`` invocation deliberately match what PlatformIO +produces so the binaries stay near-identical between the two toolchains. + +The ``PIO_FRAMEWORK_ARDUINO_*`` knob defines (lwIP variant, NONOS SDK +version, MMU layout, exceptions, waveform phase) keep working: they are read +from the build flags with the same precedence as the PlatformIO builder. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +import os +from pathlib import Path +import re +import subprocess +import sys + +from esphome.components.esp8266.boards import ( + BOARDS, + ESP8266_BOARD_BUILD, + ESP8266_LD_SCRIPTS, +) +from esphome.components.esp8266.build_surgery import ( + apply_testing_memory_patches, + relocate_ratetable, +) +from esphome.components.esp8266.const import ( + KEY_BOARD, + KEY_ESP8266, + KEY_FLASH_MODE, + KEY_FLASH_SIZE, + KEY_SCANF_FLOAT, +) +from esphome.const import KEY_CORE, KEY_FRAMEWORK_VERSION +from esphome.core import CORE, EsphomeError +from esphome.framework_helpers import get_project_cxx_compile_flags +from esphome.helpers import mkdir_p, write_file_if_changed + +_RULE_FOR_SUFFIX = {".c": "cc", ".cpp": "cxx", ".S": "asm"} + +# Always excluded from the core build: ESPHome uses its own native OTA +# backend, so the Arduino Updater (and its 228-byte global) never links. +_CORE_EXCLUDE_ALWAYS = {"Updater.cpp"} +# Excluded when no component called require_waveform(); waveform_stubs.cpp +# supplies the stopWaveform()/_stopPWM() stubs digitalWrite needs. +_CORE_EXCLUDE_WAVEFORM = { + "core_esp8266_waveform_pwm.cpp", + "core_esp8266_waveform_phase.cpp", +} + +# From platformio-build.py, in its order of precedence (first is the default). +_NONOSDK_VERSIONS = ( + ("SDK22x_190703", "NONOSDK22x_190703"), + ("SDK221", "NONOSDK221"), + ("SDK22x_190313", "NONOSDK22x_190313"), + ("SDK22x_191024", "NONOSDK22x_191024"), + ("SDK22x_191105", "NONOSDK22x_191105"), + ("SDK22x_191122", "NONOSDK22x_191122"), + ("SDK305", "NONOSDK305"), +) + +# knob define -> (TCP_MSS, LWIP_FEATURES, LWIP_IPV6, library name) +_LWIP_VARIANTS = ( + ("PIO_FRAMEWORK_ARDUINO_LWIP2_IPV6_LOW_MEMORY", (536, 1, 1, "lwip6-536-feat")), + ( + "PIO_FRAMEWORK_ARDUINO_LWIP2_IPV6_HIGHER_BANDWIDTH", + (1460, 1, 1, "lwip6-1460-feat"), + ), + ("PIO_FRAMEWORK_ARDUINO_LWIP2_HIGHER_BANDWIDTH", (1460, 1, 0, "lwip2-1460-feat")), + ("PIO_FRAMEWORK_ARDUINO_LWIP2_LOW_MEMORY_LOW_FLASH", (536, 0, 0, "lwip2-536")), + ( + "PIO_FRAMEWORK_ARDUINO_LWIP2_HIGHER_BANDWIDTH_LOW_FLASH", + (1460, 0, 0, "lwip2-1460"), + ), +) +_LWIP_DEFAULT = (536, 1, 0, "lwip2-536-feat") + +_ASFLAGS = ["-mlongcalls", "-mtext-section-literals"] +_CFLAGS = [ + "-std=gnu17", + "-Wpointer-arith", + "-Wno-implicit-function-declaration", + "-Wl,-EL", + "-fno-inline-functions", + "-nostdlib", +] +_CCFLAGS = [ + "-Os", + "-mlongcalls", + "-mtext-section-literals", + "-falign-functions=4", + "-U__STRICT_ANSI__", + "-ffunction-sections", + "-fdata-sections", + "-Wall", + "-Werror=return-type", + "-free", + "-fipa-pta", +] +_LINKFLAGS = [ + "-Os", + "-nostdlib", + "-Wl,--no-check-sections", + "-Wl,-static", + "-Wl,--gc-sections", + "-Wl,-wrap,system_restart_local", + "-Wl,-wrap,spi_flash_read", + "-u", + "app_entry", + "-u", + "_printf_float", + "-u", + "_DebugExceptionVector", + "-u", + "_DoubleExceptionVector", + "-u", + "_KernelExceptionVector", + "-u", + "_NMIExceptionVector", + "-u", + "_UserExceptionVector", +] +_SYSTEM_LIBS_PRE_LWIP = ["hal", "phy", "pp", "net80211"] +_SYSTEM_LIBS_POST_LWIP = [ + "wpa", + "crypto", + "main", + "wps", + "bearssl", + "espnow", + "smartconfig", + "airkiss", + "wpa2", +] + + +@dataclass +class _BuildConfig: + """Knob-derived build configuration (PIO_FRAMEWORK_ARDUINO_* defines).""" + + nonosdk: str + lwip_lib: str + exceptions: bool + vtables: str + fp_in_irom: bool + knob_defines: list[str] = field(default_factory=list) + mmu_defines: list[str] = field(default_factory=list) + + +def _flag_defines() -> dict[str, str]: + """Map define name -> full ``NAME[=VALUE]`` for every -D build flag.""" + defines: dict[str, str] = {} + for flag in CORE.build_flags: + if flag.startswith("-D"): + body = flag[2:] + defines[body.split("=", 1)[0]] = body + return defines + + +def _resolve_build_config(defines: dict[str, str]) -> _BuildConfig: + nonosdk = _NONOSDK_VERSIONS[0][1] + for name, define in _NONOSDK_VERSIONS: + if f"PIO_FRAMEWORK_ARDUINO_ESPRESSIF_{name}" in defines: + nonosdk = define + + tcp_mss, features, ipv6, lwip_lib = _LWIP_DEFAULT + for knob, variant in _LWIP_VARIANTS: + if knob in defines: + tcp_mss, features, ipv6, lwip_lib = variant + break + + knob_defines = [ + f"{nonosdk}=1", + f"TCP_MSS={tcp_mss}", + f"LWIP_FEATURES={features}", + f"LWIP_IPV6={ipv6}", + ] + if "PIO_FRAMEWORK_ARDUINO_WAVEFORM_LOCKED_PHASE" in defines: + knob_defines.append("WAVEFORM_LOCKED_PHASE=1") + + vtables = next( + (name for name in defines if name.startswith("VTABLES_IN_")), + "VTABLES_IN_FLASH", + ) + + if "PIO_FRAMEWORK_ARDUINO_MMU_CACHE16_IRAM48" in defines: + mmu = ["MMU_IRAM_SIZE=0xC000", "MMU_ICACHE_SIZE=0x4000"] + elif "PIO_FRAMEWORK_ARDUINO_MMU_CACHE16_IRAM48_SECHEAP_SHARED" in defines: + mmu = ["MMU_IRAM_SIZE=0xC000", "MMU_ICACHE_SIZE=0x4000", "MMU_IRAM_HEAP"] + elif "PIO_FRAMEWORK_ARDUINO_MMU_CACHE16_IRAM32_SECHEAP_NOTSHARED" in defines: + mmu = [ + "MMU_IRAM_SIZE=0x8000", + "MMU_ICACHE_SIZE=0x4000", + "MMU_SEC_HEAP_SIZE=0x4000", + "MMU_SEC_HEAP=0x40108000", + ] + elif "PIO_FRAMEWORK_ARDUINO_MMU_EXTERNAL_128K" in defines: + mmu = [ + "MMU_IRAM_SIZE=0x8000", + "MMU_ICACHE_SIZE=0x8000", + "MMU_EXTERNAL_HEAP=128", + ] + elif "PIO_FRAMEWORK_ARDUINO_MMU_EXTERNAL_1024K" in defines: + mmu = [ + "MMU_IRAM_SIZE=0x8000", + "MMU_ICACHE_SIZE=0x8000", + "MMU_EXTERNAL_HEAP=256", + ] + elif "PIO_FRAMEWORK_ARDUINO_MMU_CUSTOM" in defines: + if "MMU_IRAM_SIZE" not in defines or "MMU_ICACHE_SIZE" not in defines: + raise EsphomeError( + "PIO_FRAMEWORK_ARDUINO_MMU_CUSTOM requires MMU_IRAM_SIZE and " + "MMU_ICACHE_SIZE build flags" + ) + mmu = [body for name, body in defines.items() if name.startswith("MMU_")] + else: + mmu = ["MMU_IRAM_SIZE=0x8000", "MMU_ICACHE_SIZE=0x8000"] + + return _BuildConfig( + nonosdk=nonosdk, + lwip_lib=lwip_lib, + exceptions="PIO_FRAMEWORK_ARDUINO_ENABLE_EXCEPTIONS" in defines, + vtables=vtables, + fp_in_irom="FP_IN_IROM" in defines, + knob_defines=knob_defines, + mmu_defines=mmu, + ) + + +def _flash_ld_name(board: str) -> str: + return ESP8266_LD_SCRIPTS[BOARDS[board][KEY_FLASH_SIZE]][1] + + +def _e(value) -> str: + """Escape a path or token for a ninja file.""" + return str(value).replace("$", "$$").replace(":", "$:").replace(" ", "$ ") + + +def _q(value) -> str: + """Quote a path for use inside a ninja command line (shell/CreateProcess).""" + return f'"{value}"' + + +def _defines_flags( + config: _BuildConfig, flash_mode: str, board: str, board_defines: tuple[str, ...] +) -> list[str]: + return [ + f"-D{d}" + for d in ( + "F_CPU=80000000L", + "__ets__", + "ICACHE_FLASH", + "_GNU_SOURCE", + "ARDUINO=10805", + f'ARDUINO_BOARD=\\"PLATFORMIO_{board.upper()}\\"', + f'ARDUINO_BOARD_ID=\\"{board}\\"', + f"FLASHMODE_{flash_mode.upper()}", + "LWIP_OPEN_SRC", + *config.knob_defines, + config.vtables, + *config.mmu_defines, + "ESP8266", + "ARDUINO_ARCH_ESP8266", + *board_defines, + ) + ] + + +def _project_flags() -> tuple[list[str], list[str]]: + """Split the ESPHome build flags into (compile, linker) lists. + + Unlike ``framework_helpers.get_project_compile_flags`` this keeps every + non-linker flag, matching how PlatformIO passes ``build_flags`` to the + compiler verbatim, and applies ``build_unflags``. + """ + unflags = set(CORE.build_unflags) + flags = [f for f in sorted(CORE.build_flags) if f not in unflags] + compile_flags = [f for f in flags if not f.startswith("-Wl,")] + link_flags = [f for f in flags if f.startswith("-Wl,")] + return compile_flags, link_flags + + +def _collect_sources(root: Path, exclude: set[str] = frozenset()) -> list[Path]: + return sorted( + p + for p in root.rglob("*") + if p.suffix in _RULE_FOR_SUFFIX and p.name not in exclude + ) + + +def generate_ld_scripts( + paths: dict[str, Path], config: _BuildConfig, flash_ld_name: str +) -> None: + """Generate the common linker script (and testing-mode flash ld copy). + + Runs the same preprocessor invocation as the PlatformIO builder over + ``eagle.app.v6.common.ld.h``, then applies ESPHome's surgeries: the wifi + rate-table DRAM relocation, and enlarged memory segments in testing mode. + """ + framework = paths["framework_path"] + gcc = paths["toolchain_path"] / "bin" / "xtensa-lx106-elf-gcc" + ld_dir = CORE.relative_pioenvs_path(CORE.name, "ld") + mkdir_p(ld_dir) + + cmd = [str(gcc), "-CC", "-E", "-P", f"-D{config.vtables}"] + cmd += [f"-D{d}" for d in config.mmu_defines] + if config.fp_in_irom: + cmd.append("-DFP_IN_IROM") + cmd += [ + str(framework / "tools" / "sdk" / "ld" / "eagle.app.v6.common.ld.h"), + "-o", + "-", + ] + + # The inputs are the command line (defines + framework version, which is + # baked into the paths) plus testing mode; skip the preprocessor spawn on + # incremental builds when nothing changed. + output = ld_dir / "local.eagle.app.v6.common.ld" + stamp = ld_dir / ".local.eagle.app.v6.common.ld.stamp" + stamp_content = " ".join(cmd) + f" testing={CORE.testing_mode}" + if not ( + output.is_file() + and stamp.is_file() + and stamp.read_text(encoding="utf-8") == stamp_content + ): + result = subprocess.run( + cmd, capture_output=True, text=True, check=False, close_fds=False + ) + if result.returncode != 0: + raise EsphomeError(f"Generating the linker script failed:\n{result.stderr}") + content = relocate_ratetable(result.stdout) + if CORE.testing_mode: + content = apply_testing_memory_patches(content) + write_file_if_changed(output, content) + stamp.write_text(stamp_content, encoding="utf-8") + + if CORE.testing_mode: + # A patched copy of the flash ld in the build dir; resolved through + # the same -L path as the SDK original it shadows. + flash_ld = framework / "tools" / "sdk" / "ld" / flash_ld_name + write_file_if_changed( + ld_dir / f"testing_{flash_ld_name}", + apply_testing_memory_patches(flash_ld.read_text(encoding="utf-8")), + ) + + +def _ninja_compile_edges( + lines: list[str], + sources: list[Path], + root: Path, + group: str, + flags: str = "", +) -> list[str]: + """Emit compile edges for ``sources``; return the object paths.""" + objects = [] + for src in sources: + rel = src.relative_to(root).as_posix() + obj = f"obj/{group}/{rel}.o" + lines.append(f"build {_e(obj)}: {_RULE_FOR_SUFFIX[src.suffix]} {_e(src)}") + if flags: + lines.append(f" flags = {flags}") + objects.append(obj) + return objects + + +def _common_parent(paths: list[Path]) -> Path: + return Path(os.path.commonpath([str(p.parent) for p in paths])) + + +def write_project(paths: dict[str, Path]) -> None: + """Write the ninja build for the current configuration.""" + from esphome.arduino8266.component import resolve_libraries + from esphome.arduino8266.framework import ccache_path + + framework = paths["framework_path"] + toolchain_bin = paths["toolchain_path"] / "bin" + build_dir = CORE.relative_pioenvs_path(CORE.name) + mkdir_p(build_dir) + + flag_defines = _flag_defines() + config = _resolve_build_config(flag_defines) + esp8266_data = CORE.data[KEY_ESP8266] + # Board support was validated at config time (_validate_native_toolchain). + board = esp8266_data[KEY_BOARD] + board_build = ESP8266_BOARD_BUILD[board] + flash_ld_name = _flash_ld_name(board) + + generate_ld_scripts(paths, config, flash_ld_name) + + sdk = framework / "tools" / "sdk" + core_dir = framework / "cores" / "esp8266" + variant_dir = framework / "variants" / board_build["variant"] + src_dir = CORE.relative_src_path() + + libraries = resolve_libraries(framework) + + include_dirs = [ + src_dir, + sdk / "include", + core_dir, + paths["toolchain_path"] / "include", + sdk / "lwip2" / "include", + variant_dir, + ] + include_dirs = [d for d in include_dirs if d.is_dir()] + for lib in libraries: + include_dirs += lib.include_dirs + + project_compile_flags, project_link_flags = _project_flags() + defines = _defines_flags( + config, esp8266_data[KEY_FLASH_MODE], board, board_build["defines"] + ) + includes = [f"-I{_q(d)}" for d in include_dirs] + + common = _CCFLAGS + defines + includes + project_compile_flags + cflags = _CFLAGS + common + cpp_standard = CORE.cpp_standard or "gnu++17" + cxxflags = ( + ["-fno-rtti", f"-std={cpp_standard}"] + + ["-fexceptions" if config.exceptions else "-fno-exceptions"] + + common + + get_project_cxx_compile_flags() + ) + asflags = _ASFLAGS + defines + includes + project_compile_flags + + link_flags = list(_LINKFLAGS) + if esp8266_data.get(KEY_SCANF_FLOAT): + link_flags += ["-u", "_scanf_float"] + link_flags += project_link_flags + flash_ld = f"testing_{flash_ld_name}" if CORE.testing_mode else flash_ld_name + link_flags += ["-T", flash_ld] + + lib_dirs = [Path("ld"), sdk / "lib", sdk / "ld", sdk / "lib" / config.nonosdk] + for lib in libraries: + lib_dirs += lib.link_dirs + system_libs = ( + _SYSTEM_LIBS_PRE_LWIP + + [config.lwip_lib] + + _SYSTEM_LIBS_POST_LWIP + + [lib_name for lib in libraries for lib_name in lib.link_libs] + + ["stdc++-exc" if config.exceptions else "stdc++", "m", "c", "gcc"] + ) + + build_tool = Path(__file__).parent.parent / "arduino8266" / "build_tool.py" + ccache = ccache_path() + + lines = [ + "# Auto-generated by ESPHome", + "ninja_required_version = 1.5", + f"cc = {_q(toolchain_bin / 'xtensa-lx106-elf-gcc')}", + f"cxx = {_q(toolchain_bin / 'xtensa-lx106-elf-g++')}", + f"python = {_q(sys.executable)}", + f"buildtool = {_q(build_tool)}", + f"ccache = {_q(ccache) if ccache else ''}", + "", + "rule cc", + " command = $ccache $cc -MMD -MF $out.d $cflags $flags -c $in -o $out", + " depfile = $out.d", + " deps = gcc", + " description = CC $out", + "rule cxx", + " command = $ccache $cxx -MMD -MF $out.d $cxxflags $flags -c $in -o $out", + " depfile = $out.d", + " deps = gcc", + " description = CXX $out", + "rule asm", + " command = $ccache $cc -MMD -MF $out.d -x assembler-with-cpp $asflags $flags -c $in -o $out", + " depfile = $out.d", + " deps = gcc", + " description = AS $out", + "rule ar", + f" command = $python $buildtool ar {_q(toolchain_bin / 'xtensa-lx106-elf-ar')} $out $out.rsp", + " rspfile = $out.rsp", + " rspfile_content = $in_newline", + " description = AR $out", + "rule link", + " command = $cxx -o $out $linkflags @$out.rsp $libdirflags -Wl,--start-group $archives $libflags -Wl,--end-group", + " rspfile = $out.rsp", + " rspfile_content = $in_newline", + " description = LINK $out", + "rule elf2bin", + f" command = $python {_q(framework / 'tools' / 'elf2bin.py')} --eboot {_q(framework / 'bootloaders' / 'eboot' / 'eboot.elf')} --app $in --flash_mode {esp8266_data[KEY_FLASH_MODE]} --flash_freq 40 --flash_size {_flash_size_str(flash_ld_name)} --path {_q(toolchain_bin)} --out $out", + " description = BIN $out", + "rule copy", + " command = $python $buildtool copy $in $out", + " description = COPY $out", + "", + f"cflags = {' '.join(cflags)}", + f"cxxflags = {' '.join(cxxflags)}", + f"asflags = {' '.join(asflags)}", + f"linkflags = {' '.join(link_flags)}", + f"libdirflags = {' '.join(f'-L{_q(d)}' for d in lib_dirs)}", + f"libflags = {' '.join(f'-l{lib}' for lib in system_libs)}", + "", + ] + + core_exclude = set(_CORE_EXCLUDE_ALWAYS) + if "USE_ESP8266_WAVEFORM_STUBS" in flag_defines: + core_exclude |= _CORE_EXCLUDE_WAVEFORM + + archives = [] + variant_sources = _collect_sources(variant_dir) if variant_dir.is_dir() else [] + if variant_sources: + objs = _ninja_compile_edges(lines, variant_sources, variant_dir, "variant") + lines.append(f"build libFrameworkArduinoVariant.a: ar {' '.join(objs)}") + archives.append("libFrameworkArduinoVariant.a") + + core_objs = _ninja_compile_edges( + lines, _collect_sources(core_dir, core_exclude), core_dir, "core" + ) + lines.append(f"build libFrameworkArduino.a: ar {' '.join(core_objs)}") + archives.append("libFrameworkArduino.a") + + for lib in libraries: + if not lib.sources: + continue + lib_root = _common_parent(lib.sources) + objs = _ninja_compile_edges( + lines, + lib.sources, + lib_root, + f"lib/{lib.name}", + flags=" ".join(lib.flags), + ) + archive = f"lib{lib.name}.a" + lines.append(f"build {_e(archive)}: ar {' '.join(objs)}") + archives.append(archive) + + src_extra = f"-include {_q(src_dir / 'esphome' / 'components' / 'esp8266' / 'throw_stubs.h')}" + src_objs = _ninja_compile_edges( + lines, _collect_sources(src_dir), src_dir, "src", flags=src_extra + ) + + ld_deps = ["ld/local.eagle.app.v6.common.ld"] + if CORE.testing_mode: + ld_deps.append(f"ld/{flash_ld}") + lines.append( + f"build firmware.elf: link {' '.join(src_objs)} | " + f"{' '.join(_e(a) for a in archives)} {' '.join(_e(d) for d in ld_deps)}" + ) + lines.append(f" archives = {' '.join(archives)}") + lines.append("build firmware.bin: elf2bin firmware.elf") + lines.append("build firmware.factory.bin: copy firmware.bin") + lines.append("build firmware.ota.bin: copy firmware.bin") + lines.append("default firmware.factory.bin firmware.ota.bin") + lines.append("") + + write_file_if_changed(build_dir / "build.ninja", "\n".join(lines)) + + +def get_flash_ld_path(build_dir: Path) -> Path: + """The flash linker script the link actually uses (for size reporting).""" + from esphome.arduino8266.framework import ( + framework_package_version, + get_framework_path, + ) + + name = _flash_ld_name(CORE.data[KEY_ESP8266][KEY_BOARD]) + if CORE.testing_mode: + return build_dir / "ld" / f"testing_{name}" + version = framework_package_version(CORE.data[KEY_CORE][KEY_FRAMEWORK_VERSION]) + return get_framework_path(version) / "tools" / "sdk" / "ld" / name + + +def _flash_size_str(flash_ld_name: str) -> str: + """Flash size for elf2bin, derived from the ld script name (PIO logic).""" + match = re.search(r"\.flash\.(\d+)([mk])", flash_ld_name) + if not match: + raise EsphomeError(f"Cannot parse flash size from {flash_ld_name}") + return f"{match.group(1)}{match.group(2).upper()}" diff --git a/esphome/components/esp8266/__init__.py b/esphome/components/esp8266/__init__.py index 2161a902cb..6c1cd584bf 100644 --- a/esphome/components/esp8266/__init__.py +++ b/esphome/components/esp8266/__init__.py @@ -13,6 +13,7 @@ from esphome.const import ( CONF_FRAMEWORK, CONF_PLATFORM_VERSION, CONF_SOURCE, + CONF_TOOLCHAIN, CONF_VERSION, KEY_CORE, KEY_FRAMEWORK_VERSION, @@ -20,6 +21,7 @@ from esphome.const import ( KEY_TARGET_PLATFORM, PLATFORM_ESP8266, ThreadModel, + Toolchain, ) from esphome.core import ( CORE, @@ -33,7 +35,7 @@ from esphome.helpers import IS_MACOS, copy_file_if_changed from esphome.platformio.toolchain import copy_ccache_script from esphome.types import ConfigType -from .boards import BOARDS, ESP8266_LD_SCRIPTS +from .boards import BOARDS, ESP8266_BOARD_BUILD, ESP8266_LD_SCRIPTS from .const import ( CONF_EARLY_PIN_INIT, CONF_ENABLE_SERIAL, @@ -41,8 +43,10 @@ from .const import ( CONF_RESTORE_FROM_FLASH, KEY_BOARD, KEY_ESP8266, + KEY_FLASH_MODE, KEY_FLASH_SIZE, KEY_PIN_INITIAL_STATES, + KEY_SCANF_FLOAT, KEY_SERIAL1_REQUIRED, KEY_SERIAL_REQUIRED, KEY_WAVEFORM_REQUIRED, @@ -96,12 +100,57 @@ def set_core_data(config): config[CONF_FRAMEWORK][CONF_VERSION] ) CORE.data[KEY_ESP8266][KEY_BOARD] = config[CONF_BOARD] + CORE.data[KEY_ESP8266][KEY_FLASH_MODE] = config[CONF_BOARD_FLASH_MODE] CORE.data[KEY_ESP8266][KEY_PIN_INITIAL_STATES] = [ PinInitialState() for _ in range(16) ] return config +def _validate_toolchain(value: str) -> Toolchain: + return Toolchain( + cv.one_of(Toolchain.PLATFORMIO, Toolchain.ARDUINO, lower=True)(value) + ) + + +def _resolve_toolchain(config: ConfigType) -> ConfigType: + # Resolve toolchain: CLI (already on CORE.toolchain) > YAML > default. + if CORE.toolchain is None: + CORE.toolchain = config.get(CONF_TOOLCHAIN, Toolchain.PLATFORMIO) + if CORE.toolchain not in (Toolchain.PLATFORMIO, Toolchain.ARDUINO): + raise cv.Invalid( + f"Unsupported toolchain '{CORE.toolchain.value}' for ESP8266. " + "Supported toolchains are 'platformio' and 'arduino'." + ) + return config + + +def _validate_native_toolchain(config: ConfigType) -> ConfigType: + """Constraints of the native (non-PlatformIO) Arduino toolchain.""" + if not CORE.using_toolchain_arduino: + return config + conf = config[CONF_FRAMEWORK] + version = cv.Version.parse(conf[CONF_VERSION]) + if version < cv.Version(3, 1, 0): + raise cv.Invalid( + "'toolchain: arduino' requires framework version 3.1.0 or newer" + ) + if conf[CONF_SOURCE] != _format_framework_arduino_version(version): + raise cv.Invalid( + "'toolchain: arduino' does not support a custom framework source; " + "use 'toolchain: platformio'" + ) + if ( + config[CONF_BOARD] not in BOARDS + or config[CONF_BOARD] not in ESP8266_BOARD_BUILD + ): + raise cv.Invalid( + f"Board '{config[CONF_BOARD]}' is not supported by " + "'toolchain: arduino'; use 'toolchain: platformio'" + ) + return config + + def get_download_types(storage_json): """Binary-download entries for a built ESP8266 firmware. @@ -134,7 +183,11 @@ def _format_framework_arduino_version(ver: cv.Version) -> str: return f"~1.{ver.major}{ver.minor:02d}{ver.patch:02d}.0" if ver <= cv.Version(2, 6, 2): return f"~2.{ver.major}{ver.minor:02d}{ver.patch:02d}.0" - return f"~3.{ver.major}{ver.minor:02d}{ver.patch:02d}.0" + # Same encoding the native toolchain uses for its package download, so a + # custom-source check against this value cannot drift from what it fetches. + from esphome.arduino8266.framework import framework_package_version + + return f"~{framework_package_version(ver)}" # NOTE: Keep this in mind when updating the recommended version: @@ -242,8 +295,13 @@ CONFIG_SCHEMA = cv.All( cv.Optional(CONF_ENABLE_SERIAL1): cv.boolean, cv.Optional(CONF_ENABLE_FULL_PRINTF, default=False): cv.boolean, cv.Optional(CONF_ENABLE_SCANF_FLOAT): cv.boolean, + cv.Optional( + CONF_TOOLCHAIN, visibility=cv.Visibility.ADVANCED + ): _validate_toolchain, } ), + _resolve_toolchain, + _validate_native_toolchain, set_core_data, ) @@ -276,12 +334,13 @@ def check_rosetta() -> None: @coroutine_with_priority(CoroPriority.PLATFORM) async def to_code(config): + use_platformio = CORE.using_toolchain_platformio cg.add(esp8266_ns.setup_preferences()) - cg.add_platformio_option("lib_ldf_mode", "off") - cg.add_platformio_option("lib_compat_mode", "strict") - - cg.add_platformio_option("board", config[CONF_BOARD]) + if use_platformio: + cg.add_platformio_option("lib_ldf_mode", "off") + cg.add_platformio_option("lib_compat_mode", "strict") + cg.add_platformio_option("board", config[CONF_BOARD]) cg.add_build_flag("-DUSE_ESP8266") cg.set_cpp_standard("gnu++20") cg.add_define("ESPHOME_BOARD", config[CONF_BOARD]) @@ -297,28 +356,33 @@ async def to_code(config): "enabling scanf float support (~8KB flash)" ) - extra_scripts = [ - "pre:ccache.py", - "pre:testing_mode.py", - "pre:exclude_updater.py", - "pre:exclude_waveform.py", - "pre:relocate_ratetable.py", - ] - if not enable_scanf_float: - extra_scripts.append("pre:remove_float_scanf.py") - extra_scripts.append("post:post_build.py") - cg.add_platformio_option("extra_scripts", extra_scripts) + # The native toolchain reads this decision from CORE.data instead of the + # remove_float_scanf extra script. + CORE.data[KEY_ESP8266][KEY_SCANF_FLOAT] = bool(enable_scanf_float) + if use_platformio: + extra_scripts = [ + "pre:ccache.py", + "pre:testing_mode.py", + "pre:exclude_updater.py", + "pre:exclude_waveform.py", + "pre:relocate_ratetable.py", + ] + if not enable_scanf_float: + extra_scripts.append("pre:remove_float_scanf.py") + extra_scripts.append("post:post_build.py") + cg.add_platformio_option("extra_scripts", extra_scripts) conf = config[CONF_FRAMEWORK] - cg.add_platformio_option("framework", "arduino") cg.add_build_flag("-DUSE_ARDUINO") cg.add_build_flag("-DUSE_ESP8266_FRAMEWORK_ARDUINO") cg.add_build_flag("-Wno-nonnull-compare") - cg.add_platformio_option("platform", conf[CONF_PLATFORM_VERSION]) - cg.add_platformio_option( - "platform_packages", - [f"platformio/framework-arduinoespressif8266@{conf[CONF_SOURCE]}"], - ) + if use_platformio: + cg.add_platformio_option("framework", "arduino") + cg.add_platformio_option("platform", conf[CONF_PLATFORM_VERSION]) + cg.add_platformio_option( + "platform_packages", + [f"platformio/framework-arduinoespressif8266@{conf[CONF_SOURCE]}"], + ) # Default for platformio is LWIP2_LOW_MEMORY with: # - MSS=536 @@ -356,10 +420,12 @@ async def to_code(config): # Force-include inline std::__throw_* overrides so GCC dead-strips the unused # libstdc++ error message strings (e.g. "basic_string::_M_create") from DRAM. # See throw_stubs.h for details. Must be prepended before , so this - # uses build_src_flags with -include. - cg.add_platformio_option( - "build_src_flags", "-include esphome/components/esp8266/throw_stubs.h" - ) + # uses build_src_flags with -include. The native toolchain's build + # generator adds the equivalent flag itself. + if use_platformio: + cg.add_platformio_option( + "build_src_flags", "-include esphome/components/esp8266/throw_stubs.h" + ) # In testing mode, fake larger memory to allow linking grouped component tests # Real ESP8266 hardware only has 32KB IRAM and ~80KB RAM, but for CI testing @@ -386,7 +452,10 @@ async def to_code(config): # implementation in the Arduino ESP8266 core. cg.add_build_flag("-Wl,--wrap=millis") - cg.add_platformio_option("board_build.flash_mode", config[CONF_BOARD_FLASH_MODE]) + if use_platformio: + cg.add_platformio_option( + "board_build.flash_mode", config[CONF_BOARD_FLASH_MODE] + ) ver: cv.Version = CORE.data[KEY_CORE][KEY_FRAMEWORK_VERSION] cg.add_define( @@ -394,7 +463,7 @@ async def to_code(config): cg.RawExpression(f"VERSION_CODE({ver.major}, {ver.minor}, {ver.patch})"), ) - if config[CONF_BOARD] in BOARDS: + if use_platformio and config[CONF_BOARD] in BOARDS: flash_size = BOARDS[config[CONF_BOARD]][KEY_FLASH_SIZE] ld_scripts = ESP8266_LD_SCRIPTS[flash_size] @@ -446,8 +515,24 @@ async def finalize_serial_config() -> None: cg.add_build_flag("-DNO_GLOBAL_SERIAL1") +# Called by __main__.compile_program; returning False falls through to the +# PlatformIO toolchain. +def run_compile(args, config: ConfigType) -> bool: + if CORE.using_toolchain_platformio: + return False + from esphome.arduino8266 import toolchain + + if toolchain.run_compile(config, CORE.verbose) != 0: + raise EsphomeError("ESP8266 native build failed") + return True + + # Called by writer.py def copy_files() -> None: + if not CORE.using_toolchain_platformio: + # The extra scripts are PlatformIO/SCons-only; the native toolchain + # applies their logic in the build generator instead. + return dir = Path(__file__).parent for script in ( "post_build", @@ -505,13 +590,24 @@ ESP8266_EXCEPTION_CODES = { def _decode_pc(config, addr): - from esphome.platformio import toolchain + if CORE.using_toolchain_arduino: + from esphome.arduino8266 import toolchain as native_toolchain - idedata = toolchain.get_idedata(config) - if not idedata.addr2line_path or not idedata.firmware_elf_path: - _LOGGER.debug("decode_pc no addr2line") - return - command = [idedata.addr2line_path, "-pfiaC", "-e", idedata.firmware_elf_path, addr] + addr2line = native_toolchain.get_addr2line_path() + elf = native_toolchain.get_elf_path() + if not addr2line.is_file() or not elf.is_file(): + _LOGGER.debug("decode_pc no addr2line") + return + addr2line, elf = str(addr2line), str(elf) + else: + from esphome.platformio import toolchain + + idedata = toolchain.get_idedata(config) + if not idedata.addr2line_path or not idedata.firmware_elf_path: + _LOGGER.debug("decode_pc no addr2line") + return + addr2line, elf = idedata.addr2line_path, idedata.firmware_elf_path + command = [addr2line, "-pfiaC", "-e", elf, addr] try: translation = subprocess.check_output(command, close_fds=False).decode().strip() except Exception: # noqa: BLE001 # pylint: disable=broad-except diff --git a/esphome/components/esp8266/boards.py b/esphome/components/esp8266/boards.py index 02bfa9e662..4571cff1c3 100644 --- a/esphome/components/esp8266/boards.py +++ b/esphome/components/esp8266/boards.py @@ -360,3 +360,96 @@ BOARDS = { "flash_size": FLASH_SIZE_4_MB, }, } + +# Per-board Arduino core build metadata for the native (PlatformIO-free) +# toolchain: the variant directory (supplies pins_arduino.h) and the +# board-identity defines the PlatformIO builder passes via build.extra_flags. +# -DESP8266 and -DARDUINO_ARCH_ESP8266 are shared by every board and added by +# the generator; only the per-board defines are listed here. +# Generated from platform-espressif8266 boards/*.json (see BOARDS note above). +ESP8266_BOARD_BUILD = { + "agruminolemon": { + "variant": "agruminolemonv4", + "defines": ("ARDUINO_ESP8266_AGRUMINO_LEMON_V4",), + }, + "d1": {"variant": "d1", "defines": ("ARDUINO_ESP8266_WEMOS_D1R1",)}, + "d1_mini": {"variant": "d1_mini", "defines": ("ARDUINO_ESP8266_WEMOS_D1MINI",)}, + "d1_mini_lite": { + "variant": "d1_mini", + "defines": ("ARDUINO_ESP8266_WEMOS_D1MINILITE",), + }, + "d1_mini_pro": { + "variant": "d1_mini", + "defines": ("ARDUINO_ESP8266_WEMOS_D1MINIPRO",), + }, + "d1_wroom_02": { + "variant": "d1_mini", + "defines": ("ARDUINO_ESP8266_WEMOS_D1WROOM02",), + }, + "eduinowifi": { + "variant": "eduinowifi", + "defines": ("ARDUINO_ESP8266_SCHIRMILABS_EDUINO_WIFI",), + }, + "esp01": {"variant": "generic", "defines": ("ARDUINO_ESP8266_ESP01",)}, + "esp01_1m": {"variant": "generic", "defines": ("ARDUINO_ESP8266_ESP01",)}, + "esp07": {"variant": "generic", "defines": ("ARDUINO_ESP8266_ESP07",)}, + "esp07s": {"variant": "nodemcu", "defines": ("ARDUINO_ESP8266_ESP07",)}, + "esp12e": {"variant": "nodemcu", "defines": ("ARDUINO_ESP8266_ESP12",)}, + "esp210": {"variant": "generic", "defines": ("ARDUINO_ESP8266_ESP210",)}, + "esp8285": {"variant": "esp8285", "defines": ("ARDUINO_ESP8266_ESP01",)}, + "esp_wroom_02": { + "variant": "nodemcu", + "defines": ("ARDUINO_ESP8266_ESP_WROOM_02",), + }, + "espduino": {"variant": "ESPDuino", "defines": ("ARDUINO_ESP8266_ESP13",)}, + "espectro": {"variant": "espectro", "defines": ("ARDUINO_ESP8266_ESPECTRO_CORE",)}, + "espino": {"variant": "espino", "defines": ("ARDUINO_ESP8266_ESP12",)}, + "espinotee": {"variant": "espinotee", "defines": ("ARDUINO_ESP8266_ESP13",)}, + "espmxdevkit": { + "variant": "esp8285", + "defines": ("ARDUINO_ESP8266_ESP01", "LED_BUILTIN=16"), + }, + "espresso_lite_v1": { + "variant": "espresso_lite_v1", + "defines": ("ARDUINO_ESP8266_ESPRESSO_LITE_V1",), + }, + "espresso_lite_v2": { + "variant": "espresso_lite_v2", + "defines": ("ARDUINO_ESP8266_ESPRESSO_LITE_V2",), + }, + "gen4iod": {"variant": "generic", "defines": ("ARDUINO_GEN4_IOD",)}, + "heltec_wifi_kit_8": { + "variant": "wifi_kit_8", + "defines": ("ARDUINO_wifi_kit_8",), + }, + "huzzah": {"variant": "adafruit", "defines": ("ARDUINO_ESP8266_ADAFRUIT_HUZZAH",)}, + "inventone": {"variant": "inventone", "defines": ("ARDUINO_ESP8266_INVENT_ONE",)}, + "modwifi": {"variant": "generic", "defines": ("ARDUINO_MOD_WIFI_ESP8266",)}, + "nodemcu": {"variant": "nodemcu", "defines": ("ARDUINO_ESP8266_NODEMCU",)}, + "nodemcuv2": {"variant": "nodemcu", "defines": ("ARDUINO_ESP8266_NODEMCU_ESP12E",)}, + "oak": {"variant": "oak", "defines": ("ARDUINO_ESP8266_OAK",)}, + "phoenix_v1": { + "variant": "phoenix_v1", + "defines": ("ARDUINO_ESP8266_PHOENIX_V1",), + }, + "phoenix_v2": { + "variant": "phoenix_v2", + "defines": ("ARDUINO_ESP8266_PHOENIX_V2",), + }, + "sonoff_basic": {"variant": "itead", "defines": ("ARDUINO_ESP8266_SONOFF_BASIC",)}, + "sonoff_s20": {"variant": "itead", "defines": ("ARDUINO_ESP8266_SONOFF_S20",)}, + "sonoff_sv": {"variant": "itead", "defines": ("ARDUINO_ESP8266_SONOFF_SV",)}, + "sonoff_th": {"variant": "itead", "defines": ("ARDUINO_ESP8266_SONOFF_TH",)}, + "sparkfunBlynk": {"variant": "thing", "defines": ("ARDUINO_ESP8266_THING",)}, + "thing": {"variant": "thing", "defines": ("ARDUINO_ESP8266_THING",)}, + "thingdev": {"variant": "thing", "defines": ("ARDUINO_ESP8266_THING_DEV",)}, + "wifi_slot": {"variant": "wifi_slot", "defines": ("ARDUINO_AMPERKA_WIFI_SLOT",)}, + "wifiduino": {"variant": "wifiduino", "defines": ("ARDUINO_WIFIDUINO_ESP8266",)}, + "wifinfo": {"variant": "wifinfo", "defines": ("ARDUINO_WIFINFO",)}, + "wio_link": {"variant": "wiolink", "defines": ("ARDUINO_ESP8266_WIO_LINK",)}, + "wio_node": {"variant": "nodemcu", "defines": ("ARDUINO_ESP8266_ESP_WROOM_02",)}, + "xinabox_cw01": { + "variant": "xinabox", + "defines": ("ARDUINO_ESP8266_XINABOX_CW01",), + }, +} diff --git a/esphome/components/esp8266/build_surgery.py b/esphome/components/esp8266/build_surgery.py new file mode 100644 index 0000000000..981204a4ff --- /dev/null +++ b/esphome/components/esp8266/build_surgery.py @@ -0,0 +1,59 @@ +"""Linker-script surgery shared with the native (PlatformIO-free) toolchain. + +These mirror the PlatformIO extra scripts in this directory +(``relocate_ratetable.py.script`` and ``testing_mode.py.script``), which run +inside SCons and must stay self-contained. The native build generator applies +the same patches to the linker scripts it generates, so the logic lives here +as plain functions. Keep both in sync when changing either. +""" + +from __future__ import annotations + +import re + +# Move the NONOS SDK wifi rate tables from flash to DRAM; see +# relocate_ratetable.py.script for the full background (NONOS SDK issue 320). +RATETABLE_RULE = "*libnet80211.a:ieee80211_phy.o(.irom.text .irom.text.*)" +# Match the whole line: "_data_start" is also a substring of the +# "_dport0_data_start" line in the earlier .dport0.data section +_RATETABLE_ANCHOR = re.compile(r"^\s*_data_start = ABSOLUTE\(\.\);", re.MULTILINE) + +# Memory sizes for testing mode (allow larger builds for CI component grouping) +TESTING_IRAM_SIZE = "0x200000" # 2MB +TESTING_DRAM_SIZE = "0x200000" # 2MB +TESTING_FLASH_SIZE = "0x2000000" # 32MB + + +def relocate_ratetable(content: str) -> str: + """Insert the rate-table DRAM rule into a generated common linker script.""" + if RATETABLE_RULE in content: + return content + match = _RATETABLE_ANCHOR.search(content) + if match is None: + raise RuntimeError( + "'_data_start' anchor not found in the generated linker script; " + "cannot apply wifi rate table DRAM relocation " + "(has the Arduino core linker script changed?)" + ) + insert_pos = match.end() + return ( + content[:insert_pos] + + "\n /* ESPHome: wifi rate tables must live in DRAM, see NONOS SDK issue 320 */" + + f"\n {RATETABLE_RULE}" + + content[insert_pos:] + ) + + +def _patch_segment_size(content: str, segment_name: str, new_size: str) -> str: + pattern = ( + rf"({segment_name}\s*:\s*org\s*=\s*0x[0-9a-fA-F]+\s*,\s*len\s*=\s*)" + r"0x[0-9a-fA-F]+" + ) + return re.sub(pattern, rf"\g<1>{new_size}", content) + + +def apply_testing_memory_patches(content: str) -> str: + """Enlarge IRAM/DRAM/flash segments so grouped CI test builds can link.""" + content = _patch_segment_size(content, "iram1_0_seg", TESTING_IRAM_SIZE) + content = _patch_segment_size(content, "dram0_0_seg", TESTING_DRAM_SIZE) + return _patch_segment_size(content, "irom0_0_seg", TESTING_FLASH_SIZE) diff --git a/esphome/components/esp8266/const.py b/esphome/components/esp8266/const.py index 3e89ab989f..8729007e3a 100644 --- a/esphome/components/esp8266/const.py +++ b/esphome/components/esp8266/const.py @@ -15,6 +15,9 @@ CONF_ENABLE_SERIAL1 = "enable_serial1" KEY_WAVEFORM_REQUIRED = "waveform_required" KEY_SERIAL_REQUIRED = "serial_required" KEY_SERIAL1_REQUIRED = "serial1_required" +# Set for the native (non-PlatformIO) toolchain's build generator +KEY_FLASH_MODE = "flash_mode" +KEY_SCANF_FLOAT = "scanf_float" # esp8266 namespace is already defined by arduino, manually prefix esphome esp8266_ns = cg.global_ns.namespace("esphome").namespace("esp8266") diff --git a/esphome/const.py b/esphome/const.py index 0dd948544f..6e9378ec9a 100644 --- a/esphome/const.py +++ b/esphome/const.py @@ -21,6 +21,8 @@ class Toolchain(StrEnum): PLATFORMIO = "platformio" ESP_IDF = "esp-idf" SDK_NRF = "sdk-nrf" + # ESP8266: the Arduino core built directly (no PlatformIO) + ARDUINO = "arduino" class Platform(StrEnum): diff --git a/esphome/core/__init__.py b/esphome/core/__init__.py index 534b740a5d..e7e17fa082 100644 --- a/esphome/core/__init__.py +++ b/esphome/core/__init__.py @@ -979,6 +979,10 @@ class EsphomeCore: def using_toolchain_sdk_nrf(self): return self.toolchain == Toolchain.SDK_NRF + @property + def using_toolchain_arduino(self): + return self.toolchain == Toolchain.ARDUINO + @property def using_zephyr(self): return self.target_framework == "zephyr" diff --git a/esphome/espidf/component.py b/esphome/espidf/component.py index aa6f10c261..b9d1f21753 100644 --- a/esphome/espidf/component.py +++ b/esphome/espidf/component.py @@ -41,34 +41,10 @@ def _idf_framework() -> str: def _apply_extra_script(component: IDFComponent) -> None: - """Run a PIO ``extraScript`` and fold its captured env vars into - ``component.data["build"]["flags"]`` so the existing -L/-l/-D - extraction in ``generate_cmakelists_txt`` picks them up.""" - extra_script = component.data.get("build", {}).get("extraScript") - if not extra_script: - return - # Resolve and confine to the library's source dir so a malicious - # library.json can't escape (e.g. ``"extraScript": "../../etc/passwd"``). - source_path = component.source_dir - library_root = source_path.resolve() - script_path = (source_path / extra_script).resolve() - if not script_path.is_relative_to(library_root) or not script_path.is_file(): - return from esphome.components.esp32 import get_esp32_variant - from esphome.espidf.extra_script import captured_as_build_flags, run_extra_script + from esphome.espidf.extra_script import apply_extra_script - idf_target = variant_to_idf_target(get_esp32_variant()) - result = run_extra_script( - script_path, library_dir=source_path, idf_target=idf_target - ) - extra_flags = captured_as_build_flags(result, library_dir=source_path) - if not extra_flags: - return - flags = component.data.setdefault("build", {}).setdefault("flags", []) - if isinstance(flags, str): - flags = [flags] - flags.extend(extra_flags) - component.data["build"]["flags"] = flags + apply_extra_script(component, lambda: variant_to_idf_target(get_esp32_variant())) def generate_cmakelists_txt(component: IDFComponent) -> str: diff --git a/esphome/espidf/extra_script.py b/esphome/espidf/extra_script.py index 487fef7cc1..d9d929d47a 100644 --- a/esphome/espidf/extra_script.py +++ b/esphome/espidf/extra_script.py @@ -28,13 +28,54 @@ Caveats from __future__ import annotations +from collections.abc import Callable from dataclasses import dataclass, field import logging import os from pathlib import Path +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from esphome.platformio.library import ConvertedLibrary _LOGGER = logging.getLogger(__name__) + +def apply_extra_script( + component: ConvertedLibrary, idf_target: str | Callable[[], str] +) -> None: + """Run a library's PIO ``extraScript`` and fold its captured env vars into + ``component.data["build"]["flags"]`` so the backend's -L/-l/-D extraction + picks them up. Shared by the ESP-IDF and ESP8266 Arduino backends. + + ``idf_target`` may be a callable so a backend whose target lookup needs + build state (the esp32 variant) resolves it only when a script will run. + """ + extra_script = component.data.get("build", {}).get("extraScript") + if not extra_script: + return + # Resolve and confine to the library's source dir so a malicious + # library.json can't escape (e.g. ``"extraScript": "../../etc/passwd"``). + source_path = component.source_dir + library_root = source_path.resolve() + script_path = (source_path / extra_script).resolve() + if not script_path.is_relative_to(library_root) or not script_path.is_file(): + return + if callable(idf_target): + idf_target = idf_target() + result = run_extra_script( + script_path, library_dir=source_path, idf_target=idf_target + ) + extra_flags = captured_as_build_flags(result, library_dir=source_path) + if not extra_flags: + return + flags = component.data.setdefault("build", {}).setdefault("flags", []) + if isinstance(flags, str): + flags = [flags] + flags.extend(extra_flags) + component.data["build"]["flags"] = flags + + # Keys we know how to translate back into ESPHome's build-flag pipeline. # Other env.Append kwargs are recorded but ignored downstream. _CAPTURED_KEYS = frozenset({"LIBPATH", "LIBS", "CPPDEFINES", "LINKFLAGS", "CPPFLAGS"}) diff --git a/esphome/espidf/idedata.py b/esphome/espidf/idedata.py index 0047d568e2..355a395ebc 100644 --- a/esphome/espidf/idedata.py +++ b/esphome/espidf/idedata.py @@ -136,6 +136,9 @@ def _parse_entry(entry: dict) -> tuple[str, list[str], list[str], list[str]]: raw = os.path.normpath(directory / raw) return raw.replace("\\", "/") + # A ccache-wrapped command ("ccache g++ ...") names the compiler second. + if Path(tokens[0]).stem == "ccache": + tokens = tokens[1:] # token0 is the compiler path; the rest of the command already uses forward # slashes on Windows, so normalize it too for a consistent idedata file. cxx_path = tokens[0].replace("\\", "/") @@ -219,6 +222,38 @@ def _cc_path_from_cxx(cxx_path: str) -> str: return f"{stem}{suffix}" +def load_or_build_idedata( + compile_commands: Path, elf_path: Path, cache: Path +) -> dict | None: + """Return idedata for a compile_commands.json build, cached on mtime. + + Shared by the native ESP-IDF and ESP8266 Arduino toolchains. Returns None + when the compile DB doesn't exist yet (nothing was built). + """ + if not compile_commands.is_file(): + _LOGGER.debug("No %s yet; skipping idedata generation", compile_commands) + return None + + if cache.is_file() and cache.stat().st_mtime >= compile_commands.stat().st_mtime: + try: + cached = json.loads(cache.read_text(encoding="utf-8")) + except ValueError: + pass + else: + # Caches written before cc_path was emitted stay newer than + # compile_commands.json forever, so rebuild them on the field rather + # than on the timestamp. Check the type too: a corrupted cache can + # still be valid JSON, and "in" would match a substring of a string. + if isinstance(cached, dict) and "cc_path" in cached: + return cached + + data = idedata_from_build(compile_commands) + data["prog_path"] = str(elf_path) + cache.parent.mkdir(parents=True, exist_ok=True) + cache.write_text(json.dumps(data, indent=2) + "\n", encoding="utf-8") + return data + + def idedata_from_build(compile_commands: Path) -> dict: """Parse compile_commands.json into the idedata fields consumers expect. diff --git a/esphome/espidf/size_summary.py b/esphome/espidf/size_summary.py index 7a5305ff0c..9af2dcc9e4 100644 --- a/esphome/espidf/size_summary.py +++ b/esphome/espidf/size_summary.py @@ -67,7 +67,7 @@ def _find_app_partition_size(partitions_csv: Path) -> int: raise ValueError(f"No app+factory or app+ota_0 partition in {partitions_csv}") -def _format_bar(used: int, total: int) -> str: +def format_bar(used: int, total: int) -> str: """Match PlatformIO's ``_format_availale_bytes`` (pioupload.py) exactly.""" pct_raw = used / total if total else 0 blocks = 10 @@ -99,7 +99,7 @@ def print_summary(size_json: Path, partitions_csv: Path | None) -> None: ram_used = ram_region.get("used") ram_total = ram_region.get("size") if ram_total and ram_used is not None: - print(f"RAM: {_format_bar(ram_used, ram_total)}") + print(f"RAM: {format_bar(ram_used, ram_total)}") image_size = data.get("image_size") if image_size is None or partitions_csv is None: @@ -109,4 +109,4 @@ def print_summary(size_json: Path, partitions_csv: Path | None) -> None: except ValueError as e: _LOGGER.debug("Skipping Flash summary: %s", e) return - print(f"Flash: {_format_bar(image_size, app_size)}") + print(f"Flash: {format_bar(image_size, app_size)}") diff --git a/esphome/espidf/toolchain.py b/esphome/espidf/toolchain.py index bb6452acf2..7538aa91da 100644 --- a/esphome/espidf/toolchain.py +++ b/esphome/espidf/toolchain.py @@ -504,32 +504,13 @@ def get_idedata() -> dict | None: idedata fields IDE integrations and clang-tidy expect, cached alongside the PlatformIO idedata path. Returns None if the compile DB doesn't exist yet. """ - from esphome.espidf.idedata import idedata_from_build + from esphome.espidf.idedata import load_or_build_idedata - compile_commands = CORE.relative_build_path("build", "compile_commands.json") - if not compile_commands.is_file(): - _LOGGER.debug("No %s yet; skipping idedata generation", compile_commands) - return None - - cache = CORE.relative_internal_path("idedata", f"{CORE.name}.json") - if cache.is_file() and cache.stat().st_mtime >= compile_commands.stat().st_mtime: - try: - cached = json.loads(cache.read_text(encoding="utf-8")) - except ValueError: - pass - else: - # Caches written before cc_path was emitted stay newer than - # compile_commands.json forever, so rebuild them on the field rather - # than on the timestamp. Check the type too: a corrupted cache can - # still be valid JSON, and "in" would match a substring of a string. - if isinstance(cached, dict) and "cc_path" in cached: - return cached - - data = idedata_from_build(compile_commands) - data["prog_path"] = str(get_elf_path()) - cache.parent.mkdir(parents=True, exist_ok=True) - cache.write_text(json.dumps(data, indent=2) + "\n", encoding="utf-8") - return data + return load_or_build_idedata( + CORE.relative_build_path("build", "compile_commands.json"), + get_elf_path(), + CORE.relative_internal_path("idedata", f"{CORE.name}.json"), + ) def create_factory_bin() -> bool: diff --git a/esphome/platformio/library.py b/esphome/platformio/library.py index ee0a758a31..22a86af34f 100644 --- a/esphome/platformio/library.py +++ b/esphome/platformio/library.py @@ -469,7 +469,7 @@ def _parse_library_json(library_json_path: PathType): return json.load(fp) -def _parse_library_properties(library_properties_path: PathType): +def parse_library_properties(library_properties_path: PathType): """ Parse a key-value platformio .properties style file into a dictionary. @@ -553,7 +553,7 @@ def _resolve_registry_version( return owner, name, best["name"], pkgfile["download_url"] -def _normalize_dependencies(dependencies: Any) -> list[dict]: +def normalize_dependencies(dependencies: Any) -> list[dict]: """Normalize a library manifest's ``dependencies`` to a list of dicts. PIO's library.json accepts both the list-of-dicts form and the shorthand @@ -840,7 +840,7 @@ def convert_libraries( if has_json: component.data = _parse_library_json(library_json_path) elif has_properties: - component.data = _parse_library_properties(library_properties_path) + component.data = parse_library_properties(library_properties_path) else: # For a local library a missing manifest is user input, so raise # EsphomeError (clean CLI message) like the missing-directory case; @@ -869,7 +869,7 @@ def convert_libraries( # Requirements changed (we got past the short-circuit above), so # (re)walk this component's dependencies. node.edges = set() - for dependency in _normalize_dependencies(component.data.get("dependencies")): + for dependency in normalize_dependencies(component.data.get("dependencies")): if "name" not in dependency or "version" not in dependency: continue try: diff --git a/esphome/writer.py b/esphome/writer.py index 866377d2f5..26d091a823 100644 --- a/esphome/writer.py +++ b/esphome/writer.py @@ -660,11 +660,17 @@ def clean_all(configuration: list[str]): # that live outside it. import platformdirs + from esphome.arduino8266.framework import get_arduino8266_tools_path from esphome.components.nrf52.framework import get_sdk_nrf_tools_path from esphome.espidf.framework import get_idf_tools_path cache_root = Path(platformdirs.user_cache_dir("esphome", appauthor=False)).resolve() - for install_path in (cache_root, get_idf_tools_path(), get_sdk_nrf_tools_path()): + for install_path in ( + cache_root, + get_idf_tools_path(), + get_sdk_nrf_tools_path(), + get_arduino8266_tools_path(), + ): if install_path.is_dir(): _LOGGER.info("Deleting %s", install_path) rmtree(install_path) diff --git a/script/determine-jobs.py b/script/determine-jobs.py index e4d002975c..bac890d26e 100755 --- a/script/determine-jobs.py +++ b/script/determine-jobs.py @@ -610,6 +610,69 @@ def should_run_esp32_platformio(branch: str | None = None) -> bool: return bool(esp32_platformio_components_to_test(branch)) +# Components tested by the native (PlatformIO-free) ESP8266 Arduino toolchain +# compile-test job. The regular component matrix builds esp8266 with the +# default platformio toolchain; this list is the `--toolchain arduino` smoke +# test, chosen to exercise the core, the bundled libraries (ESP8266WiFi, +# ESP8266mDNS, Wire, SPI, DNSServer, Hash), the converted registry libraries +# (ESPAsyncTCP/WebServer, AsyncMqttClient, NeoPixelBus), and the waveform path. +ESP8266_NATIVE_TEST_COMPONENTS = frozenset( + { + "esp8266", + "api", + "web_server", + "captive_portal", + "mqtt", + "esp8266_pwm", + "neopixelbus", + "bme280_i2c", + "uart", + } +) + +# Infrastructure whose changes always trigger the native ESP8266 compile test. +ESP8266_NATIVE_TRIGGER_PATH_PREFIXES = ("esphome/arduino8266/",) +ESP8266_NATIVE_TRIGGER_FILES = frozenset( + { + "esphome/build_gen/arduino8266.py", + "esphome/components/esp8266/build_surgery.py", + "esphome/components/esp8266/boards.py", + "script/test_build_components.py", + ".github/workflows/ci.yml", + } +) + + +def _esp8266_native_path_or_file_trigger(files: list[str]) -> bool: + """Whether any changed file is native-ESP8266 infrastructure / harness.""" + for file in files: + if file in ESP8266_NATIVE_TRIGGER_FILES: + return True + if any( + file.startswith(prefix) for prefix in ESP8266_NATIVE_TRIGGER_PATH_PREFIXES + ): + return True + return False + + +def esp8266_native_components_to_test(branch: str | None = None) -> list[str]: + """Subset of ``ESP8266_NATIVE_TEST_COMPONENTS`` the job needs to compile. + + Same narrowing logic as ``esp32_platformio_components_to_test``: the full + list on core or infrastructure changes, otherwise the intersection with + the changed-component dependency closure (empty list skips the job). + """ + files = changed_files(branch) + + if core_changed(files) or _esp8266_native_path_or_file_trigger(files): + return sorted(ESP8266_NATIVE_TEST_COMPONENTS) + + component_files = [f for f in files if filter_component_and_test_files(f)] + changed = get_components_with_dependencies(component_files, True) + + return sorted(ESP8266_NATIVE_TEST_COMPONENTS & set(changed)) + + def determine_cpp_unit_tests( branch: str | None = None, ) -> tuple[bool, list[str]]: @@ -1205,6 +1268,8 @@ def main() -> None: run_device_builder = True esp32_platformio_components = sorted(ESP32_PLATFORMIO_TEST_COMPONENTS) run_esp32_platformio = True + esp8266_native_components = sorted(ESP8266_NATIVE_TEST_COMPONENTS) + run_esp8266_native = True else: integration_run_all, integration_test_files = determine_integration_tests( args.branch @@ -1216,6 +1281,8 @@ def main() -> None: run_device_builder = should_run_device_builder(args.branch) esp32_platformio_components = esp32_platformio_components_to_test(args.branch) run_esp32_platformio = bool(esp32_platformio_components) + esp8266_native_components = esp8266_native_components_to_test(args.branch) + run_esp8266_native = bool(esp8266_native_components) run_integration, integration_test_buckets = _compute_integration_test_buckets( integration_run_all, integration_test_files ) @@ -1410,6 +1477,8 @@ def main() -> None: "device_builder": run_device_builder, "esp32_platformio": run_esp32_platformio, "esp32_platformio_components": ",".join(esp32_platformio_components), + "esp8266_native": run_esp8266_native, + "esp8266_native_components": ",".join(esp8266_native_components), "changed_components": changed_components, "changed_components_with_tests": changed_components_with_tests, "directly_changed_components_with_tests": list(directly_changed_with_tests), diff --git a/tests/unit_tests/build_gen/test_arduino8266.py b/tests/unit_tests/build_gen/test_arduino8266.py new file mode 100644 index 0000000000..60db123598 --- /dev/null +++ b/tests/unit_tests/build_gen/test_arduino8266.py @@ -0,0 +1,261 @@ +"""Drift tests for the native ESP8266 Arduino build generator. + +These pin the build spec transliterated from the PlatformIO builder +(framework-arduinoespressif8266/tools/platformio-build.py and +platform-espressif8266/builder/main.py) so a change on either side of the +toolchain seam is caught: the knob-define precedence, the define/flag sets, +the link line, and the core source exclusions must keep matching what the +PlatformIO toolchain produces for the same configuration. +""" + +from __future__ import annotations + +from pathlib import Path +from unittest.mock import patch + +import pytest + +from esphome.components.esp8266.boards import BOARDS, ESP8266_BOARD_BUILD +from esphome.components.esp8266.const import ( + KEY_BOARD, + KEY_ESP8266, + KEY_FLASH_MODE, + KEY_SCANF_FLOAT, +) +import esphome.config_validation as cv +from esphome.const import KEY_CORE, KEY_FRAMEWORK_VERSION +from esphome.core import CORE, EsphomeError + + +@pytest.fixture(autouse=True) +def _setup_core(tmp_path: Path) -> None: + CORE.name = "test8266" + CORE.build_path = tmp_path + CORE.testing_mode = False + CORE.cpp_standard = "gnu++20" + CORE.data[KEY_CORE] = {KEY_FRAMEWORK_VERSION: cv.Version(3, 1, 2)} + CORE.data[KEY_ESP8266] = { + KEY_BOARD: "nodemcuv2", + KEY_FLASH_MODE: "dout", + KEY_SCANF_FLOAT: False, + } + + +def _set_flags(*flags: str) -> None: + CORE.build_flags = set(flags) + + +def test_board_build_covers_every_board() -> None: + """Every supported board must have variant/define metadata.""" + assert set(BOARDS) <= set(ESP8266_BOARD_BUILD) + + +def test_build_config_defaults() -> None: + from esphome.build_gen.arduino8266 import _flag_defines, _resolve_build_config + + _set_flags() + config = _resolve_build_config(_flag_defines()) + assert config.nonosdk == "NONOSDK22x_190703" + assert config.lwip_lib == "lwip2-536-feat" + assert not config.exceptions + assert config.vtables == "VTABLES_IN_FLASH" + assert config.knob_defines == [ + "NONOSDK22x_190703=1", + "TCP_MSS=536", + "LWIP_FEATURES=1", + "LWIP_IPV6=0", + ] + assert config.mmu_defines == ["MMU_IRAM_SIZE=0x8000", "MMU_ICACHE_SIZE=0x8000"] + + +def test_build_config_esphome_lwip_knob() -> None: + """The lwIP variant ESPHome selects maps to the same defines and library + as the PlatformIO builder.""" + from esphome.build_gen.arduino8266 import _flag_defines, _resolve_build_config + + _set_flags("-DPIO_FRAMEWORK_ARDUINO_LWIP2_HIGHER_BANDWIDTH_LOW_FLASH") + config = _resolve_build_config(_flag_defines()) + assert config.lwip_lib == "lwip2-1460" + assert "TCP_MSS=1460" in config.knob_defines + assert "LWIP_FEATURES=0" in config.knob_defines + assert "LWIP_IPV6=0" in config.knob_defines + + +def test_build_config_knobs() -> None: + from esphome.build_gen.arduino8266 import _flag_defines, _resolve_build_config + + _set_flags( + "-DPIO_FRAMEWORK_ARDUINO_ESPRESSIF_SDK305", + "-DPIO_FRAMEWORK_ARDUINO_ENABLE_EXCEPTIONS", + "-DPIO_FRAMEWORK_ARDUINO_MMU_CACHE16_IRAM48", + "-DVTABLES_IN_DRAM", + ) + config = _resolve_build_config(_flag_defines()) + assert config.nonosdk == "NONOSDK305" + assert config.exceptions + assert config.vtables == "VTABLES_IN_DRAM" + assert config.mmu_defines == ["MMU_IRAM_SIZE=0xC000", "MMU_ICACHE_SIZE=0x4000"] + + +def test_build_config_mmu_custom_requires_sizes() -> None: + from esphome.build_gen.arduino8266 import _flag_defines, _resolve_build_config + + _set_flags("-DPIO_FRAMEWORK_ARDUINO_MMU_CUSTOM") + with pytest.raises(EsphomeError, match="MMU_IRAM_SIZE"): + _resolve_build_config(_flag_defines()) + + _set_flags( + "-DPIO_FRAMEWORK_ARDUINO_MMU_CUSTOM", + "-DMMU_IRAM_SIZE=0xC000", + "-DMMU_ICACHE_SIZE=0x4000", + ) + config = _resolve_build_config(_flag_defines()) + assert sorted(config.mmu_defines) == [ + "MMU_ICACHE_SIZE=0x4000", + "MMU_IRAM_SIZE=0xC000", + ] + + +def test_defines_match_platformio_builder() -> None: + """The exact define set the PlatformIO builder passes for nodemcuv2/dout.""" + from esphome.build_gen.arduino8266 import ( + _defines_flags, + _flag_defines, + _resolve_build_config, + ) + + _set_flags("-DPIO_FRAMEWORK_ARDUINO_LWIP2_HIGHER_BANDWIDTH_LOW_FLASH") + assert _defines_flags( + _resolve_build_config(_flag_defines()), + "dout", + "nodemcuv2", + ESP8266_BOARD_BUILD["nodemcuv2"]["defines"], + ) == [ + "-DF_CPU=80000000L", + "-D__ets__", + "-DICACHE_FLASH", + "-D_GNU_SOURCE", + "-DARDUINO=10805", + '-DARDUINO_BOARD=\\"PLATFORMIO_NODEMCUV2\\"', + '-DARDUINO_BOARD_ID=\\"nodemcuv2\\"', + "-DFLASHMODE_DOUT", + "-DLWIP_OPEN_SRC", + "-DNONOSDK22x_190703=1", + "-DTCP_MSS=1460", + "-DLWIP_FEATURES=0", + "-DLWIP_IPV6=0", + "-DVTABLES_IN_FLASH", + "-DMMU_IRAM_SIZE=0x8000", + "-DMMU_ICACHE_SIZE=0x8000", + "-DESP8266", + "-DARDUINO_ARCH_ESP8266", + "-DARDUINO_ESP8266_NODEMCU_ESP12E", + ] + + +def _make_framework(tmp_path: Path) -> dict[str, Path]: + framework = tmp_path / "framework" + core = framework / "cores" / "esp8266" + core.mkdir(parents=True) + for name in ( + "core_esp8266_main.cpp", + "Updater.cpp", + "core_esp8266_waveform_pwm.cpp", + "core_esp8266_waveform_phase.cpp", + "cont.S", + "abi.c", + ): + (core / name).write_text("") + (framework / "variants" / "nodemcu").mkdir(parents=True) + for sub in ("include", "ld", "lwip2/include", "lib"): + (framework / "tools" / "sdk" / sub).mkdir(parents=True) + (framework / "libraries").mkdir() + toolchain = tmp_path / "toolchain" + (toolchain / "bin").mkdir(parents=True) + return { + "framework_path": framework, + "toolchain_path": toolchain, + "ninja_path": Path("ninja"), + } + + +def _write_ninja(paths: dict[str, Path]) -> str: + from esphome.build_gen import arduino8266 + + src = CORE.relative_src_path() + (src / "esphome" / "components" / "esp8266").mkdir(parents=True, exist_ok=True) + (src / "main.cpp").write_text("") + (src / "esphome" / "vendor.c").write_text("") + + with ( + patch.object(arduino8266, "generate_ld_scripts"), + patch("esphome.arduino8266.framework.ccache_path", return_value=None), + ): + arduino8266.write_project(paths) + return (CORE.relative_pioenvs_path(CORE.name) / "build.ninja").read_text() + + +def test_write_project_link_line_and_exclusions(tmp_path: Path) -> None: + paths = _make_framework(tmp_path) + _set_flags( + "-DPIO_FRAMEWORK_ARDUINO_LWIP2_HIGHER_BANDWIDTH_LOW_FLASH", + "-DUSE_ESP8266_WAVEFORM_STUBS", + "-Wl,--wrap=millis", + "-Wl,--wrap=printf", + "-Wno-nonnull-compare", + ) + content = _write_ninja(paths) + + # Base link flags from the PlatformIO builder + for flag in ( + "-Wl,--no-check-sections", + "-Wl,-static", + "-Wl,--gc-sections", + "-Wl,-wrap,system_restart_local", + "-Wl,-wrap,spi_flash_read", + "-u app_entry", + "-u _printf_float", + "-u _DebugExceptionVector", + "-u _DoubleExceptionVector", + "-u _KernelExceptionVector", + "-u _NMIExceptionVector", + "-u _UserExceptionVector", + ): + assert flag in content + # ESPHome's link flags and the board linker script + assert "-Wl,--wrap=millis" in content + assert "-Wl,--wrap=printf" in content + assert "-T eagle.flash.4m.ld" in content + # scanf float disabled: the forced-link flag must not appear + assert "_scanf_float" not in content + # System libraries with the selected lwIP variant, in the builder's order + assert ( + "-lhal -lphy -lpp -lnet80211 -llwip2-1460 -lwpa -lcrypto -lmain -lwps " + "-lbearssl -lespnow -lsmartconfig -lairkiss -lwpa2 -lstdc++ -lm -lc -lgcc" + in content + ) + # Core exclusions: native OTA backend and waveform stubs + assert "Updater.cpp" not in content + assert "core_esp8266_waveform_pwm.cpp" not in content + assert "core_esp8266_waveform_phase.cpp" not in content + assert "core_esp8266_main.cpp.o" in content + # Assembly and C sources compile through their own rules + assert "cont.S.o: asm" in content + assert "abi.c.o: cc" in content + # throw_stubs is force-included for ESPHome sources only + src_lines = [line for line in content.splitlines() if "obj/src/" in line] + assert any("main.cpp.o: cxx" in line for line in src_lines) + assert content.count("throw_stubs.h") == len( + [line for line in content.splitlines() if line.startswith(" flags = ")] + ) + + +def test_write_project_scanf_float_and_waveform_kept(tmp_path: Path) -> None: + paths = _make_framework(tmp_path) + CORE.data[KEY_ESP8266][KEY_SCANF_FLOAT] = True + _set_flags("-DPIO_FRAMEWORK_ARDUINO_LWIP2_HIGHER_BANDWIDTH_LOW_FLASH") + content = _write_ninja(paths) + assert "-u _scanf_float" in content + # Waveform not stubbed out: both implementations stay in the archive + assert "core_esp8266_waveform_pwm.cpp.o" in content + assert "core_esp8266_waveform_phase.cpp.o" in content diff --git a/tests/unit_tests/components/esp8266/test_build_surgery.py b/tests/unit_tests/components/esp8266/test_build_surgery.py new file mode 100644 index 0000000000..a05c33f242 --- /dev/null +++ b/tests/unit_tests/components/esp8266/test_build_surgery.py @@ -0,0 +1,64 @@ +"""Tests for the linker-script surgery shared with the native toolchain.""" + +from __future__ import annotations + +import pytest + +from esphome.components.esp8266.build_surgery import ( + RATETABLE_RULE, + apply_testing_memory_patches, + relocate_ratetable, +) + +_COMMON_LD_SNIPPET = """\ + .dport0.data : ALIGN(4) + { + _dport0_data_start = ABSOLUTE(.); + } >dport0_0_seg :dport0_0_phdr + .data : ALIGN(4) + { + _data_start = ABSOLUTE(.); + *(.data) + } >dram0_0_seg :dram0_0_phdr +""" + +_FLASH_LD_SNIPPET = """\ +MEMORY +{ + dport0_0_seg : org = 0x3FF00000, len = 0x10 + dram0_0_seg : org = 0x3FFE8000, len = 0x14000 + iram1_0_seg : org = 0x40100000, len = 0x8000 + irom0_0_seg : org = 0x40201010, len = 0xfeff0 +} +""" + + +def test_relocate_ratetable_inserts_after_data_start() -> None: + patched = relocate_ratetable(_COMMON_LD_SNIPPET) + assert RATETABLE_RULE in patched + # Inserted after the .data section's anchor, not the .dport0.data one + assert patched.index("_data_start = ABSOLUTE(.);") < patched.index(RATETABLE_RULE) + assert patched.index(RATETABLE_RULE) < patched.index("*(.data)") + # Idempotent on an already-patched script + assert relocate_ratetable(patched) == patched + + +def test_relocate_ratetable_requires_anchor() -> None: + with pytest.raises(RuntimeError, match="_data_start"): + relocate_ratetable("SECTIONS { }") + + +def test_testing_memory_patches_enlarge_segments() -> None: + patched = apply_testing_memory_patches(_FLASH_LD_SNIPPET) + assert ( + "iram1_0_seg : org = 0x40100000, len = 0x200000" + in patched + ) + assert ( + "dram0_0_seg : org = 0x3FFE8000, len = 0x200000" + in patched + ) + assert ( + "irom0_0_seg : org = 0x40201010, len = 0x2000000" + in patched + ) diff --git a/tests/unit_tests/test_espidf_component.py b/tests/unit_tests/test_espidf_component.py index f9e048f6f4..a4adf734c0 100644 --- a/tests/unit_tests/test_espidf_component.py +++ b/tests/unit_tests/test_espidf_component.py @@ -26,11 +26,11 @@ from esphome.platformio.library import ( GitSource, URLSource, _node_key, - _normalize_dependencies, _parse_library_json, - _parse_library_properties, _resolve_registry_version, collect_filtered_files, + normalize_dependencies, + parse_library_properties, split_list_by_condition, ) @@ -499,7 +499,7 @@ def test_parse_library_json(tmp_path): assert result["name"] == "test" -def test_parse_library_properties(tmp_path): +def testparse_library_properties(tmp_path): f = tmp_path / "library.properties" f.write_text( """ @@ -510,7 +510,7 @@ empty= """ ) - result = _parse_library_properties(f) + result = parse_library_properties(f) assert result["name"] == "Test" assert result["version"] == "1.0" @@ -679,23 +679,23 @@ def test_node_key_registry_bare_name(): assert (key, kind, locator) == ("bar", "registry", (None, "bar")) -def test_normalize_dependencies_none(): - assert _normalize_dependencies(None) == [] +def testnormalize_dependencies_none(): + assert normalize_dependencies(None) == [] -def test_normalize_dependencies_list_form(): +def testnormalize_dependencies_list_form(): deps = [{"name": "foo", "version": "1.0"}] - assert _normalize_dependencies(deps) == [{"name": "foo", "version": "1.0"}] + assert normalize_dependencies(deps) == [{"name": "foo", "version": "1.0"}] -def test_normalize_dependencies_dict_form(): - out = _normalize_dependencies({"nanopb/Nanopb": "^0.4.91", "BareName": "1.2.3"}) +def testnormalize_dependencies_dict_form(): + out = normalize_dependencies({"nanopb/Nanopb": "^0.4.91", "BareName": "1.2.3"}) assert {"name": "Nanopb", "owner": "nanopb", "version": "^0.4.91"} in out assert {"name": "BareName", "owner": None, "version": "1.2.3"} in out -def test_normalize_dependencies_dict_form_nested_spec(): - out = _normalize_dependencies( +def testnormalize_dependencies_dict_form_nested_spec(): + out = normalize_dependencies( {"nanopb/Nanopb": {"version": "^0.4.91", "platforms": "espidf"}} ) assert out == [