github/esphome
2
0
Fork 0
mirror of https://github.com/esphome/esphome.git synced 2026-08-26 16:10:29 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'esp8266-native-toolchain-plumbing' into esp8266-native-build-infra

Browse Source
This commit is contained in:
J. Nick Koston
2026-08-25 12:39:02 -05:00
parent fce5a94234 b6e9845f27
commit ed369ae353
15 changed files with 469 additions and 389 deletions
Download Patch File Download Diff File Expand all files Collapse all files
esphome/components/dashboard_import/__init__.py
+12 -7
View File
@@ -4,7 +4,6 @@ import re
import secrets
from typing import Any
import requests
from ruamel.yaml import YAML
from esphome import git
@@ -13,7 +12,7 @@ from esphome.components.packages import validate_source_shorthand
import esphome.config_validation as cv
from esphome.const import CONF_ESPHOME, CONF_PROJECT, CONF_REF, CONF_WIFI
import esphome.final_validate as fv
from esphome.happy_eyeballs import ensure_happy_eyeballs
from esphome.net_retry import fetch_with_retry, http_request
from esphome.types import ConfigType
from esphome.yaml_util import dump
@@ -111,14 +110,20 @@ def import_config(
if git_file.query and "full_config" in git_file.query:
url = git_file.raw_url
try:
ensure_happy_eyeballs()
req = requests.get(url, timeout=30)
# Deferred so config-time imports of this component stay light;
# http_request does the lazy import for the request itself.
import requests
def _fetch() -> str:
req = http_request("GET", url, timeout=30)
req.raise_for_status()
return req.text
try:
contents = fetch_with_retry(url, _fetch, what="Import")
except requests.exceptions.RequestException as e:
raise ValueError(f"Error while fetching {url}: {e}") from e
contents = req.text
yaml = YAML()
loaded_yaml = yaml.load(contents)
if (
esphome/components/nrf52/framework.py
+30 -29
View File
@@ -5,16 +5,14 @@ from pathlib import Path
import platform
import shutil
import sys
import tempfile
from esphome.build_helpers.tools_cache import SDK_NRF_TOOLS_CACHE, tools_cache_path
import esphome.config_validation as cv
from esphome.const import KEY_CORE, KEY_FRAMEWORK_VERSION
from esphome.core import CORE, EsphomeError
from esphome.framework_helpers import (
archive_extract_all,
create_venv,
download_from_mirrors,
download_and_extract,
get_python_env_executable_path,
rmdir,
run_command_ok,
@@ -338,34 +336,37 @@ def check_and_install() -> None:
if not sentinel.exists():
rmdir(toolchains_dir, msg=f"Clean up {TOOLCHAIN_VERSION} toolchain environment")
sysname, machine, extension = _get_toolchain_platform_info()
with tempfile.NamedTemporaryFile() as tmp:
_LOGGER.info("Downloading Zephyr SDK %s minimal ...", TOOLCHAIN_VERSION)
download_from_mirrors(
SDK_NG_MINIMAL_MIRRORS,
{
"VERSION": TOOLCHAIN_VERSION,
"sysname": sysname,
"machine": machine,
"extension": extension,
},
tmp.file,
)
archive_extract_all(tmp.file, toolchains_dir, progress_header="Extracting")
with tempfile.NamedTemporaryFile() as tmp:
_LOGGER.info("Downloading %s toolchain ...", TOOLCHAIN_VERSION)
download_from_mirrors(
substitutions = {
"VERSION": TOOLCHAIN_VERSION,
"sysname": sysname,
"machine": machine,
"extension": extension,
}
# Downloaded next to the destination (not a temp file) so an
# interrupted download's .part file resumes on the next run.
for mirrors, extract_dir, what, slug in (
(SDK_NG_MINIMAL_MIRRORS, toolchains_dir, "Zephyr SDK minimal", "minimal"),
(
SDK_NG_TOOLCHAIN_MIRRORS,
{
"VERSION": TOOLCHAIN_VERSION,
"sysname": sysname,
"machine": machine,
"extension": extension,
},
tmp.file,
)
archive_extract_all(
tmp.file,
toolchains_dir / "arm-zephyr-eabi",
"toolchain",
"toolchain",
),
):
_LOGGER.info("Downloading %s %s ...", TOOLCHAIN_VERSION, what)
download_and_extract(
mirrors,
substitutions,
toolchains_dir.with_name(f"{toolchains_dir.name}.{slug}.archive"),
extract_dir,
progress_header="Extracting",
)
# Best-effort prune of resume leftovers, including a previous
# TOOLCHAIN_VERSION's orphans; the SDK archives are hundreds of MB.
# A locked file must not discard the just-completed install.
for leftover in toolchains_dir.parent.glob("*.archive.part*"):
try:
leftover.unlink()
except OSError as err:
_LOGGER.debug("Could not remove %s: %s", leftover, err)
sentinel.touch()
esphome/espidf/framework.py
+8 -15
View File
@@ -21,8 +21,8 @@ from esphome.build_helpers.tools_cache import IDF_TOOLS_CACHE, tools_cache_path
from esphome.core import Version
from esphome.framework_helpers import (
PathType,
archive_extract_all,
create_venv,
download_and_extract,
download_from_mirrors,
download_with_resume,
failure_reason,
@@ -901,20 +901,13 @@ def _check_esphome_idf_framework_install(
# a temp file) so an interrupted download resumes on the next
# run; the cache is pruned after a successful install anyway.
tarball_path = get_idf_tools_path() / "dist" / f"esp-idf-{version}.tar.xz"
download_from_mirrors(mirrors, substitutions, tarball_path)
_LOGGER.info("Extracting ESP-IDF %s framework ...", version)
try:
with tarball_path.open("rb") as tarball:
archive_extract_all(
tarball, framework_path, progress_header="Extracting"
)
finally:
# Success: drop the archive rather than caching ~70MB twice.
# Failure: a corrupt archive (e.g. torn by an unclean
# shutdown) must not be reused — without a checksum only a
# failed extraction can expose it, so force a re-download.
tarball_path.unlink(missing_ok=True)
download_and_extract(
mirrors,
substitutions,
tarball_path,
framework_path,
progress_header="Extracting",
)
extracted_marker.touch()
# Idempotent post-extract patch: written every invocation so a build
esphome/external_files.py
+5 -9
View File
@@ -5,6 +5,7 @@ from concurrent.futures import ThreadPoolExecutor
import contextlib
from dataclasses import dataclass, field
from datetime import UTC, datetime
from functools import partial
import hashlib
import logging
import os
@@ -14,9 +15,8 @@ import time
import esphome.config_validation as cv
from esphome.const import CONF_FILE, CONF_TYPE, CONF_URL, __version__
from esphome.core import CORE, EsphomeError, TimePeriodSeconds
from esphome.happy_eyeballs import ensure_happy_eyeballs
from esphome.helpers import write_file
from esphome.net_retry import fetch_with_retry
from esphome.net_retry import fetch_with_retry, http_request
from esphome.types import ConfigType
_LOGGER = logging.getLogger(__name__)
@@ -143,7 +143,6 @@ def has_remote_file_changed(
# Deferred so configs with no remote files skip the heavy import.
import requests
ensure_happy_eyeballs()
if local_file_path.exists():
_LOGGER.debug("has_remote_file_changed: File exists at %s", local_file_path)
try:
@@ -165,9 +164,7 @@ def has_remote_file_changed(
# the GET's own retry.
response = fetch_with_retry(
url,
lambda: requests.head(
url, headers=headers, timeout=timeout, allow_redirects=True
),
partial(http_request, "HEAD", url, headers=headers, timeout=timeout),
what="Revalidation",
)
@@ -282,7 +279,6 @@ def download_content(
) from failure.cause
# The file appeared since the failure; revalidate normally.
del run_data.failed_paths[path]
ensure_happy_eyeballs()
if CORE.skip_external_update and path.exists():
_LOGGER.debug("Skipping update for %s (refresh disabled)", url)
run_data.unchecked_paths.add(path)
@@ -304,7 +300,8 @@ def download_content(
_LOGGER.debug("Saving to %s", path)
def _fetch() -> tuple[requests.Response, bytes]:
req = requests.get(
req = http_request(
"GET",
url,
timeout=timeout,
headers={"User-agent": f"ESPHome/{__version__} (https://esphome.io)"},
@@ -371,7 +368,6 @@ def download_content_many(
unique = list(seen.values())
if not unique:
return
ensure_happy_eyeballs()
_LOGGER.info("Checking %d %s for updates", len(unique), description)
def _download_one(file: RemoteFile) -> None:
esphome/framework_helpers.py
+79 -141
View File
@@ -15,9 +15,12 @@ import threading
import time
from typing import IO, TYPE_CHECKING
from esphome.happy_eyeballs import ensure_happy_eyeballs
from esphome.helpers import ProgressBar, rmtree
from esphome.net_retry import NETWORK_MAX_ATTEMPTS, is_transient_download_error
from esphome.net_retry import (
NETWORK_MAX_ATTEMPTS,
http_request,
is_transient_download_error,
)
if TYPE_CHECKING:
import requests
@@ -624,12 +627,10 @@ def _open_ranged(
Raises on connect errors and HTTP error statuses; the response is closed
on failure.
"""
import requests
headers = {"Range": f"bytes={offset}-"} if offset else {}
if offset and validator:
headers["If-Range"] = validator
resp = requests.get(url, stream=True, timeout=timeout, headers=headers)
resp = http_request("GET", url, stream=True, timeout=timeout, headers=headers)
if offset and resp.status_code == 416:
resp.close()
return None, offset
@@ -965,8 +966,6 @@ def download_with_resume(
from esphome.core import EsphomeError
ensure_happy_eyeballs()
dest = Path(dest)
part = _part_path(dest)
meta = part.with_name(part.name + ".meta")
@@ -1102,20 +1101,9 @@ def failure_reason(e: BaseException) -> str:
return str(e).split(" for url: ", maxsplit=1)[0] or repr(e)
def _spent_attempts_error(e: Exception, attempts: int) -> Exception:
"""Wrap a failure whose mirror already consumed download attempts, so
the sweep classifies it as permanent."""
from esphome.core import EsphomeError
err = EsphomeError(f"failed after {attempts} attempts: {failure_reason(e)}")
err.__cause__ = e
return err
def _try_mirrors_once(
urls: list[str],
path_target: Path | None,
f: IO[bytes] | None,
path_target: Path,
timeout: int,
failures: list[tuple[str, Exception]],
progress: Callable[[int], None] | None = None,
@@ -1134,109 +1122,73 @@ def _try_mirrors_once(
for url in urls:
_LOGGER.debug("Trying to download from %s", url)
# Path targets delegate to download_with_resume so a partial
# download persists (and resumes) across esphome runs.
if path_target is not None:
try:
download_with_resume(
url,
path_target,
attempts=_MIRROR_ATTEMPTS,
timeout=timeout,
# Pre-body failures (connect/HTTP errors) fall to the
# next mirror immediately; only mid-stream drops
# retry-with-resume on the same URL.
retry_connect_errors=False,
progress=progress,
)
return url
except (requests.RequestException, OSError, EsphomeError) as e:
# Everything download_with_resume classifies as a download
# failure; programming errors propagate.
_LOGGER.debug("Failed to download %s: %s", url, str(e))
failures.append((url, e))
continue
# File-like targets download here; mid-stream failures retry the
# same mirror with resume (see download_with_resume) instead of
# starting over. There is no checksum to verify a resumed file
# against, so a stitch is only trusted when the server proves
# consistency: the If-Range validator guarantees 206 only for
# unchanged content, and the expected total length (when the first
# response carried one) guards against short or shifted bodies.
# Without a validator the retry restarts from zero.
offset = 0
expected_total = 0
validator = None
for attempt in range(_MIRROR_ATTEMPTS):
try:
resp, offset = _open_ranged(url, offset, timeout, validator)
except (requests.RequestException, OSError) as e:
# Connect/HTTP error, no bytes flowed — next mirror. Wrap
# when earlier attempts were already spent on this mirror.
_LOGGER.debug("Failed to download %s: %s", url, str(e))
failures.append(
(url, _spent_attempts_error(e, attempt + 1) if attempt else e)
)
break
try:
# A None response means HTTP 416: the file already holds
# every byte the server has (a drop after the last byte);
# only the length check below remains.
if resp is not None:
with resp:
if offset == 0:
validator = _response_validator(resp)
expected_total = _content_length(resp)
_stream_response_to_file(resp, f, offset, progress=progress)
if expected_total and f.tell() != expected_total:
raise EsphomeError(
f"size mismatch: expected {expected_total}, got {f.tell()}"
)
if not expected_total:
# Same trust decision as download_with_resume's
# unverifiable promotion; surface it at the same level.
_LOGGER.debug(
"Downloaded %s without any way to verify completeness",
url,
)
_LOGGER.debug("Downloaded successfully from: %s", url)
# Reset file pointer and return
f.seek(0)
return url
except (requests.RequestException, OSError, EsphomeError) as e:
# Mid-stream drop: keep the received bytes and retry this
# mirror from the current position — but only when the
# server gave a validator to resume against safely AND a
# total length to prove the stitched file complete (the
# length check above is the only verification here).
_LOGGER.debug("Failed to download %s: %s", url, str(e))
if validator and expected_total:
offset = f.tell()
else:
_LOGGER.debug(
"Restarting %s from zero: cannot prove a "
"resumed file complete (validator=%s, total=%s)",
url,
validator is not None,
expected_total,
)
offset = 0
if attempt == _MIRROR_ATTEMPTS - 1:
failures.append((url, _spent_attempts_error(e, _MIRROR_ATTEMPTS)))
# Delegate to download_with_resume so a partial download persists
# (and resumes) across esphome runs.
try:
download_with_resume(
url,
path_target,
attempts=_MIRROR_ATTEMPTS,
timeout=timeout,
# Pre-body failures (connect/HTTP errors) fall to the
# next mirror immediately; only mid-stream drops
# retry-with-resume on the same URL.
retry_connect_errors=False,
progress=progress,
)
return url
except (requests.RequestException, OSError, EsphomeError) as e:
# Everything download_with_resume classifies as a download
# failure; programming errors propagate.
_LOGGER.debug("Failed to download %s: %s", url, str(e))
failures.append((url, e))
return None
def download_and_extract(
mirrors: list[str],
substitutions: dict[str, str],
archive_path: PathType,
extract_dir: PathType,
timeout: int = 30,
progress_header: str | None = None,
progress: Callable[[int], None] | None = None,
) -> str:
"""Download an archive from ``mirrors`` to ``archive_path``, extract it
into ``extract_dir``, and delete the archive.
The archive should live next to its destination (not in a temp dir) so
an interrupted download's ``.part`` file resumes on the next run. The
archive is deleted whether extraction succeeds or fails: a
complete-but-corrupt file (e.g. torn by an unclean shutdown) must not
poison the next run, and without a checksum only a failed extraction
can expose it.
Returns the source URL the download came from.
"""
archive_path = Path(archive_path)
url = download_from_mirrors(
mirrors, substitutions, archive_path, timeout=timeout, progress=progress
)
try:
archive_extract_all(archive_path, extract_dir, progress_header=progress_header)
finally:
# Best-effort: an AV handle on the just-written archive (Windows)
# must not replace the real extraction error or fail a successful
# extraction. A surviving archive is harmless; download_with_resume
# re-verifies or re-downloads it next run.
try:
archive_path.unlink(missing_ok=True)
except OSError as err:
_LOGGER.debug("Could not remove archive %s: %s", archive_path, err)
return url
def download_from_mirrors(
mirrors: list[str],
substitutions: dict[str, str],
target: io.RawIOBase | IO[bytes] | PathType,
target: PathType,
timeout: int = 30,
progress: Callable[[int], None] | None = None,
) -> str:
@@ -1246,7 +1198,7 @@ def download_from_mirrors(
Args:
mirrors: list of mirror URLs
substitutions: Dictionary of substitutions to apply to URLs
target: Target file path or file-like object
target: Target file path
timeout: Download timeout in seconds
progress: Passed through to the download (see ``download_with_resume``);
replaces the built-in per-file bar
@@ -1258,9 +1210,8 @@ def download_from_mirrors(
``substitutions`` are skipped, so callers can offer templates that only
apply to some downloads.
A path target downloads through ``download_with_resume``, so an
interrupted download resumes on the next esphome run; a file-like target
only resumes mid-stream drops within this call.
The target downloads through ``download_with_resume``, so an
interrupted download resumes on the next esphome run.
When every mirror fails and at least one failure is transient (dropped
connection, timeout, HTTP 429/5xx), the whole list is retried with a
@@ -1274,21 +1225,11 @@ def download_from_mirrors(
"""
from esphome.core import EsphomeError
ensure_happy_eyeballs()
if not isinstance(target, (str, os.PathLike)):
raise TypeError(f"target must be a str or Path: {type(target)}")
path_target = Path(target)
# 1. Classify the target: filesystem path or open file object
path_target: Path | None = None
f: IO[bytes] | None = None
if isinstance(target, (str, os.PathLike)):
path_target = Path(target)
elif isinstance(target, (io.RawIOBase, io.IOBase)):
f = target
else:
raise TypeError(
f"target must be str, Path, or file-like object: {type(target)}"
)
# 2. Resolve the mirror templates (invariant across retry sweeps)
# 1. Resolve the mirror templates (invariant across retry sweeps)
urls: list[str] = []
skipped: list[tuple[str, str]] = []
for mirror in mirrors:
@@ -1307,7 +1248,7 @@ def download_from_mirrors(
_LOGGER.warning("Skipping malformed mirror URL template %s: %r", mirror, e)
skipped.append((mirror, f"skipped ({e!r})"))
# 3. Sweep the mirror list, retrying transient failures with backoff:
# 2. Sweep the mirror list, retrying transient failures with backoff:
# a single pass keeps mirror failover fast, re-sweeping keeps one
# network blip from failing the build when only one mirror applies.
failures: list[tuple[str, Exception]] = []
@@ -1315,7 +1256,7 @@ def download_from_mirrors(
sweep_failures: list[tuple[str, Exception]] = []
if (
url := _try_mirrors_once(
urls, path_target, f, timeout, sweep_failures, progress
urls, path_target, timeout, sweep_failures, progress
)
) is not None:
return url
@@ -1342,14 +1283,11 @@ def download_from_mirrors(
# steady during the backoff instead of rewinding to zero
done = 0
if progress is not None:
if f is not None:
done = f.tell()
else:
part = _part_path(path_target)
done = part.stat().st_size if part.is_file() else 0
part = _part_path(path_target)
done = part.stat().st_size if part.is_file() else 0
_cancellable_sleep(delay, progress, done)
# 4. Report every attempted URL if all mirrors failed. failures spans
# 3. Report every attempted URL if all mirrors failed. failures spans
# all sweeps (deduplicated by URL and reason), so neither an early
# mirror's failure nor an earlier sweep's failure mode is hidden.
if failures:
esphome/happy_eyeballs.py
+13 -5
View File
@@ -12,6 +12,7 @@ from __future__ import annotations
import logging
import socket
import threading
from typing import TYPE_CHECKING, Any
if TYPE_CHECKING:
@@ -27,20 +28,27 @@ HAPPY_EYEBALLS_DELAY = 0.25
_THREAD_WAIT_BUFFER = 5.0
# Serialises the check-then-patch so concurrent first calls (download worker
# threads fanning out) build the replacement exactly once.
_PATCH_LOCK = threading.Lock()
def ensure_happy_eyeballs() -> None:
"""Make urllib3 (and therefore requests) connect with Happy Eyeballs.
Idempotent; call before performing requests-based downloads.
Idempotent and thread-safe; call before performing requests-based
downloads.
"""
stock: Callable[..., socket.socket] | None = None
try:
import urllib3.util.connection
stock = urllib3.util.connection.create_connection
if getattr(stock, "_esphome_patched", False):
return
with _PATCH_LOCK:
stock = urllib3.util.connection.create_connection
if getattr(stock, "_esphome_patched", False):
return
urllib3.util.connection.create_connection = _make_create_connection()
urllib3.util.connection.create_connection = _make_create_connection()
except (ImportError, AttributeError) as err: # urllib3 internals moved
# WARNING: degraded mode brings back the stalls this module prevents.
_LOGGER.warning(
esphome/net_retry.py
+38 -1
View File
@@ -1,4 +1,4 @@
"""Retry policy for HTTP downloads.
"""Retry policy and raw HTTP entry point for downloads.
Kept import-light on purpose: this module is imported at config time, so it
must not pull in requests (a heavy import, ~85ms) at module scope.
@@ -9,6 +9,12 @@ from __future__ import annotations
from collections.abc import Callable
import logging
import time
from typing import TYPE_CHECKING, Literal
from esphome.happy_eyeballs import ensure_happy_eyeballs
if TYPE_CHECKING:
import requests
_LOGGER = logging.getLogger(__name__)
@@ -112,3 +118,34 @@ def fetch_with_retry[T](url: str, fetch: Callable[[], T], what: str = "Download"
)
time.sleep(delay)
return fetch()
def http_request(
method: Literal["GET", "HEAD"],
url: str,
*,
timeout: float | tuple[float, float],
stream: bool = False,
headers: dict[str, str] | None = None,
allow_redirects: bool = True,
) -> requests.Response:
"""Perform one HTTP request with the Happy Eyeballs patch in place.
Every ESPHome file download funnels through here so the urllib3 patch
and the lazy requests import live in exactly one place. Status handling,
retries and streaming stay with the caller. The web server OTA and log
clients bypass this on purpose: they iterate already-resolved device
addresses themselves, so the patch buys them nothing.
"""
import requests
ensure_happy_eyeballs()
# Dispatched through requests.get/head/... (not requests.request) so
# tests patching those entry points keep working.
return getattr(requests, method.lower())(
url,
timeout=timeout,
stream=stream,
headers=headers or {},
allow_redirects=allow_redirects,
)
esphome/platformio/library.py
+15 -14
View File
@@ -24,7 +24,6 @@ import logging
import os
from pathlib import Path, PurePosixPath
import re
import tempfile
from typing import Any
from urllib.parse import urlsplit, urlunsplit
from urllib.request import url2pathname
@@ -32,8 +31,7 @@ from urllib.request import url2pathname
from esphome import git
from esphome.core import CORE, EsphomeError, Library
from esphome.framework_helpers import (
archive_extract_all,
download_from_mirrors,
download_and_extract,
failure_reason,
rmdir,
run_batch_downloads,
@@ -147,18 +145,21 @@ class URLSource(Source):
if not extracted_marker.is_file() or force:
rmdir(path, msg=f"Clean up library directory {path}")
# Download in temporary file
with tempfile.NamedTemporaryFile() as tmp:
if progress is None:
# A batch caller draws one combined bar and logs the list
_LOGGER.info("Downloading %s ...", self.url)
_LOGGER.debug("Location: %s", path)
if progress is None:
# A batch caller draws one combined bar and logs the list
_LOGGER.info("Downloading %s ...", self.url)
_LOGGER.debug("Location: %s", path)
download_from_mirrors([self.url], {}, tmp.file, progress=progress)
_LOGGER.debug("Extracting archive to %s ...", path)
archive_extract_all(tmp.file, path)
extracted_marker.touch()
# The sibling archive path lets an interrupted download's .part
# file survive and resume on the next esphome run.
download_and_extract(
[self.url],
{},
path.with_name(f"{path.name}.archive"),
path,
progress=progress,
)
extracted_marker.touch()
return path
def __str__(self):
tests/unit_tests/test_dashboard_import.py
+55
View File
@@ -10,8 +10,10 @@ during the adoption flow and depend on the output's ``esphome.name``
from __future__ import annotations
from pathlib import Path
from unittest.mock import MagicMock, patch
import pytest
import requests as req
import yaml as pyyaml
from esphome.components.dashboard_import import import_config
@@ -201,3 +203,56 @@ def test_import_refuses_to_overwrite_existing_yaml(tmp_path: Path) -> None:
)
# Original content survives unchanged.
assert yaml_path.read_text() == "# user's hand-edited config\n"
def _full_config_kwargs(yaml_path: Path) -> dict:
return {
"path": str(yaml_path),
"name": "kitchen",
"friendly_name": None,
"project_name": "acme.kitchen-light",
"import_url": "github://acme/firmware/kitchen.yaml@main?full_config",
}
def test_full_config_import_fetches_and_writes_contents(tmp_path: Path) -> None:
yaml_path = tmp_path / "kitchen.yaml"
resp = MagicMock(text="esphome:\n name: orig\n")
with patch(
"esphome.components.dashboard_import.http_request", return_value=resp
) as mock_req:
import_config(**_full_config_kwargs(yaml_path))
assert yaml_path.read_text() == "esphome:\n name: orig\n"
assert mock_req.call_args[0][0] == "GET"
def test_full_config_import_retries_transient_errors(tmp_path: Path) -> None:
"""The fetch goes through the shared retry policy: a transient network
error is retried instead of failing the adoption immediately."""
yaml_path = tmp_path / "kitchen.yaml"
resp = MagicMock(text="esphome:\n name: orig\n")
with (
patch(
"esphome.components.dashboard_import.http_request",
side_effect=[req.ConnectionError("reset"), resp],
),
patch("esphome.net_retry.time.sleep") as mock_sleep,
):
import_config(**_full_config_kwargs(yaml_path))
assert yaml_path.exists()
mock_sleep.assert_called_once_with(2)
def test_full_config_import_wraps_permanent_errors_in_value_error(
tmp_path: Path,
) -> None:
"""device-builder depends on the ValueError contract for fetch failures."""
resp = MagicMock()
resp.raise_for_status.side_effect = req.HTTPError(
"404", response=MagicMock(status_code=404)
)
with (
patch("esphome.components.dashboard_import.http_request", return_value=resp),
pytest.raises(ValueError, match="Error while fetching"),
):
import_config(**_full_config_kwargs(tmp_path / "kitchen.yaml"))
tests/unit_tests/test_espidf_framework.py
+10 -9
View File
@@ -371,10 +371,9 @@ def _fake_download_from_mirrors(
) -> str:
"""Stand-in for download_from_mirrors that creates path targets, since
the framework code opens the downloaded tarball afterwards."""
if isinstance(target, (str, os.PathLike)):
path = Path(target)
path.parent.mkdir(parents=True, exist_ok=True)
path.touch()
path = Path(target)
path.parent.mkdir(parents=True, exist_ok=True)
path.touch()
return "https://example.com/idf.tar.xz"
@@ -384,13 +383,15 @@ def espidf_mocks(setup_core: Path):
# archive_extract_all is mocked, so pre-create the framework dir that the
# extracted-marker touch writes into.
_get_framework_path(_IDF_VERSION).mkdir(parents=True, exist_ok=True)
# One mock covers the tarball (via framework_helpers.download_and_extract)
# and the constraints file (espidf-bound download_from_mirrors), so call
# counts and ordering assertions span the two.
download = MagicMock(side_effect=_fake_download_from_mirrors)
with (
patch("esphome.espidf.framework.rmdir") as rmdir_mock,
patch(
"esphome.espidf.framework.download_from_mirrors",
side_effect=_fake_download_from_mirrors,
) as download,
patch("esphome.espidf.framework.archive_extract_all") as extract,
patch("esphome.framework_helpers.download_from_mirrors", download),
patch("esphome.espidf.framework.download_from_mirrors", download),
patch("esphome.framework_helpers.archive_extract_all") as extract,
patch("esphome.espidf.framework.create_venv") as venv,
patch("esphome.espidf.framework.run_command_ok", return_value=True) as run_ok,
patch(
tests/unit_tests/test_framework_helpers.py
+98 -148
View File
@@ -2,6 +2,7 @@
# pylint: disable=protected-access
import gzip
import hashlib
import importlib.util
import io
@@ -31,6 +32,7 @@ from esphome.framework_helpers import (
_zip_extract_all,
archive_extract_all,
create_venv,
download_and_extract,
download_from_mirrors,
download_with_resume,
get_project_compile_flags,
@@ -1339,20 +1341,20 @@ class TestDownloadFromMirrors:
assert url == "https://example.com/f"
assert target.read_bytes() == b"filedata"
def test_file_object_target_reports_progress(self) -> None:
"""The library prefetch's production path: a file-object target
streams through the mirror fallback and ticks the tracker."""
buf = io.BytesIO()
def test_progress_callback_reports_bytes(self, tmp_path: Path) -> None:
"""The library prefetch's production path: the mirror download ticks
the caller's tracker instead of drawing its own bar."""
target = tmp_path / "f.bin"
ticks: list[int] = []
with patch(
"requests.get",
return_value=_mock_response(b"filedata"),
):
url = download_from_mirrors(
["https://example.com/f"], {}, buf, progress=ticks.append
["https://example.com/f"], {}, target, progress=ticks.append
)
assert url == "https://example.com/f"
assert buf.getvalue() == b"filedata"
assert target.read_bytes() == b"filedata"
assert ticks and ticks[-1] == len(b"filedata")
def test_substitutions_applied_to_url(self, tmp_path: Path) -> None:
@@ -1460,8 +1462,8 @@ class TestDownloadFromMirrors:
ei.value
)
def test_falls_back_to_second_mirror(self) -> None:
buf = io.BytesIO()
def test_falls_back_to_second_mirror(self, tmp_path: Path) -> None:
target = tmp_path / "f.bin"
with patch(
"requests.get",
side_effect=[_mock_response(b"", ok=False), _mock_response(b"second")],
@@ -1469,18 +1471,18 @@ class TestDownloadFromMirrors:
url = download_from_mirrors(
["https://mirror1.com/f", "https://mirror2.com/f"],
{},
buf,
target,
)
assert url == "https://mirror2.com/f"
assert buf.getvalue() == b"second"
assert target.read_bytes() == b"second"
def test_mid_stream_drop_resumes_same_mirror(self) -> None:
def test_mid_stream_drop_resumes_same_mirror(self, tmp_path: Path) -> None:
"""A mid-stream failure retries the same mirror with Range and
If-Range headers, keeping the bytes already received, before falling
to the next."""
first = _interrupted_response(b"1234", etag='"v1"')
first.headers = {**first.headers, "content-length": "8"}
buf = io.BytesIO()
target = tmp_path / "f.bin"
with patch(
"requests.get",
side_effect=[first, _resumed_response(b"5678")],
@@ -1488,10 +1490,10 @@ class TestDownloadFromMirrors:
url = download_from_mirrors(
["https://mirror1.com/f", "https://mirror2.com/f"],
{},
buf,
target,
)
assert url == "https://mirror1.com/f"
assert buf.getvalue() == b"12345678"
assert target.read_bytes() == b"12345678"
assert mock_get.call_count == 2
assert mock_get.call_args_list[1][0][0] == "https://mirror1.com/f"
# the resume is conditional on the content being unchanged
@@ -1500,48 +1502,6 @@ class TestDownloadFromMirrors:
"If-Range": '"v1"',
}
def test_mid_stream_drop_without_validator_restarts(self) -> None:
"""A server offering no ETag/Last-Modified cannot be resumed safely;
the retry restarts from zero instead of stitching unverified bytes."""
buf = io.BytesIO()
with patch(
"requests.get",
side_effect=[_interrupted_response(b"1234"), _mock_response(b"full")],
) as mock_get:
download_from_mirrors(["https://mirror1.com/f"], {}, buf)
assert buf.getvalue() == b"full"
assert "Range" not in mock_get.call_args_list[1][1]["headers"]
def test_drop_after_last_byte_recovers_via_416(self) -> None:
"""A connection drop after the final body byte leaves a complete file;
the retry's 416 answer plus the length check turn it into success
instead of a wasted refetch."""
first = _interrupted_response(b"1234", etag='"v1"')
first.headers = {**first.headers, "content-length": "4"}
r416 = _mock_response(b"", ok=False)
r416.status_code = 416
buf = io.BytesIO()
with patch("requests.get", side_effect=[first, r416]) as mock_get:
url = download_from_mirrors(["https://mirror1.com/f"], {}, buf)
assert url == "https://mirror1.com/f"
assert buf.getvalue() == b"1234"
assert mock_get.call_count == 2
def test_mirror_drop_without_length_restarts(self) -> None:
"""With no content-length there is no way to prove a stitched file
complete, so the retry restarts even though a validator exists."""
buf = io.BytesIO()
with patch(
"requests.get",
side_effect=[
_interrupted_response(b"1234", etag='"v1"'),
_mock_response(b"full"),
],
) as mock_get:
download_from_mirrors(["https://mirror1.com/f"], {}, buf)
assert buf.getvalue() == b"full"
assert "Range" not in mock_get.call_args_list[1][1]["headers"]
def test_path_target_resumes_across_runs(self, tmp_path: Path) -> None:
"""A path target routes through download_with_resume: a part file and
metadata from a previous run resume instead of restarting."""
@@ -1573,32 +1533,14 @@ class TestDownloadFromMirrors:
assert url == "https://mirror2.com/f"
assert dest.read_bytes() == b"data"
def test_resumed_short_body_fails_length_check(self) -> None:
"""A stitched file whose final length disagrees with the advertised
total is rejected instead of reported as success."""
first = _interrupted_response(b"1234", etag='"v1"')
first.headers = {**first.headers, "content-length": "8"}
# the resume ends early (5 of 8 bytes); the poisoned part is then
# discarded and the fresh retry also delivers a short body
short_resume = _resumed_response(b"5")
short_fresh = _mock_response(b"56")
short_fresh.headers = {**short_fresh.headers, "content-length": "8"}
buf = io.BytesIO()
with (
patch("requests.get", side_effect=[first, short_resume, short_fresh]),
pytest.raises(EsphomeError, match="all mirrors"),
):
download_from_mirrors(["https://mirror1.com/f"], {}, buf)
def test_failed_mirror_leftovers_not_kept_for_next_mirror(self) -> None:
"""Bytes from a mirror that failed all attempts must not leak into the
next mirror's download (no bogus Range request, fresh content)."""
exhausted = [_interrupted_response(b"AAAA", etag='"a1"')]
for _ in range(2):
r = _interrupted_response(b"BB")
r.status_code = 206
exhausted.append(r)
buf = io.BytesIO()
def test_failed_mirror_leftovers_not_resumed_on_next_mirror(
self, tmp_path: Path
) -> None:
"""A part file left by a mirror that failed all attempts must not be
stitched onto the next mirror's download (its meta names the other
URL, so the retry restarts from zero without a Range request)."""
exhausted = [_interrupted_response(b"AAAA") for _ in range(3)]
target = tmp_path / "f.bin"
with patch(
"requests.get",
side_effect=exhausted + [_mock_response(b"clean")],
@@ -1606,15 +1548,17 @@ class TestDownloadFromMirrors:
url = download_from_mirrors(
["https://mirror1.com/f", "https://mirror2.com/f"],
{},
buf,
target,
)
assert url == "https://mirror2.com/f"
assert buf.getvalue() == b"clean"
assert target.read_bytes() == b"clean"
# the second mirror starts fresh, without a Range header
assert mock_get.call_args_list[3][0][0] == "https://mirror2.com/f"
assert "Range" not in mock_get.call_args_list[3][1]["headers"]
def test_all_mirrors_fail_raises_error_listing_every_attempt(self) -> None:
def test_all_mirrors_fail_raises_error_listing_every_attempt(
self, tmp_path: Path
) -> None:
with (
patch(
"requests.get",
@@ -1625,7 +1569,7 @@ class TestDownloadFromMirrors:
download_from_mirrors(
["https://mirror1.com/f", "https://mirror2.com/f"],
{},
io.BytesIO(),
tmp_path / "out.bin",
)
# Every attempted URL appears in the message, and the first mirror's
# exception (the primary URL, usually the one that matters) is chained.
@@ -1641,16 +1585,6 @@ class TestDownloadFromMirrors:
with pytest.raises(TypeError, match="target must be"):
download_from_mirrors(["https://example.com/f"], {}, 42) # type: ignore[arg-type]
def test_file_like_target_written(self) -> None:
buf = io.BytesIO()
with patch(
"requests.get",
return_value=_mock_response(b"bytes"),
):
download_from_mirrors(["https://example.com/f"], {}, buf)
buf.seek(0)
assert buf.read() == b"bytes"
def test_progress_bar_shown_when_content_length_known(self, tmp_path: Path) -> None:
r = _mock_response(b"1234567890")
r.headers = {"content-length": "10"}
@@ -1676,13 +1610,10 @@ class TestDownloadFromMirrors:
assert target.exists()
assert target.read_bytes() == b""
@pytest.mark.parametrize("target_kind", ["path", "file-like"])
def test_transient_failure_retries_mirror_sweep(
self, tmp_path: Path, target_kind: str
) -> None:
def test_transient_failure_retries_mirror_sweep(self, tmp_path: Path) -> None:
"""A transient connect error on the only applicable mirror retries the
whole mirror list with backoff instead of failing the build."""
target = tmp_path / "idf.tar.xz" if target_kind == "path" else io.BytesIO()
target = tmp_path / "idf.tar.xz"
with (
patch(
"requests.get",
@@ -1695,33 +1626,10 @@ class TestDownloadFromMirrors:
):
url = download_from_mirrors(["https://mirror1.com/f"], {}, target)
assert url == "https://mirror1.com/f"
data = target.read_bytes() if target_kind == "path" else target.getvalue()
assert data == b"data"
assert target.read_bytes() == b"data"
assert mock_get.call_count == 2
mock_sleep.assert_called_once_with(2)
def test_backoff_tick_reports_filelike_bytes(self) -> None:
"""For a file-like target the backoff tick carries f.tell(), so the
combined bar holds steady through the sweep retry."""
target = io.BytesIO()
ticks: list[int] = []
with (
patch(
"requests.get",
side_effect=[
req.ConnectionError("down"),
_mock_response(b"data"),
],
),
patch("esphome.framework_helpers._cancellable_sleep") as mock_sleep,
):
download_from_mirrors(
["https://mirror1.com/f"], {}, target, progress=ticks.append
)
# No bytes had streamed at backoff time, so the tick carries 0
assert mock_sleep.call_args == call(2, ticks.append, 0)
assert target.getvalue() == b"data"
def test_backoff_tick_reports_partial_bytes(self, tmp_path: Path) -> None:
"""The backoff tick carries the bytes already in the part file, so a
combined bar holds steady instead of rewinding to zero."""
@@ -1831,41 +1739,83 @@ class TestDownloadFromMirrors:
assert isinstance(ei.value.__cause__, req.ConnectionError)
mock_sleep.assert_called_once_with(2)
def test_exhausted_mid_stream_attempts_not_swept(self) -> None:
"""A file-like mirror that spent all its mid-stream attempts is not
retried again at the sweep level (unlike a path target, it has no
part file to resume from on a later sweep)."""
buf = io.BytesIO()
def test_exhausted_mid_stream_attempts_not_swept(self, tmp_path: Path) -> None:
"""A mirror that spent all its mid-stream attempts fails permanently
instead of re-arming the sweep, and its part file survives so the
next esphome run resumes it."""
with (
patch(
"requests.get",
side_effect=[_interrupted_response(b"1234") for _ in range(3)],
) as mock_get,
patch("esphome.framework_helpers.time.sleep") as mock_sleep,
pytest.raises(EsphomeError, match="failed after 3 attempts"),
pytest.raises(EsphomeError, match="after 3 attempts"),
):
download_from_mirrors(["https://mirror1.com/f"], {}, buf)
download_from_mirrors(["https://mirror1.com/f"], {}, tmp_path / "out.bin")
assert mock_get.call_count == 3
mock_sleep.assert_not_called()
assert (tmp_path / "out.bin.part").exists()
class TestDownloadAndExtract:
def test_downloads_extracts_and_deletes_archive(self, tmp_path: Path) -> None:
content = gzip.compress(
_make_tar([_reg("file.txt")], {"file.txt": b"data"}).getvalue()
)
dest = tmp_path / "out"
with patch("requests.get", return_value=_mock_response(content)):
url = download_and_extract(
["https://example.com/lib.tar.gz"],
{},
tmp_path / "lib.archive",
dest,
)
assert url == "https://example.com/lib.tar.gz"
assert (dest / "file.txt").read_bytes() == b"data"
# the archive is consumed; only the extraction remains
assert not (tmp_path / "lib.archive").exists()
def test_locked_archive_does_not_mask_result(self, tmp_path: Path) -> None:
"""A cleanup unlink blocked by e.g. an AV handle (Windows) must not
replace the extraction result; the archive simply survives."""
content = gzip.compress(
_make_tar([_reg("file.txt")], {"file.txt": b"data"}).getvalue()
)
real_unlink = Path.unlink
def locked_unlink(self: Path, missing_ok: bool = False) -> None:
if self.name.endswith(".archive"):
raise PermissionError("held by antivirus")
real_unlink(self, missing_ok=missing_ok)
def test_mid_stream_drop_then_connect_error_not_swept(self) -> None:
"""A connect error on a later attempt (after a mid-stream drop spent
one) also counts as spent budget and does not re-arm the sweep."""
buf = io.BytesIO()
with (
patch(
"requests.get",
side_effect=[
_interrupted_response(b"1234"),
req.ConnectionError("down"),
],
) as mock_get,
patch("esphome.framework_helpers.time.sleep") as mock_sleep,
pytest.raises(EsphomeError, match="failed after 2 attempts"),
patch("requests.get", return_value=_mock_response(content)),
patch("pathlib.Path.unlink", locked_unlink),
):
download_from_mirrors(["https://mirror1.com/f"], {}, buf)
assert mock_get.call_count == 2
mock_sleep.assert_not_called()
url = download_and_extract(
["https://example.com/lib.tar.gz"],
{},
tmp_path / "lib.archive",
tmp_path / "out",
)
assert url == "https://example.com/lib.tar.gz"
assert (tmp_path / "out" / "file.txt").read_bytes() == b"data"
assert (tmp_path / "lib.archive").exists() # left behind, harmless
def test_corrupt_archive_deleted_on_extract_failure(self, tmp_path: Path) -> None:
"""A complete-but-corrupt archive must not survive to poison the next
run; without a checksum only a failed extraction can expose it."""
with (
patch("requests.get", return_value=_mock_response(b"not an archive")),
pytest.raises(ValueError, match="Unsupported archive format"),
):
download_and_extract(
["https://example.com/lib.tar.gz"],
{},
tmp_path / "lib.archive",
tmp_path / "out",
)
assert not (tmp_path / "lib.archive").exists()
def test_importing_framework_helpers_does_not_import_requests() -> None:
tests/unit_tests/test_happy_eyeballs.py
+37
View File
@@ -4,7 +4,9 @@ from __future__ import annotations
import asyncio
from collections.abc import Generator
from concurrent.futures import ThreadPoolExecutor
import socket
import threading
from typing import Any
from unittest.mock import Mock, patch
@@ -61,6 +63,41 @@ def test_ensure_happy_eyeballs_patches_and_is_idempotent(
assert urllib3.util.connection.create_connection is patched
def test_ensure_happy_eyeballs_concurrent_first_calls_patch_once(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""Worker threads fanning out (download_content_many, run_batch_downloads)
may race the first call; the replacement is built exactly once."""
import urllib3.util.connection
from esphome import happy_eyeballs
def stock(*args: Any, **kwargs: Any) -> None:
pass
monkeypatch.setattr(urllib3.util.connection, "create_connection", stock)
barrier = threading.Barrier(8)
builds: list[int] = []
real_make = happy_eyeballs._make_create_connection
def counting_make() -> Any:
builds.append(1)
return real_make()
monkeypatch.setattr(happy_eyeballs, "_make_create_connection", counting_make)
def racer() -> None:
barrier.wait(timeout=10)
ensure_happy_eyeballs()
with ThreadPoolExecutor(max_workers=8) as ex:
list(ex.map(lambda _: racer(), range(8)))
assert builds == [1]
assert urllib3.util.connection.create_connection._esphome_patched
def test_connects_and_restores_socket_state(
create_connection: Any, listener: tuple[str, int], mock_gai: Any
) -> None:
tests/unit_tests/test_net_retry.py
+42 -1
View File
@@ -7,7 +7,11 @@ import pytest
import requests as req
from esphome.core import EsphomeError
from esphome.net_retry import fetch_with_retry, is_transient_download_error
from esphome.net_retry import (
fetch_with_retry,
http_request,
is_transient_download_error,
)
def _http_error(status: int) -> req.HTTPError:
@@ -141,3 +145,40 @@ class TestFetchWithRetry:
assert mock_sleep.call_args_list == [call(2), call(4)]
assert "(attempt 2/3)" in caplog.text
assert "(attempt 3/3)" in caplog.text
class TestHttpRequest:
def test_applies_happy_eyeballs_and_forwards_arguments(self) -> None:
with (
patch("esphome.net_retry.ensure_happy_eyeballs") as mock_he,
patch("requests.get", return_value=MagicMock()) as mock_get,
):
resp = http_request(
"GET",
"https://example.com/f",
timeout=30,
stream=True,
headers={"Range": "bytes=4-"},
)
mock_he.assert_called_once_with()
assert resp is mock_get.return_value
assert mock_get.call_args == call(
"https://example.com/f",
timeout=30,
stream=True,
headers={"Range": "bytes=4-"},
allow_redirects=True,
)
def test_dispatches_head_through_requests_head(self) -> None:
"""Dispatch goes through requests.get/head so tests patching those
entry points keep working."""
with patch("requests.head", return_value=MagicMock()) as mock_head:
http_request("HEAD", "https://example.com/f", timeout=(5, 30))
assert mock_head.call_args[1]["timeout"] == (5, 30)
def test_no_status_handling(self) -> None:
"""Error statuses are the caller's problem; nothing raises here."""
resp = MagicMock(status_code=404)
with patch("requests.get", return_value=resp):
assert http_request("GET", "https://example.com/f", timeout=1) is resp
tests/unit_tests/test_nrf52_framework.py
+4 -2
View File
@@ -107,11 +107,13 @@ def mock_nrf52_ops():
patch(
"esphome.components.nrf52.framework.run_command_ok", return_value=True
) as mock_run_cmd,
# download_and_extract resolves its internals in framework_helpers,
# so the download/extract seams are patched there.
patch(
"esphome.components.nrf52.framework.download_from_mirrors",
"esphome.framework_helpers.download_from_mirrors",
return_value="https://example.com/tc.tar.xz",
) as mock_download,
patch("esphome.components.nrf52.framework.archive_extract_all") as mock_extract,
patch("esphome.framework_helpers.archive_extract_all") as mock_extract,
):
yield SimpleNamespace(
rmdir=mock_rmdir,
tests/unit_tests/test_platformio_library.py
+23 -8
View File
@@ -158,16 +158,12 @@ def test_urlsource_download_extracts_then_reuses_marker(
):
monkeypatch.setattr(lib, "rmdir", lambda path, msg="": None)
dl_calls: list[list[str]] = []
monkeypatch.setattr(
lib,
"download_from_mirrors",
lambda urls, headers, f, progress=None: dl_calls.append(urls),
)
def fake_extract(fileobj, path):
Path(path).mkdir(parents=True, exist_ok=True)
def fake_download_and_extract(urls, subs, archive_path, extract_dir, **kwargs):
dl_calls.append(urls)
Path(extract_dir).mkdir(parents=True, exist_ok=True)
monkeypatch.setattr(lib, "archive_extract_all", fake_extract)
monkeypatch.setattr(lib, "download_and_extract", fake_download_and_extract)
src = URLSource("http://example.test/lib.tar.gz")
out = src.download("mylib")
@@ -187,6 +183,25 @@ def test_urlsource_download_extracts_then_reuses_marker(
assert "Downloading" not in caplog.text
def test_urlsource_downloads_to_sibling_archive_path(setup_core, monkeypatch):
"""The archive downloads to a deterministic path next to the cache dir
(not a random temp file), so an interrupted download's .part file
resumes on the next run."""
monkeypatch.setattr(lib, "rmdir", lambda path, msg="": None)
targets: list[Path] = []
def fake_download_and_extract(urls, subs, archive_path, extract_dir, **kwargs):
targets.append(Path(archive_path))
Path(extract_dir).mkdir(parents=True, exist_ok=True)
monkeypatch.setattr(lib, "download_and_extract", fake_download_and_extract)
src = URLSource("http://example.test/lib.tar.gz")
out = src.download("mylib")
assert targets == [out.with_name(f"{out.name}.archive")]
def test_resolve_registry_version_raises_without_pkg_file(monkeypatch):
registry = lib._make_registry_client()
monkeypatch.setattr(