mirror of
https://github.com/esphome/esphome.git
synced 2026-10-01 08:50:22 +00:00
Merge branch 'dev' into jesserockz-2026-446
This commit is contained in:
@@ -97,6 +97,7 @@ ISOLATED_COMPONENTS = {
|
||||
"neopixelbus": "RMT type conflict with ESP32 Arduino/ESP-IDF headers (enum vs struct rmt_channel_t)",
|
||||
"packages": "cannot merge packages",
|
||||
"tinyusb": "Conflicts with usb_host component - cannot be used together",
|
||||
"uart_mux": "Depends on tinyusb which conflicts with usb_host",
|
||||
"usb_cdc_acm": "Depends on tinyusb which conflicts with usb_host",
|
||||
}
|
||||
|
||||
|
||||
+211
-14
@@ -3,6 +3,7 @@
|
||||
import argparse
|
||||
import codecs
|
||||
import collections
|
||||
from collections.abc import Iterator
|
||||
import fnmatch
|
||||
import functools
|
||||
import os.path
|
||||
@@ -247,6 +248,9 @@ def lint_ext_check(fname):
|
||||
"CLAUDE.md",
|
||||
"GEMINI.md",
|
||||
".github/copilot-instructions.md",
|
||||
# Symlinks to the shared .agents/skills directory
|
||||
".claude/skills",
|
||||
".github/skills",
|
||||
# Symlink to the real wifi scan_list.h so the test stub cannot drift
|
||||
"tests/integration/fixtures/external_components/wifi/scan_list.h",
|
||||
]
|
||||
@@ -525,21 +529,67 @@ def lint_conf_matches(fname, match):
|
||||
CONF_RE = r'^(CONF_[a-zA-Z0-9_]+)\s*=\s*[\'"].*?[\'"]\s*?$'
|
||||
with codecs.open("esphome/const.py", "r", encoding="utf-8") as const_f_handle:
|
||||
constants_content = const_f_handle.read()
|
||||
with codecs.open(
|
||||
"esphome/components/const/__init__.py", "r", encoding="utf-8"
|
||||
) as component_const_f_handle:
|
||||
component_constants_content = component_const_f_handle.read()
|
||||
|
||||
# The two canonical homes for shared constants: esphome/const.py (core, frozen) and
|
||||
# esphome/components/const/__init__.py (shared by components). A constant defined in
|
||||
# either must be imported from there rather than redefined in a component.
|
||||
CONST_HOMES = ["esphome/const.py", "esphome/components/const/__init__.py"]
|
||||
|
||||
CONSTANTS = [m.group(1) for m in re.finditer(CONF_RE, constants_content, re.MULTILINE)]
|
||||
COMPONENT_CONSTANTS = [
|
||||
m.group(1) for m in re.finditer(CONF_RE, component_constants_content, re.MULTILINE)
|
||||
]
|
||||
|
||||
CONSTANTS_USES = collections.defaultdict(list)
|
||||
|
||||
|
||||
@lint_re_check(CONF_RE, include=["*.py"], exclude=["esphome/const.py"])
|
||||
def _const_home_error(name, core_constants, component_constants):
|
||||
"""Return an error if the constant already lives in one of the canonical homes."""
|
||||
if name in core_constants:
|
||||
return (
|
||||
f"Constant {highlight(name)} has already been defined in const.py - "
|
||||
"please import the constant from const.py directly."
|
||||
)
|
||||
if name in component_constants:
|
||||
return (
|
||||
f"Constant {highlight(name)} has already been defined in "
|
||||
"esphome/components/const/__init__.py - please import the constant from "
|
||||
"esphome.components.const directly."
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
@lint_re_check(CONF_RE, include=["*.py"], exclude=CONST_HOMES)
|
||||
def lint_conf_from_const_py(fname, match):
|
||||
name = match.group(1)
|
||||
if name not in CONSTANTS:
|
||||
err = _const_home_error(name, CONSTANTS, COMPONENT_CONSTANTS)
|
||||
if err is None:
|
||||
CONSTANTS_USES[name].append(fname)
|
||||
return None
|
||||
return (
|
||||
f"Constant {highlight(name)} has already been defined in const.py - "
|
||||
"please import the constant from const.py directly."
|
||||
)
|
||||
return err
|
||||
|
||||
|
||||
UNIT_RE = r'^(UNIT_[a-zA-Z0-9_]+)\s*=\s*[\'"].*?[\'"]\s*?$'
|
||||
UNIT_CONSTANTS = [
|
||||
m.group(1) for m in re.finditer(UNIT_RE, constants_content, re.MULTILINE)
|
||||
]
|
||||
COMPONENT_UNIT_CONSTANTS = [
|
||||
m.group(1) for m in re.finditer(UNIT_RE, component_constants_content, re.MULTILINE)
|
||||
]
|
||||
|
||||
UNIT_CONSTANTS_USES = collections.defaultdict(list)
|
||||
|
||||
|
||||
@lint_re_check(UNIT_RE, include=["*.py"], exclude=CONST_HOMES)
|
||||
def lint_unit_from_const_py(fname, match):
|
||||
name = match.group(1)
|
||||
err = _const_home_error(name, UNIT_CONSTANTS, COMPONENT_UNIT_CONSTANTS)
|
||||
if err is None:
|
||||
UNIT_CONSTANTS_USES[name].append(fname)
|
||||
return err
|
||||
|
||||
|
||||
RAW_PIN_ACCESS_RE = (
|
||||
@@ -707,10 +757,24 @@ def lint_constants_usage():
|
||||
return errs
|
||||
|
||||
|
||||
@lint_post_check
|
||||
def lint_unit_constants_usage():
|
||||
errs = []
|
||||
for constant, uses in UNIT_CONSTANTS_USES.items():
|
||||
if len(uses) < 3:
|
||||
continue
|
||||
errs.append(
|
||||
f"Constant {highlight(constant)} is defined in {len(uses)} files. Please move all definitions of the "
|
||||
f"constant to esphome/components/const/__init__.py (Uses: {', '.join(str(u) for u in uses)}) in a separate PR. "
|
||||
"See https://developers.esphome.io/contributing/code/#python"
|
||||
)
|
||||
return errs
|
||||
|
||||
|
||||
# Maximum allowed CONF_ constants in esphome/const.py.
|
||||
# This file is frozen — new constants go in esphome/components/const/__init__.py.
|
||||
# Decrease this number when constants are moved out of const.py.
|
||||
CONST_PY_MAX_CONF = 1017
|
||||
CONST_PY_MAX_CONF = 1016
|
||||
|
||||
|
||||
@lint_content_check(include=["esphome/const.py"])
|
||||
@@ -1135,7 +1199,56 @@ def lint_no_std_bind(fname, match):
|
||||
)
|
||||
|
||||
|
||||
LOG_MULTILINE_RE = re.compile(r"ESP_LOG\w+\s*\(.*?;", re.DOTALL)
|
||||
LOG_CALL_START_RE = re.compile(r"ESP_LOG\w+\s*\(")
|
||||
# Comments, raw/plain string literals and single char literals are consumed whole so ; ( ) ? :
|
||||
# inside them are never seen. A char literal is exactly one (escaped) char so a digit separator
|
||||
# like 1'000'000 cannot open one.
|
||||
CPP_COMMENT_RE = r"//[^\n]*|/\*.*?\*/"
|
||||
CPP_SKIP_RE = (
|
||||
CPP_COMMENT_RE
|
||||
+ r'|R"(?P<raw_delim>[^(\s]*)\(.*?\)(?P=raw_delim)"|"(?:[^"\\]|\\.)*"|\'(?:[^\'\\\n]|\\.)\''
|
||||
)
|
||||
LOG_CALL_TOKEN_RE = re.compile(CPP_SKIP_RE + r"|[()]", re.DOTALL)
|
||||
# The last alternative matches a ? or : followed (after spaces or comments) by an opening quote,
|
||||
# i.e. a string literal used as a ternary branch.
|
||||
LOG_TERNARY_LITERAL_RE = re.compile(
|
||||
CPP_SKIP_RE + r"|[?:](?:\s|" + CPP_COMMENT_RE + r')*(?=")', re.DOTALL
|
||||
)
|
||||
# A bare NOLINT; a clang-tidy NOLINT(check-name) is aimed at a different tool.
|
||||
NOLINT_RE = re.compile(r"\bNOLINT\b(?!\()")
|
||||
|
||||
|
||||
def _line_col(content: str, pos: int) -> tuple[int, int]:
|
||||
"""1-based line and column of an offset in content."""
|
||||
return content.count("\n", 0, pos) + 1, pos - content.rfind("\n", 0, pos)
|
||||
|
||||
|
||||
def _iter_log_calls(content: str) -> Iterator[tuple[int, str | None]]:
|
||||
"""Yield (start, text) for every ESP_LOG*(...) call, text running to the matching close paren.
|
||||
text is None when no matching paren exists so callers can report the call instead of skipping it."""
|
||||
for head in LOG_CALL_START_RE.finditer(content):
|
||||
depth = 1
|
||||
for tok in LOG_CALL_TOKEN_RE.finditer(content, head.end()):
|
||||
if tok.group(0) == "(":
|
||||
depth += 1
|
||||
elif tok.group(0) == ")":
|
||||
depth -= 1
|
||||
if depth == 0:
|
||||
yield head.start(), content[head.start() : tok.end()]
|
||||
break
|
||||
else:
|
||||
yield head.start(), None
|
||||
|
||||
|
||||
def _unbalanced_log_call_error(content: str, pos: int) -> tuple[int, int, str]:
|
||||
lineno, col = _line_col(content, pos)
|
||||
return (
|
||||
lineno,
|
||||
col,
|
||||
"ESP_LOG call has no matching closing parenthesis, so it cannot be checked.",
|
||||
)
|
||||
|
||||
|
||||
LOG_BAD_CONTINUATION_RE = re.compile(r'\\n(?:[^ \\"\r\n\t]|"\s*\n\s*"[^ \\])')
|
||||
LOG_PERCENT_S_CONTINUATION_RE = re.compile(r'\\n(?:%s|"\s*\n\s*"%s)')
|
||||
|
||||
@@ -1143,16 +1256,16 @@ LOG_PERCENT_S_CONTINUATION_RE = re.compile(r'\\n(?:%s|"\s*\n\s*"%s)')
|
||||
@lint_content_check(include=cpp_include)
|
||||
def lint_log_multiline_continuation(fname, content):
|
||||
errs = []
|
||||
for log_match in LOG_MULTILINE_RE.finditer(content):
|
||||
log_text = log_match.group(0)
|
||||
for log_start, log_text in _iter_log_calls(content):
|
||||
if log_text is None:
|
||||
errs.append(_unbalanced_log_call_error(content, log_start))
|
||||
continue
|
||||
for bad_match in LOG_BAD_CONTINUATION_RE.finditer(log_text):
|
||||
# %s may expand to a whitespace prefix at runtime, skip those
|
||||
if LOG_PERCENT_S_CONTINUATION_RE.match(log_text, bad_match.start()):
|
||||
continue
|
||||
# Calculate line number from position in full content
|
||||
abs_pos = log_match.start() + bad_match.start()
|
||||
lineno = content.count("\n", 0, abs_pos) + 1
|
||||
col = abs_pos - content.rfind("\n", 0, abs_pos)
|
||||
lineno, col = _line_col(content, log_start + bad_match.start())
|
||||
errs.append(
|
||||
(
|
||||
lineno,
|
||||
@@ -1170,6 +1283,90 @@ def lint_log_multiline_continuation(fname, content):
|
||||
return errs
|
||||
|
||||
|
||||
def _find_ternary_literals(text: str) -> Iterator[tuple[int, str]]:
|
||||
"""Yield (offset, literal) for every string literal used as a ternary branch."""
|
||||
branch = False
|
||||
for m in LOG_TERNARY_LITERAL_RE.finditer(text):
|
||||
tok = m.group(0)
|
||||
# An empty literal is merged with every other string's terminator, so it costs no RAM,
|
||||
# while a PSTR("") would add its own flash array; leave it alone.
|
||||
if branch and tok[0] == '"' and tok != '""':
|
||||
yield m.start(), tok
|
||||
branch = tok[0] in "?:"
|
||||
|
||||
|
||||
# LOG_STR_LITERAL is a no op everywhere except ESP8266, so code that never builds there is skipped
|
||||
# to avoid churn: platform specific sources and components for ESP32, LibreTiny, RP2 and Zephyr only.
|
||||
# A component belongs here only if it has no tests/components/<name>/test.esp8266-ard.yaml.
|
||||
LOG_LITERAL_LINT_EXCLUDE = [
|
||||
"*_esp32.cpp",
|
||||
"*_esp32_*.cpp",
|
||||
"*_esp_idf.cpp",
|
||||
"*_rmt.cpp",
|
||||
"*_zephyr.cpp",
|
||||
"*_bk72xx.cpp",
|
||||
"*_libretiny.cpp",
|
||||
"*_pico_w.cpp",
|
||||
"*_host.cpp",
|
||||
"esphome/components/esp32*/*",
|
||||
"esphome/components/bk72xx*/*",
|
||||
"esphome/components/ln882h*/*",
|
||||
"esphome/components/ln882x*/*",
|
||||
"esphome/components/rp2*/*",
|
||||
"esphome/components/zephyr*/*",
|
||||
"esphome/components/host/*",
|
||||
"esphome/components/libretiny*/*",
|
||||
"esphome/components/bluetooth_proxy/*",
|
||||
"esphome/components/bluetooth_connection/*",
|
||||
"esphome/components/ble_client/*",
|
||||
"esphome/components/bedjet/*",
|
||||
"esphome/components/anova/*",
|
||||
"esphome/components/xiaomi_ble/*",
|
||||
"esphome/components/bthome_mithermometer/*",
|
||||
"esphome/components/usb_host/*",
|
||||
"esphome/components/zigbee/*",
|
||||
"esphome/components/lvgl/*",
|
||||
# Test fixtures and host only unit tests - not production embedded code
|
||||
"tests/integration/fixtures/*",
|
||||
"tests/components/*",
|
||||
]
|
||||
|
||||
|
||||
@lint_content_check(include=cpp_include, exclude=LOG_LITERAL_LINT_EXCLUDE)
|
||||
def lint_log_no_bare_literal_ternary(
|
||||
fname: Path, content: str
|
||||
) -> list[tuple[int, int, str]]:
|
||||
errs = []
|
||||
for log_start, log_text in _iter_log_calls(content):
|
||||
if log_text is None:
|
||||
continue # reported by lint_log_multiline_continuation, which sees every file
|
||||
# A NOLINT anywhere on the lines the call spans silences every branch in it
|
||||
first_line = content.rfind("\n", 0, log_start) + 1
|
||||
last_line = content.find("\n", log_start + len(log_text))
|
||||
if NOLINT_RE.search(
|
||||
content[first_line : last_line if last_line != -1 else None]
|
||||
):
|
||||
continue
|
||||
for offset, literal in _find_ternary_literals(log_text):
|
||||
lineno, col = _line_col(content, log_start + offset)
|
||||
errs.append(
|
||||
(
|
||||
lineno,
|
||||
col,
|
||||
(
|
||||
"String literal used as a ternary branch in a log call. On ESP8266 the "
|
||||
"log macro moves the format string to flash, but bare literal arguments "
|
||||
"stay in RAM. Wrap each branch passed straight to the log call in "
|
||||
f"{highlight('LOG_STR_LITERAL(...)')}:\n"
|
||||
f" Before: {highlight(literal)}\n"
|
||||
f" After: {highlight(f'LOG_STR_LITERAL({literal})')}\n"
|
||||
f"(If strictly necessary, add `{highlight('// NOLINT')}` to the end of the line)"
|
||||
),
|
||||
)
|
||||
)
|
||||
return errs
|
||||
|
||||
|
||||
@lint_content_find_check(
|
||||
"ESP_LOG",
|
||||
include=["*.h", "*.tcc"],
|
||||
|
||||
@@ -1,377 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
# This script is used to preinstall
|
||||
# all platformio libraries in the global storage
|
||||
|
||||
import argparse
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
import configparser
|
||||
from contextlib import suppress
|
||||
import os
|
||||
from pathlib import Path
|
||||
import queue
|
||||
import subprocess
|
||||
import threading
|
||||
import traceback
|
||||
|
||||
# esphome is not installed at this docker layer; pio's fs.rmtree is the
|
||||
# same chmod-on-readonly shape its own installer uses
|
||||
try:
|
||||
from platformio import fs
|
||||
from platformio.cache import ContentCache
|
||||
from platformio.package.manager.base import BasePackageManager
|
||||
from platformio.package.manager.library import LibraryPackageManager
|
||||
from platformio.package.manager.tool import ToolPackageManager
|
||||
from platformio.package.meta import PackageCompatibility
|
||||
|
||||
PARALLEL_AVAILABLE = True
|
||||
except ImportError as err: # pragma: no cover
|
||||
# A moved pio module must degrade to the serial pass, not kill the
|
||||
# image build; the tripwire test makes the drift loud in CI
|
||||
PARALLEL_AVAILABLE = False
|
||||
IMPORT_ERROR = repr(err)
|
||||
|
||||
# Network-bound downloads release the GIL, so the pool oversubscribes
|
||||
# the cores. This bypasses pio's 500ms registry throttle and races its
|
||||
# self-unlinking cache LockFiles; both are cache-only and self-healing.
|
||||
MAX_WORKERS = 16
|
||||
|
||||
|
||||
class CleanupError(RuntimeError):
|
||||
"""A torn destination could not be removed; the serial pass would
|
||||
trust it, so the build must fail rather than bake a corrupt image."""
|
||||
|
||||
|
||||
class LockReleaseError(RuntimeError):
|
||||
"""The manager lock could not be released; the serial pass would
|
||||
block on it, so the build must fail with the cause named."""
|
||||
|
||||
|
||||
def parse_specs(path: str, args: argparse.Namespace) -> tuple[list, list, list]:
|
||||
"""Extract lib/platform/tool specs from every section of a platformio.ini."""
|
||||
config = configparser.ConfigParser(inline_comment_prefixes=(";",))
|
||||
if not config.read(path):
|
||||
# ConfigParser silently ignores unreadable files; an empty spec
|
||||
# list would build an image with no dependencies at all
|
||||
raise SystemExit(f"Could not read {path}")
|
||||
libs = []
|
||||
tools = []
|
||||
platforms = []
|
||||
for section in config.sections():
|
||||
conf = config[section]
|
||||
if "lib_deps" in conf and args.libraries:
|
||||
for lib_dep in conf["lib_deps"].splitlines():
|
||||
if not lib_dep:
|
||||
# Empty line or comment
|
||||
continue
|
||||
if lib_dep.startswith("${"):
|
||||
# Extending from another section
|
||||
continue
|
||||
if "@" not in lib_dep:
|
||||
# No version pinned, this is an internal lib
|
||||
continue
|
||||
libs.append(lib_dep)
|
||||
if "platform" in conf and args.platforms:
|
||||
platforms.append(conf["platform"])
|
||||
if "platform_packages" in conf and args.tools:
|
||||
for tool in conf["platform_packages"].splitlines():
|
||||
if not tool:
|
||||
# Empty line or comment
|
||||
continue
|
||||
if tool.startswith("${"):
|
||||
# Extending from another section
|
||||
continue
|
||||
if tool.find("https://github.com") != -1:
|
||||
split = tool.find("@")
|
||||
tool = tool[split + 1 :]
|
||||
tools.append(tool)
|
||||
# Exact-string dedupe only: name-level dedupe would change which
|
||||
# version conflicts the pkg install pass reconciles
|
||||
return (
|
||||
list(dict.fromkeys(libs)),
|
||||
list(dict.fromkeys(platforms)),
|
||||
list(dict.fromkeys(tools)),
|
||||
)
|
||||
|
||||
|
||||
def piopm_matches(package_dir: str, spec) -> list[Path]:
|
||||
"""Dirs whose .piopm metadata names this spec; a positive match beats
|
||||
guessing the manifest-derived dirname from the registry name."""
|
||||
want = (BasePackageManager.ensure_spec(spec).name or "").lower()
|
||||
matches: list[Path] = []
|
||||
if not want:
|
||||
return matches
|
||||
try:
|
||||
entries = list(Path(package_dir).iterdir())
|
||||
except FileNotFoundError:
|
||||
return matches
|
||||
for d in entries:
|
||||
if not d.is_dir():
|
||||
continue # pio's get_installed skips files and *.pio-link too
|
||||
try:
|
||||
meta = fs.load_json(str(d / ".piopm"))
|
||||
except FileNotFoundError:
|
||||
continue # no metadata means pio does not trust it either
|
||||
except (OSError, ValueError):
|
||||
if d.name.lower() == want:
|
||||
# A corrupt .piopm under this spec's own name would crash
|
||||
# pio's whole storage scan; remove it
|
||||
matches.append(d)
|
||||
continue
|
||||
mspec = meta.get("spec") or {}
|
||||
if (mspec.get("name") or meta.get("name") or "").lower() == want:
|
||||
matches.append(d)
|
||||
return matches
|
||||
|
||||
|
||||
def remove_dir(spec, dest: Path) -> None:
|
||||
# fs.rmtree never raises (errors go to a printing onexc handler);
|
||||
# only the destination's absence proves the cleanup worked
|
||||
fs.rmtree(str(dest))
|
||||
if dest.exists():
|
||||
# Failing the build beats baking a corrupt image
|
||||
raise CleanupError(
|
||||
f"could not remove the failed pre-install of {spec} at {dest}"
|
||||
)
|
||||
print(f"Removed torn destination {dest}", flush=True)
|
||||
|
||||
|
||||
def cleanup_or_die(mgr, spec) -> None:
|
||||
"""Cleanup that did not demonstrably succeed must fail the build."""
|
||||
try:
|
||||
clean_torn(mgr, spec)
|
||||
except CleanupError:
|
||||
raise
|
||||
except Exception as err: # noqa: BLE001
|
||||
raise CleanupError(f"cleanup failed for {spec}: {err!r}") from err
|
||||
|
||||
|
||||
def clean_torn(mgr, spec) -> None:
|
||||
"""Remove a torn destination so the serial pass cannot trust it."""
|
||||
pkg = None
|
||||
with suppress(Exception):
|
||||
# get_package memoizes a pre-install snapshot; reset to see the
|
||||
# torn dir. It also recognizes manifest-only legacy dirs pio's
|
||||
# storage scan would trust, which the .piopm fallback cannot see.
|
||||
mgr.memcache_reset()
|
||||
pkg = mgr.get_package(spec)
|
||||
if pkg is not None:
|
||||
remove_dir(spec, Path(pkg.path))
|
||||
elif dests := piopm_matches(mgr.package_dir, spec):
|
||||
# A .piopm naming this spec is the exact shape the serial pass
|
||||
# trusts; a dir without one is overwritten by pio's own install
|
||||
for dest in dests:
|
||||
remove_dir(spec, dest)
|
||||
else:
|
||||
print(f"No resolvable destination to clean for {spec}", flush=True)
|
||||
|
||||
|
||||
def spec_key(spec) -> str | None:
|
||||
"""The destination identity of a spec: PlatformIO installs by package
|
||||
name, so two specs sharing a name share a directory. ``None`` means
|
||||
the name could not be derived; such a spec must stay out of the wave
|
||||
(a raw-string key would break the one-per-destination guarantee)."""
|
||||
name = BasePackageManager.ensure_spec(spec).name
|
||||
return name.lower() if name else None
|
||||
|
||||
|
||||
def dependency_specs(manager, specs: list) -> list:
|
||||
"""``(spec, compatibility)`` registry dependencies of installed
|
||||
packages, from local manifest reads. Name-only dependencies
|
||||
(platform-bundled libs like SPI) stay with the ``pkg install`` pass;
|
||||
the compatibility qualifiers mirror pio's install_dependency, so a
|
||||
qualified dep resolves to the same package the serial pass picks."""
|
||||
return [
|
||||
(manager.dependency_to_spec(dep), PackageCompatibility.from_dependency(dep))
|
||||
for spec in specs
|
||||
if (pkg := manager.get_package(spec)) is not None
|
||||
for dep in manager.get_pkg_dependencies(pkg) or []
|
||||
if dep.get("owner") or dep.get("version")
|
||||
]
|
||||
|
||||
|
||||
def parallel_install(manager_cls, specs: list, prior_names: set | None = None) -> None:
|
||||
"""Best-effort parallel top-level install.
|
||||
|
||||
PlatformIO's own installer downloads and unpacks one package at a time
|
||||
on one core. Dependencies are skipped (two packages sharing one must
|
||||
not extract into the same directory from two threads) and failures are
|
||||
only reported: the stock ``pkg install`` pass afterwards installs
|
||||
whatever is missing and is the authority on the final state.
|
||||
"""
|
||||
if not specs:
|
||||
return
|
||||
manager = manager_cls(None)
|
||||
# One spec per destination: two threads must not extract into the
|
||||
# same directory. Second versions of a name and URL specs (their dir
|
||||
# comes from the archive manifest) stay with the pkg install pass.
|
||||
seen_names: set = prior_names if prior_names is not None else set()
|
||||
# Wave-1 items are strings; dependency waves carry (spec, compatibility)
|
||||
pairs = [item if isinstance(item, tuple) else (item, None) for item in specs]
|
||||
unique = {}
|
||||
for spec, compat in pairs:
|
||||
# Normalize once: a dependency's URL version surfaces as spec.uri
|
||||
parsed = BasePackageManager.ensure_spec(spec)
|
||||
if parsed.uri:
|
||||
continue
|
||||
if (key := spec_key(parsed)) is None:
|
||||
# No name, no destination identity; leave it to the serial pass
|
||||
print(f"Skipping unresolvable spec {spec!r} in the wave", flush=True)
|
||||
continue
|
||||
unique.setdefault(key, (spec, compat)) # first-wins, like pio's walk
|
||||
pending = [
|
||||
(spec, compat)
|
||||
for spec, compat in unique.values()
|
||||
if not manager.get_package(spec)
|
||||
]
|
||||
if not pending:
|
||||
# Nothing to install, but a warm store's dependencies must still
|
||||
# feed the next wave (a transitive dep may be missing)
|
||||
_next_wave(manager_cls, manager, unique, seen_names)
|
||||
return
|
||||
workers = min(len(pending), MAX_WORKERS)
|
||||
# One manager per worker (_install mutates instance state); built
|
||||
# serially because construction rewires the shared manager logger
|
||||
managers: queue.SimpleQueue = queue.SimpleQueue()
|
||||
for _ in range(workers):
|
||||
managers.put(manager_cls(None))
|
||||
local = threading.local()
|
||||
|
||||
def install_one(item) -> bool:
|
||||
spec, compat = item
|
||||
if (mgr := getattr(local, "mgr", None)) is None:
|
||||
mgr = local.mgr = managers.get_nowait()
|
||||
try:
|
||||
mgr._install( # noqa: SLF001
|
||||
spec, skip_dependencies=True, compatibility=compat
|
||||
)
|
||||
return True
|
||||
except Exception as err: # noqa: BLE001
|
||||
print(f"Pre-install of {spec} failed ({err!r})", flush=True)
|
||||
cleanup_or_die(mgr, spec)
|
||||
return False
|
||||
except BaseException:
|
||||
# A worker SystemExit (main() guards against it) must not skip
|
||||
# the cleanup and leave a torn dir the serial pass trusts
|
||||
cleanup_or_die(mgr, spec)
|
||||
raise
|
||||
|
||||
print(f"Preinstalling {len(pending)} package(s) with {workers} workers", flush=True)
|
||||
# The serial getter calls create pio's lazy dirs (made without
|
||||
# exist_ok) before cold-cache workers can race the creation
|
||||
manager.get_download_dir()
|
||||
manager.get_tmp_dir()
|
||||
ContentCache("http")
|
||||
cwd = Path.cwd()
|
||||
manager.lock()
|
||||
try:
|
||||
with ThreadPoolExecutor(max_workers=workers) as ex:
|
||||
futures = [ex.submit(install_one, item) for item in pending]
|
||||
# The with-block joined every future; drain them all so a
|
||||
# concurrent CleanupError is never dropped
|
||||
errors = [err for f in futures if (err := f.exception()) is not None]
|
||||
for err in errors:
|
||||
# Every failure is on the record; the raised one is a summary
|
||||
print(f"Wave failure: {err!r}", flush=True)
|
||||
if errors:
|
||||
raise next((e for e in errors if isinstance(e, CleanupError)), errors[0])
|
||||
results = [f.result() for f in futures]
|
||||
finally:
|
||||
try:
|
||||
manager.unlock()
|
||||
except Exception as unlock_err: # noqa: BLE001
|
||||
# A held flock would hang the serial pass in another process;
|
||||
# failing loudly beats an unexplained stuck docker build. Any
|
||||
# in-flight error stays attached as the context.
|
||||
raise LockReleaseError(
|
||||
f"could not release the manager lock: {unlock_err!r}"
|
||||
) from unlock_err
|
||||
# Worker postinstall scripts chdir process-wide (pio's fs.cd);
|
||||
# restore between waves. The serial pass pins its own cwd.
|
||||
with suppress(OSError):
|
||||
os.chdir(cwd)
|
||||
if failures := len(results) - sum(results):
|
||||
# The stock pass retries CLI specs and re-walks installed
|
||||
# packages' dependencies, so failed deps retry too
|
||||
print(
|
||||
f"Pre-install failed for {failures} of {len(results)} package(s); "
|
||||
"pkg install retries them serially",
|
||||
flush=True,
|
||||
)
|
||||
|
||||
# Waves skip dependencies (a shared one must not extract from two
|
||||
# threads); the installed manifests feed the next wave
|
||||
_next_wave(manager_cls, manager, unique, seen_names)
|
||||
|
||||
|
||||
def _next_wave(manager_cls, manager, unique: dict, seen_names: set) -> None:
|
||||
"""Queue the dependency wave for every requested spec, installed or
|
||||
freshly waved; a warm store can still be missing a transitive dep.
|
||||
Terminates without a cap: each wave admits only never-seen names."""
|
||||
seen_names.update(unique)
|
||||
# The pre-wave get_package calls memoized an empty storage snapshot
|
||||
manager.memcache_reset()
|
||||
next_specs = [
|
||||
item
|
||||
for item in dependency_specs(manager, [spec for spec, _ in unique.values()])
|
||||
if spec_key(item[0]) not in seen_names
|
||||
]
|
||||
if next_specs:
|
||||
parallel_install(manager_cls, next_specs, seen_names)
|
||||
|
||||
|
||||
def build_cli_args(libs: list, platforms: list, tools: list) -> list:
|
||||
return [
|
||||
arg
|
||||
for flag, specs in (("-l", libs), ("-p", platforms), ("-t", tools))
|
||||
for spec in specs
|
||||
for arg in (flag, spec)
|
||||
]
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description="")
|
||||
parser.add_argument("file", help="Path to platformio.ini", nargs=1)
|
||||
parser.add_argument(
|
||||
"-l", "--libraries", help="Install libraries", action="store_true"
|
||||
)
|
||||
parser.add_argument(
|
||||
"-p", "--platforms", help="Install platforms", action="store_true"
|
||||
)
|
||||
parser.add_argument("-t", "--tools", help="Install tools", action="store_true")
|
||||
args = parser.parse_args()
|
||||
start_cwd = Path.cwd()
|
||||
libs, platforms, tools = parse_specs(args.file[0], args)
|
||||
|
||||
# Platforms stay serial: PlatformPackageManager.install runs an
|
||||
# on_installed hook the private _install path would skip
|
||||
if PARALLEL_AVAILABLE:
|
||||
wave_groups = [(ToolPackageManager, tools), (LibraryPackageManager, libs)]
|
||||
else: # pragma: no cover
|
||||
wave_groups = []
|
||||
print(
|
||||
f"PlatformIO layout changed ({IMPORT_ERROR}); serial install only",
|
||||
flush=True,
|
||||
)
|
||||
for manager_cls, specs in wave_groups:
|
||||
try:
|
||||
parallel_install(manager_cls, specs)
|
||||
except (CleanupError, LockReleaseError, KeyboardInterrupt):
|
||||
# A torn package or a held lock must fail the build
|
||||
raise
|
||||
except BaseException: # noqa: BLE001
|
||||
# BaseException: a worker postinstall's SystemExit must not
|
||||
# skip the authoritative serial pass (partial deps, exit 0)
|
||||
print("Parallel preinstall failed, falling back to serial", flush=True)
|
||||
traceback.print_exc()
|
||||
|
||||
# Postinstall scripts chdir process-wide (pio's fs.cd captures its
|
||||
# restore path at construction); pin the authoritative pass's cwd
|
||||
subprocess.check_call(
|
||||
["platformio", "pkg", "install", "-g", *build_cli_args(libs, platforms, tools)],
|
||||
close_fds=False,
|
||||
cwd=start_cwd,
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user