Merge remote-tracking branch 'origin/dev' into store-yaml-firmware

# Conflicts:
#	esphome/components/api/api.proto
#	esphome/components/api/api_pb2.h
#	esphome/components/api/api_pb2_service.cpp
#	esphome/yaml_util.py
#	tests/integration/conftest.py
#	tests/unit_tests/test_yaml_util.py

script/analyze_component_buses.py

@@ -52,6 +52,7 @@ COMMON_BUS_PATH = (
 # the packages on the right as well
 PACKAGE_DEPENDENCIES = {
     "modbus": ["uart"],  # modbus packages include uart packages
+    "modbus_server": ["uart"],  # modbus_server packages include uart packages
     # Add more package dependencies here as needed
 }
 
@@ -80,6 +81,7 @@ ISOLATED_SIGNATURE_PREFIX = "isolated_"
 # NOTE: This should be kept in sync with both test_build_components and split_components_for_ci.py
 ISOLATED_COMPONENTS = {
     "animation": "Has display lambda in common.yaml that requires existing display platform - breaks when merged without display",
+    "cdc_acm_uart": "Depends on tinyusb which conflicts with usb_host",
     "esphome": "Defines devices/areas in esphome: section that are referenced in other sections - breaks when merged",
     "ethernet": "Defines ethernet: which conflicts with wifi: used by most components",
     "ethernet_info": "Related to ethernet component which conflicts with wifi",

script/api_protobuf/api_protobuf.py

@@ -475,6 +475,19 @@ TYPE_INFO: dict[int, TypeInfo] = {}
 # TYPE_DOUBLE = 1, TYPE_FIXED64 = 6, TYPE_SFIXED64 = 16, TYPE_SINT64 = 18
 UNSUPPORTED_TYPES = {1: "double", 6: "fixed64", 16: "sfixed64", 18: "sint64"}
 
+# The plaintext frame header budgets 2 varint bytes for the message type
+# (APIPlaintextFrameHelper::HEADER_PADDING), which caps message IDs at 16383.
+MAX_MESSAGE_ID = 16383
+
+
+def validate_message_id(message_id: int, message_name: str) -> None:
+    """Reject message IDs whose plaintext type varint would not fit in 2 bytes."""
+    if message_id > MAX_MESSAGE_ID:
+        raise ValueError(
+            f"Message ID {message_id} for {message_name} exceeds the plaintext "
+            f"2-byte type varint maximum ({MAX_MESSAGE_ID})"
+        )
+
 
 def validate_field_type(field_type: int, field_name: str = "") -> None:
     """Validate that the field type is supported by ESPHome API.
@@ -498,6 +511,15 @@ def create_field_type_info(
     needs_encode: bool = True,
 ) -> TypeInfo:
     """Create the appropriate TypeInfo instance for a field, handling repeated fields and custom options."""
+    if get_field_opt(field, pb.track_presence, False) and (
+        field.label == FieldDescriptorProto.LABEL_REPEATED
+        or field.type != 11
+        or not needs_decode
+    ):
+        raise ValueError(
+            f"track_presence on field '{field.name}' has no effect; it requires "
+            "a non-repeated message field in a message that is decoded"
+        )
     if field.label == FieldDescriptorProto.LABEL_REPEATED:
         # Check if this is a packed_buffer field (zero-copy packed repeated)
         if get_field_opt(field, pb.packed_buffer, False):
@@ -541,6 +563,8 @@ def create_field_type_info(
         return PointerToStringBufferType(field, None)
 
     validate_field_type(field.type, field.name)
+    if field.type == 11:
+        return MessageType(field, needs_decode, needs_encode)
     return TYPE_INFO[field.type](field)
 
 
@@ -937,9 +961,33 @@ class MessageType(TypeInfo):
         # runtime polymorphism through virtual function calls.
         return None
 
+    @property
+    def public_content(self) -> list[str]:
+        content = [self.class_member]
+        if self._track_presence:
+            content.append(f"bool has_{self.name}{{false}};")
+        return content
+
+    @property
+    def _track_presence(self) -> bool:
+        # Presence is only observable on the decode side
+        return self._needs_decode and get_field_opt(
+            self._field, pb.track_presence, False
+        )
+
     @property
     def decode_length_content(self) -> str:
         # Custom decode that doesn't use templates
+        if self._track_presence:
+            # decode_to_message() cannot report failure, so setting the flag
+            # afterwards only documents intent; a status-returning decode could
+            # gate it for real without touching callers.
+            return (
+                f"case {self.number}:\n"
+                f"      value.decode_to_message(this->{self.field_name});\n"
+                f"      this->has_{self.name} = true;\n"
+                f"      break;"
+            )
         return f"case {self.number}: value.decode_to_message(this->{self.field_name}); break;"
 
     def dump(self, name: str) -> str:
@@ -947,7 +995,10 @@ class MessageType(TypeInfo):
 
     @property
     def dump_content(self) -> str:
-        o = f'out.append(2, \' \').append_p(ESPHOME_PSTR("{self.name}")).append(": ");\n'
+        o = ""
+        if self._track_presence:
+            o += f'dump_field(out, ESPHOME_PSTR("has_{self.name}"), this->has_{self.name});\n'
+        o += f'out.append(2, \' \').append_p(ESPHOME_PSTR("{self.name}")).append(": ");\n'
         o += f"this->{self.field_name}.dump_to(out);\n"
         o += 'out.append("\\n");'
         return o
@@ -2401,7 +2452,10 @@ def get_varint64_ifdef(
         # At least one 64-bit varint field is unconditional, so the guard must be unconditional.
         return True, None
     ifdefs.discard(None)
-    return True, ifdefs.pop() if len(ifdefs) == 1 else None
+    # Several guards: the define is needed under any of them, so emit the union.
+    # Falling back to unconditional would pull 64-bit varint support into builds
+    # that have none of them.
+    return True, " || ".join(sorted(ifdefs))
 
 
 def build_enum_type(desc, enum_ifdef_map) -> tuple[str, str, str]:
@@ -2508,14 +2562,10 @@ def build_message_type(
 
     # Add MESSAGE_TYPE method if this is a service message
     if message_id is not None:
-        # Validate that message_id fits in uint8_t
-        if message_id > 255:
-            raise ValueError(
-                f"Message ID {message_id} for {desc.name} exceeds uint8_t maximum (255)"
-            )
+        validate_message_id(message_id, desc.name)
 
         # Add static constexpr for message type
-        public_content.append(f"static constexpr uint8_t MESSAGE_TYPE = {message_id};")
+        public_content.append(f"static constexpr uint16_t MESSAGE_TYPE = {message_id};")
 
         # Add estimated size constant
         estimated_size = calculate_message_estimated_size(desc)
@@ -3171,8 +3221,12 @@ def main() -> None:
 #include "api_pb2_includes.h"
 """
 
-    content += """
-namespace esphome::api {
+    content += f"""
+namespace esphome::api {{
+
+// Upper bound on message IDs, enforced by the code generator: the plaintext
+// frame header budgets 2 varint bytes for the type (HEADER_PADDING).
+static constexpr uint16_t MAX_MESSAGE_TYPE = {MAX_MESSAGE_ID};
 
 """
 

script/build_alias_registry.py

@@ -0,0 +1,59 @@
+#!/usr/bin/env python3
+"""Generate esphome/component_aliases.py from component ALIASES declarations.
+
+Run without arguments to regenerate the registry; ``--check`` (run in CI)
+verifies it is up to date.
+"""
+
+import argparse
+from pathlib import Path
+import sys
+
+# The root directory of the repo
+root = Path(__file__).parent.parent
+# Make the repo's esphome package win over any installed copy
+sys.path.insert(0, str(root))
+
+from esphome.helpers import write_file_if_changed  # noqa: E402
+from esphome.loader import _build_alias_map  # noqa: E402
+
+parser = argparse.ArgumentParser()
+parser.add_argument(
+    "--check",
+    help="Check if the alias registry is up to date.",
+    action="store_true",
+)
+args = parser.parse_args()
+
+registry_file = root / "esphome" / "component_aliases.py"
+
+HEADER = '''"""Component alias registry.
+
+Generated by script/build_alias_registry.py - do not edit manually.
+See the component-alias section of esphome/loader.py.
+"""
+
+# alias -> (canonical component, removal version or None)
+COMPONENT_ALIASES: dict[str, tuple[str, str | None]] = {
+'''
+
+# _build_alias_map scans the real component tree and already rejects
+# duplicate and shadowing aliases with an EsphomeError.
+_, alias_meta = _build_alias_map()
+
+lines = [HEADER]
+for alias, meta in sorted(alias_meta.items()):
+    removal = f'"{meta.removal_version}"' if meta.removal_version else "None"
+    lines.append(f'    "{alias}": ("{meta.canonical}", {removal}),\n')
+lines.append("}\n")
+content = "".join(lines)
+
+if args.check:
+    if registry_file.read_text(encoding="utf-8") != content:
+        print("Component alias registry is not up to date.")
+        print("Please run `script/build_alias_registry.py`")
+        sys.exit(1)
+    print("Component alias registry is up to date")
+else:
+    write_file_if_changed(registry_file, content)
+    print(f"Wrote {registry_file}")

script/build_helpers.py

@@ -10,7 +10,7 @@ from pathlib import Path
 import subprocess
 import sys
 
-from helpers import get_all_dependencies, root_path as _root_path
+from helpers import get_all_dependencies, has_cpp_unit_tests, root_path as _root_path
 import yaml
 
 # Ensure the repo root is on sys.path so that ``tests.testing_helpers`` and
@@ -131,14 +131,11 @@ def filter_components_with_files(components: list[str], tests_dir: Path) -> list
     """
     filtered_components: list[str] = []
     for component in components:
-        test_dir = tests_dir / component
-        if test_dir.is_dir() and (
-            any(test_dir.glob("*.cpp")) or any(test_dir.glob("*.h"))
-        ):
+        if has_cpp_unit_tests(component, tests_dir):
             filtered_components.append(component)
         else:
             print(
-                f"WARNING: No files found for component '{component}' in {test_dir}, skipping.",
+                f"WARNING: No files found for component '{component}' in {tests_dir / component}, skipping.",
                 file=sys.stderr,
             )
     return filtered_components

script/build_language_schema.py

@@ -250,6 +250,16 @@ def add_pin_validators():
         "modes": ["input"],
     }
 
+    from esphome.components import gpio_expander
+
+    # Wraps pins.internal_gpio_input_pin_schema, so the editor schema must keep
+    # treating the config var as a pin
+    pin_validators[repr(gpio_expander.validate_interrupt_pin)] = {
+        "schema": True,
+        "internal": True,
+        "modes": ["input"],
+    }
+
 
 def add_module_registries(domain, module):
     for attr_name in dir(module):
@@ -1134,13 +1144,29 @@ def convert_keys(converted, schema, path):
         else:
             converted["key"] = "String"
             key_string_match = re.search(
-                r"<function (\w*) at \w*>", str(k), re.IGNORECASE
+                r"<function ([^ ]+) at \w+>", str(k), re.IGNORECASE
             )
             if key_string_match:
                 converted["key_type"] = key_string_match.group(1)
             else:
                 converted["key_type"] = str(k)
 
+        # A marker-wrapped callable key (e.g. script.execute's
+        # ``cv.Optional(validate_parameter_name)``) is a wildcard matcher;
+        # ``str(marker)`` is the function repr, whose heap address would
+        # churn the dump every build. Normalize like the bare-callable
+        # branch above: record the validator name in ``key_type`` and file
+        # the config var under ``string``.
+        key_name = str(k)
+        if isinstance(k, vol.Marker) and callable(k.schema):
+            key_string_match = re.search(
+                r"<function ([^ ]+) at \w+>", key_name, re.IGNORECASE
+            )
+            result["key_type"] = (
+                key_string_match.group(1) if key_string_match else key_name
+            )
+            key_name = "string"
+
         # ``cv.OnlyWith`` / ``cv.OnlyWithout`` expose ``default`` as
         # a property that returns ``vol.UNDEFINED`` when the gating
         # component isn't loaded — and at schema-generation time
@@ -1220,7 +1246,7 @@ def convert_keys(converted, schema, path):
         for base_k, base_v in get_overridden_config(k, converted).items():
             if base_k in result and base_v == result[base_k]:
                 result.pop(base_k)
-        converted["schema"][S_CONFIG_VARS][str(k)] = result
+        converted["schema"][S_CONFIG_VARS][key_name] = result
         if "key" in converted and converted["key"] == "String":
             config_vars = converted["schema"]["config_vars"]
             assert len(config_vars) == 1

script/check_import_time.py

@@ -194,7 +194,7 @@ def cmd_update(args: argparse.Namespace) -> int:
 
 
 def cmd_har_only(args: argparse.Namespace) -> int:
-    Path(args.har).write_text(run_waterfall(TARGET_MODULE))
+    Path(args.har).write_text(run_waterfall(TARGET_MODULE), encoding="utf-8")
     print(f"Wrote waterfall HAR to {args.har}")
     return 0
 

script/ci-custom.py

@@ -247,6 +247,8 @@ def lint_ext_check(fname):
         "CLAUDE.md",
         "GEMINI.md",
         ".github/copilot-instructions.md",
+        # Symlink to the real wifi scan_list.h so the test stub cannot drift
+        "tests/integration/fixtures/external_components/wifi/scan_list.h",
     ]
 )
 def lint_executable_bit(fname: Path) -> str | None:
@@ -292,6 +294,9 @@ def highlight(s):
         "esphome/components/socket/headers.h",
         "esphome/core/defines.h",
         "esphome/components/http_request/httplib.h",
+        # Shared C wire header (byte-identical with the co-processor firmware);
+        # these are protocol constants and constexpr is C++-only.
+        "esphome/components/esp32_hosted/esp_now_hosted_rpc.h",
     ],
 )
 def lint_no_defines(fname, match):
@@ -317,6 +322,154 @@ def lint_no_long_delays(fname, match):
     )
 
 
+# An if/else/for/while whose only body is an unbraced ESP_LOG*() call. When the build's compile-time
+# log level drops that macro, the body expands to nothing and the compiler warns (-Wempty-body).
+# clang-tidy's brace check does not catch these (ShortStatementLines allows short unbraced bodies), so
+# this fills that gap. Matched against comment/string-masked content, so commented-out or quoted code
+# is ignored. Both spellings are covered: core/log.h defines the uppercase ESP_LOG*() macros and
+# the lowercase esph_log_*() ones, and both expand to nothing below their log level.
+# 'for' allows ';' inside its parentheses (the classic C-style header); 'if'/'while' do not, so their
+# condition cannot run past the statement it guards. The 'for' header permits one level of nested
+# parens so it stays bounded to its own statement: without that, it can run past the loop body and
+# latch onto a later ')', mis-reporting the line and skipping the '#' preprocessor check below.
+ESP_LOG_NEEDS_BRACES_RE = re.compile(
+    r"(?:\bif\s*\([^{};]*\)|\bwhile\s*\([^{};]*\)|\bfor\s*\((?:[^{}()]|\([^{}()]*\))*\)|\belse\b)"
+    r"[ \t]*\n?[ \t]*(?:ESP_LOG[A-Z]*|esph_log_[a-z]+)\s*\(",
+    re.MULTILINE,
+)
+
+
+def _mask_cpp_comments_strings(s):
+    """Return s with // and /* */ comments and string/char/raw-string literals blanked to spaces
+    (length and newlines preserved) so a regex only matches real code. Parentheses in real code are
+    kept, so callers can still balance them on the masked text."""
+    out = list(s)
+    i = 0
+    n = len(s)
+    while i < n:
+        c = s[i]
+        # Raw string literal: an optional encoding prefix, then R"delim( ... )delim". The body may
+        # contain quotes, //, /* and unbalanced parens, so it must be consumed as one unit.
+        if c == "R" and i + 1 < n and s[i + 1] == '"':
+            j = i + 2
+            delim = ""
+            while j < n and s[j] not in "( \t\r\n\\" and len(delim) < 16:
+                delim += s[j]
+                j += 1
+            if j < n and s[j] == "(":
+                closing = ")" + delim + '"'
+                end = s.find(closing, j + 1)
+                end = n if end == -1 else end + len(closing)
+                for k in range(i, end):
+                    if s[k] != "\n":
+                        out[k] = " "
+                i = end
+                continue
+            i += 1
+        elif c == "/" and i + 1 < n and s[i + 1] == "/":
+            while i < n and s[i] != "\n":
+                out[i] = " "
+                i += 1
+        elif c == "/" and i + 1 < n and s[i + 1] == "*":
+            out[i] = out[i + 1] = " "
+            i += 2
+            while i < n and not (s[i] == "*" and i + 1 < n and s[i + 1] == "/"):
+                if s[i] != "\n":
+                    out[i] = " "
+                i += 1
+            if i < n:
+                out[i] = " "
+            if i + 1 < n:
+                out[i + 1] = " "
+            i += 2
+        # A "'" after an alphanumeric or '_' is a C++ digit separator (1'000), not a literal opener.
+        elif c == '"' or (
+            c == "'" and not (i and (s[i - 1].isalnum() or s[i - 1] == "_"))
+        ):
+            quote = c
+            out[i] = " "
+            i += 1
+            while i < n:
+                if s[i] == "\\":
+                    out[i] = " "
+                    if i + 1 < n:
+                        out[i + 1] = " "
+                    i += 2
+                    continue
+                if s[i] == quote:
+                    out[i] = " "
+                    i += 1
+                    break
+                if s[i] != "\n":
+                    out[i] = " "
+                i += 1
+        else:
+            i += 1
+    return "".join(out)
+
+
+def _log_statement_end(masked, open_paren):
+    """Index of the ';' ending the ESP_LOG call whose '(' is at open_paren, or None. Balanced on the
+    masked text so quotes/comments inside the arguments do not confuse the paren count."""
+    depth = 0
+    i = open_paren
+    n = len(masked)
+    while i < n:
+        ch = masked[i]
+        if ch == "(":
+            depth += 1
+        elif ch == ")":
+            depth -= 1
+            if depth == 0:
+                j = i + 1
+                while j < n and masked[j] != ";":
+                    if not masked[j].isspace():
+                        return None
+                    j += 1
+                return j if j < n else None
+        i += 1
+    return None
+
+
+@lint_content_check(include=cpp_include)
+def lint_esp_log_needs_braces(fname, content):
+    # Cheap bailout: no log call means nothing to flag, and skips masking the file entirely.
+    if "ESP_LOG" not in content and "esph_log_" not in content:
+        return []
+    masked = _mask_cpp_comments_strings(content)
+    errors = []
+    for match in ESP_LOG_NEEDS_BRACES_RE.finditer(masked):
+        pos = match.start()
+        line_start = content.rfind("\n", 0, pos) + 1
+        # Skip preprocessor conditionals (#if/#else/#elif): not C++ control statements.
+        if content[line_start:pos].lstrip().startswith("#"):
+            continue
+        # A '// NOLINT' may sit at the end of the log line (where the message says to put it) or on the
+        # control-statement line, so scan the whole statement rather than only up to the ESP_LOG token.
+        stmt_end = _log_statement_end(masked, match.end() - 1)
+        nolint_end = (
+            content.find("\n", stmt_end) if stmt_end is not None else match.end()
+        )
+        if nolint_end == -1:
+            nolint_end = len(content)
+        if "NOLINT" in content[pos:nolint_end]:
+            continue
+        snippet = content[pos : match.end()].replace("\n", " ").strip()
+        errors.append(
+            (
+                content.count("\n", 0, pos) + 1,
+                pos - line_start + 1,
+                (
+                    f"{highlight(snippet)} - an if/else/for/while body that is a single log "
+                    "call must be wrapped in braces. When the log level compiles the macro out, the "
+                    "body becomes empty and the compiler warns (-Wempty-body). Add { } around the "
+                    "log call (or a '// NOLINT' comment if this is genuinely intended)."
+                ),
+            )
+        )
+    return errors
+
+
 @lint_content_check(
     include=[
         "esphome/const.py",
@@ -345,9 +498,11 @@ def lint_const_ordered(fname, content):
                 (
                     mi,
                     1,
-                    f"Constant {highlight(mline)} is not ordered, please make sure all "
-                    f"constants are ordered. See line {mi} (should go to line {target}, "
-                    f"{target_text})",
+                    (
+                        f"Constant {highlight(mline)} is not ordered, please make sure all "
+                        f"constants are ordered. See line {mi} (should go to line {target}, "
+                        f"{target_text})"
+                    ),
                 )
             )
     return errs
@@ -555,7 +710,7 @@ def lint_constants_usage():
 # Maximum allowed CONF_ constants in esphome/const.py.
 # This file is frozen — new constants go in esphome/components/const/__init__.py.
 # Decrease this number when constants are moved out of const.py.
-CONST_PY_MAX_CONF = 1015
+CONST_PY_MAX_CONF = 1017
 
 
 @lint_content_check(include=["esphome/const.py"])
@@ -664,6 +819,10 @@ def lint_relative_py_import(fname: Path, line, col, content):
         "esphome/components/host/helpers.cpp",
         "esphome/components/zephyr/helpers.cpp",
         "esphome/components/http_request/httplib.h",
+        # Global extern "C" esp_now_* linker symbols + shared C wire header;
+        # neither can live in a C++ namespace.
+        "esphome/components/esp32_hosted/esp_now_hosted.cpp",
+        "esphome/components/esp32_hosted/esp_now_hosted_rpc.h",
     ],
 )
 def lint_namespace(fname: Path, content: str) -> str | None:
@@ -689,7 +848,15 @@ def lint_esphome_h(fname, line, col, content):
     )
 
 
-@lint_content_check(include=["*.h"], exclude=["esphome/core/entity_types.h"])
+@lint_content_check(
+    include=["*.h"],
+    exclude=[
+        "esphome/core/entity_types.h",
+        # Shared C wire header; uses a classic #ifndef guard for portability
+        # across the co-processor firmware repo it stays byte-identical with.
+        "esphome/components/esp32_hosted/esp_now_hosted_rpc.h",
+    ],
+)
 def lint_pragma_once(fname, content):
     if "#pragma once" not in content:
         return (
@@ -990,12 +1157,14 @@ def lint_log_multiline_continuation(fname, content):
                 (
                     lineno,
                     col,
-                    "Multi-line log message has a continuation line that does "
-                    "not start with a space. The log viewer uses leading "
-                    "whitespace to detect continuation lines and re-add the "
-                    f"log tag prefix (e.g. {highlight('[C][component:042]:')}).\n"
-                    "Either start the continuation with a space/indent, or "
-                    "split into separate ESP_LOG* calls.",
+                    (
+                        "Multi-line log message has a continuation line that does "
+                        "not start with a space. The log viewer uses leading "
+                        "whitespace to detect continuation lines and re-add the "
+                        f"log tag prefix (e.g. {highlight('[C][component:042]:')}).\n"
+                        "Either start the continuation with a space/indent, or "
+                        "split into separate ESP_LOG* calls."
+                    ),
                 )
             )
     return errs
@@ -1073,10 +1242,12 @@ def lint_test_package_key_matches_bus(fname, content):
                 (
                     lineno,
                     1,
-                    f"Package key {highlight(pkg_key)} does not match bus directory "
-                    f"{highlight(bus_dir)}. The package key must match the directory "
-                    f"name under tests/test_build_components/common/. "
-                    f"Change {highlight(pkg_key)} to {highlight(bus_dir)}.",
+                    (
+                        f"Package key {highlight(pkg_key)} does not match bus directory "
+                        f"{highlight(bus_dir)}. The package key must match the directory "
+                        f"name under tests/test_build_components/common/. "
+                        f"Change {highlight(pkg_key)} to {highlight(bus_dir)}."
+                    ),
                 )
             )
     return errs

script/ci_check_test_fixture_list_form.py

@@ -0,0 +1,103 @@
+#!/usr/bin/env python3
+"""Fail when a test fixture writes a platform-list domain as a single dict.
+
+Component tests are merged and built in groups in CI (see
+``script/merge_component_configs.py``). ESPHome's ``merge_config`` concatenates
+two lists, but when one side is a dict it replaces the other side wholesale
+(``esphome/config_helpers.py``). A domain such as ``one_wire:`` or ``ota:``
+written in single-dict form therefore deletes every entry other components
+contributed to that domain before it in the merge, and is itself deleted by any
+list that merges after it. The resulting failure only appears when the affected
+components land in the same group -- usually a full component matrix run on an
+unrelated PR long after the fixture was written (this is what broke the
+dallas_temp tests when ds2484 was added, see #17868).
+
+This guard scans every fixture under ``tests/components/`` and rejects any
+top-level domain written as a dict with a ``platform`` key. Such a domain is by
+definition a platform list (single-dict form is only user-config sugar), so the
+fix is always to write it as a one-element list:
+
+    one_wire:
+      - platform: gpio
+        pin: 4
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+import sys
+
+sys.path.insert(0, str(Path(__file__).parent.parent))
+
+from esphome.core import EsphomeError  # noqa: E402
+from script.analyze_component_buses import ISOLATED_COMPONENTS  # noqa: E402
+from script.merge_component_configs import load_yaml_file  # noqa: E402
+
+# Resolved relative to this file (not the CWD) so the scan cannot silently cover
+# nothing when run from a different directory.
+ROOT_DIR = Path(__file__).resolve().parent.parent
+TESTS_DIR = ROOT_DIR / "tests" / "components"
+
+
+def main() -> int:
+    offenders: list[str] = []
+    parse_errors: list[str] = []
+    fixtures_scanned = 0
+
+    for fixture in sorted(TESTS_DIR.glob("*/*.yaml")):
+        # Isolated components are never merged with others, so dict form
+        # cannot clobber anyone there.
+        if fixture.parent.name in ISOLATED_COMPONENTS:
+            continue
+        try:
+            data = load_yaml_file(fixture)
+        except EsphomeError as err:
+            parse_errors.append(f"{fixture.relative_to(ROOT_DIR)}: {err}")
+            continue
+        fixtures_scanned += 1
+        if not isinstance(data, dict):
+            continue
+        for key, value in data.items():
+            if isinstance(value, dict) and "platform" in value:
+                offenders.append(f"{fixture.relative_to(ROOT_DIR)}: '{key}:'")
+
+    if offenders:
+        print("Test fixtures with platform domains in single-dict form:\n")
+        for line in offenders:
+            print(f"  - {line}")
+        print(
+            "\nWrite the domain as a one-element list ('- platform: ...') so "
+            "grouped CI builds can merge it with other components' entries; "
+            "in dict form it replaces or is replaced by their lists wholesale."
+        )
+
+    if parse_errors:
+        # A fixture we could not parse was never scanned, so the run is not a
+        # clean pass even if no offenders were found among the rest.
+        print(
+            f"\n{len(parse_errors)} test fixture(s) could not be parsed and "
+            "were not checked:"
+        )
+        for line in parse_errors:
+            print(f"  - {line}")
+
+    if fixtures_scanned == 0:
+        # A scan that covered nothing is a false green -- the whole point of the
+        # guard is defeated. Fail loudly (wrong working directory or layout change).
+        print(
+            f"\nERROR: scanned 0 test fixtures under {TESTS_DIR}; "
+            "the guard covered nothing.",
+            file=sys.stderr,
+        )
+
+    if offenders or parse_errors or fixtures_scanned == 0:
+        return 1
+
+    print(
+        f"No single-dict platform domains found ({fixtures_scanned} fixtures scanned)."
+    )
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

script/ci_memory_impact_comment.py

@@ -20,17 +20,21 @@ from jinja2 import Environment, FileSystemLoader
 sys.path.insert(0, str(Path(__file__).parent.parent))
 
 # pylint: disable=wrong-import-position
+from helpers import run_gh_command  # noqa: E402
 
 # Comment marker to identify our memory impact comments
 COMMENT_MARKER = "<!-- esphome-memory-impact-analysis -->"
 
 
-def run_gh_command(args: list[str], operation: str) -> subprocess.CompletedProcess:
-    """Run a gh CLI command with error handling.
+def run_gh_command_logged(
+    args: list[str], operation: str, *, retry: bool = True
+) -> subprocess.CompletedProcess:
+    """Run a gh CLI command with retries and error reporting.
 
     Args:
         args: Command arguments (including 'gh')
         operation: Description of the operation for error messages
+        retry: Pass False for non-idempotent commands (see run_gh_command)
 
     Returns:
         CompletedProcess result
@@ -39,12 +43,7 @@ def run_gh_command(args: list[str], operation: str) -> subprocess.CompletedProce
         subprocess.CalledProcessError: If command fails (with detailed error output)
     """
     try:
-        return subprocess.run(
-            args,
-            check=True,
-            capture_output=True,
-            text=True,
-        )
+        return run_gh_command(args, retry=retry)
     except subprocess.CalledProcessError as e:
         print(
             f"ERROR: {operation} failed with exit code {e.returncode}", file=sys.stderr
@@ -472,7 +471,7 @@ def find_existing_comment(pr_number: str) -> str | None:
     print(f"DEBUG: Looking for existing comment on PR #{pr_number}", file=sys.stderr)
 
     # Use gh api to get comments directly - this returns the numeric id field
-    result = run_gh_command(
+    result = run_gh_command_logged(
         [
             "gh",
             "api",
@@ -535,7 +534,7 @@ def update_existing_comment(comment_id: str, comment_body: str) -> None:
     """
     print(f"DEBUG: Updating existing comment {comment_id}", file=sys.stderr)
     print(f"DEBUG: Comment body length: {len(comment_body)} bytes", file=sys.stderr)
-    result = run_gh_command(
+    result = run_gh_command_logged(
         [
             "gh",
             "api",
@@ -562,9 +561,12 @@ def create_new_comment(pr_number: str, comment_body: str) -> None:
     """
     print(f"DEBUG: Posting new comment on PR #{pr_number}", file=sys.stderr)
     print(f"DEBUG: Comment body length: {len(comment_body)} bytes", file=sys.stderr)
-    result = run_gh_command(
+    # Creating a comment is not idempotent: a retry after a dropped response
+    # could post the same comment twice, so fail on the first error instead.
+    result = run_gh_command_logged(
         ["gh", "pr", "comment", pr_number, "--body", comment_body],
         operation="Create PR comment",
+        retry=False,
     )
     print(f"DEBUG: Post response: {result.stdout}", file=sys.stderr)
 

script/ci_memory_impact_extract.py

@@ -269,7 +269,7 @@ def main() -> int:
     if args.output_build_dir and build_dir:
         build_dir_path = Path(args.output_build_dir)
         build_dir_path.parent.mkdir(parents=True, exist_ok=True)
-        build_dir_path.write_text(build_dir)
+        build_dir_path.write_text(build_dir, encoding="utf-8")
         print(f"Wrote build directory to {args.output_build_dir}", file=sys.stderr)
 
     # Run detailed analysis if build directory available

script/clang-tidy

@@ -5,11 +5,13 @@ import os
 from pathlib import Path
 import queue
 import re
+import shlex
 import shutil
 import subprocess
 import sys
 import tempfile
 import threading
+from typing import Any
 
 import click
 import colorama
@@ -29,6 +31,36 @@ from helpers import (
 )
 
 
+def gcc_multilib_directory(idedata: dict[str, Any]) -> str | None:
+    """The toolchain's active multilib subdirectory (e.g. "thumb"), if any.
+
+    PlatformIO's idedata lists the generic toolchain include directories; GCC
+    resolves the active multilib subdirectory internally while searching them.
+    Toolchains without a default multilib (pico-quick-toolchain 5.0.0+) ship
+    the libstdc++ target config (bits/c++config.h) only inside the multilib
+    subdirectories, so clang needs the resolved directory spelled out.
+    """
+    machine_flags = [f for f in idedata["cxx_flags"] if f.startswith("-m")]
+    cmd = [idedata["cxx_path"], *machine_flags, "-print-multi-directory"]
+    try:
+        multilib = subprocess.run(
+            cmd,
+            capture_output=True,
+            text=True,
+            check=True,
+        ).stdout.strip()
+    except (OSError, subprocess.CalledProcessError) as err:
+        # Without the multilib dir, toolchains lacking a default multilib fail
+        # later with "bits/c++config.h not found"; point at the probe instead.
+        stderr = getattr(err, "stderr", "") or ""
+        print(
+            f"WARNING: multilib probe failed ({shlex.join(cmd)}): {err} {stderr}".strip(),
+            file=sys.stderr,
+        )
+        return None
+    return None if multilib in ("", ".") else multilib
+
+
 def clang_options(idedata, environment):
     cmd = []
 
@@ -203,9 +235,12 @@ def clang_options(idedata, environment):
     # toolchain include directories, using -isystem to suppress their errors
     # idedata contains include directories for all toolchains of this platform, only use those from the one in use
     toolchain_dir = os.path.normpath(f"{idedata['cxx_path']}/../../")
+    multilib = gcc_multilib_directory(idedata)
     toolchain_includes = []
     for directory in idedata["includes"]["toolchain"]:
         if directory.startswith(toolchain_dir) and "picolibc" not in directory:
+            if multilib and (multilib_dir := Path(directory) / multilib).is_dir():
+                toolchain_includes.extend(["-isystem", str(multilib_dir)])
             toolchain_includes.extend(["-isystem", directory])
 
     # library include directories, using -isystem to suppress their errors

script/clang_tidy_hash.py

@@ -1,13 +1,10 @@
-"""Files that affect clang-tidy results, and a content hash over them.
+"""Files that affect clang-tidy results and the idedata built from them.
 
-``CLANG_TIDY_GLOBAL_FILES`` (plus ``SDKCONFIG_DEFAULTS_PREFIX``) is the single
-source of truth for which files influence clang-tidy output. A change to any of
-them can surface warnings in source files a PR didn't touch, so:
-
-* ``script/determine-jobs.py`` runs a full clang-tidy scan when one changes, and
-* ``calculate_clang_tidy_hash()`` folds them into the idedata cache key used by
-  ``script/helpers.py`` (a content hash, unlike an mtime check, stays correct
-  across git checkouts).
+``CLANG_TIDY_GLOBAL_FILES`` (plus ``SDKCONFIG_DEFAULTS_PREFIX``) lists the files
+that influence clang-tidy output; ``script/determine-jobs.py`` runs a full scan
+when one changes. ``ESP_IDF_INFRA_TRIGGER_*`` lists the native ESP-IDF build
+code. ``idedata_cache_hash()`` folds the right set into the idedata cache key
+used by ``script/helpers.py`` and the CI cache action.
 """
 
 from __future__ import annotations
@@ -18,6 +15,7 @@ from pathlib import Path
 # Root-relative paths whose contents affect clang-tidy results.
 CLANG_TIDY_GLOBAL_FILES = (
     ".clang-tidy",
+    "script/clang-tidy",
     "platformio.ini",
     "requirements_dev.txt",
     "esphome/idf_component.yml",
@@ -30,6 +28,18 @@ CLANG_TIDY_GLOBAL_FILES = (
 # this prefix at the repo root.
 SDKCONFIG_DEFAULTS_PREFIX = "sdkconfig.defaults"
 
+# Native ESP-IDF build infra: determine-jobs forces an esp32 compile when these
+# change, and they feed the clang-tidy idedata cache key.
+ESP_IDF_INFRA_TRIGGER_PATH_PREFIXES = ("esphome/espidf/", "esphome/build_helpers/")
+ESP_IDF_INFRA_TRIGGER_FILES = frozenset(
+    {
+        "esphome/build_gen/espidf.py",
+        "esphome/framework_helpers.py",
+        "esphome/platformio/library.py",
+        "esphome/platformio/extra_script.py",
+    }
+)
+
 
 def read_file_bytes(path: Path) -> bytes:
     """Read bytes from a file."""
@@ -65,3 +75,33 @@ def calculate_clang_tidy_hash(repo_root: Path | None = None) -> str:
         hasher.update(read_file_bytes(path))
 
     return hasher.hexdigest()
+
+
+def calculate_idedata_cache_hash(repo_root: Path | None = None) -> str:
+    """Clang-tidy hash plus the Python that generates the idedata."""
+    repo_root = _ensure_repo_root(repo_root)
+
+    hasher = hashlib.sha256()
+    hasher.update(calculate_clang_tidy_hash(repo_root).encode())
+
+    paths = {repo_root / name for name in ESP_IDF_INFRA_TRIGGER_FILES}
+    for prefix in ESP_IDF_INFRA_TRIGGER_PATH_PREFIXES:
+        # .pyc files appear between the CI key computation and load_idedata's.
+        paths.update(
+            path
+            for path in (repo_root / prefix).rglob("*")
+            if "__pycache__" not in path.parts
+        )
+    for path in sorted(paths):
+        if path.is_file():
+            hasher.update(str(path.relative_to(repo_root)).encode())
+            hasher.update(read_file_bytes(path))
+
+    return hasher.hexdigest()
+
+
+def idedata_cache_hash(environment: str, repo_root: Path | None = None) -> str:
+    """Hash gating the cached idedata of one clang-tidy environment."""
+    if "esp32" in environment:
+        return calculate_idedata_cache_hash(repo_root)
+    return calculate_clang_tidy_hash(repo_root)

script/cpp_unit_test.py

@@ -36,7 +36,8 @@ PLATFORMIO_OPTIONS = {
 
 
 def run_tests(selected_components: list[str]) -> int:
-    os.environ["ASAN_OPTIONS"] = "detect_leaks=0"
+    # allocator_may_return_null: an oversized request must come back empty, not abort the run
+    os.environ["ASAN_OPTIONS"] = "detect_leaks=0:allocator_may_return_null=1"
     return build_and_run(
         selected_components=selected_components,
         tests_dir=COMPONENTS_TESTS_DIR,

script/determine-jobs.py

@@ -23,7 +23,7 @@ what files have changed. It outputs JSON with the following structure:
 }
 
 The CI workflow uses this information to:
-- Gate the unconditional jobs (ci-custom, pytest, pre-commit-ci-lite) via core_ci;
+- Gate the unconditional jobs (ci-custom, pytest, lint-format) via core_ci;
   false when a pull_request only touches CI-irrelevant meta paths (other workflow
   files, .github/actions/build-image/*, .yamllint, .github/dependabot.yml, docker/**)
   so workflow-only PRs satisfy the required CI Status check without running the
@@ -53,19 +53,28 @@ from collections import Counter
 from enum import StrEnum
 from functools import cache
 import json
+import math
 import os
 from pathlib import Path
+import statistics
 import sys
 from typing import Any
 
-from clang_tidy_hash import CLANG_TIDY_GLOBAL_FILES, SDKCONFIG_DEFAULTS_PREFIX
+from clang_tidy_hash import (
+    CLANG_TIDY_GLOBAL_FILES,
+    ESP_IDF_INFRA_TRIGGER_FILES,
+    ESP_IDF_INFRA_TRIGGER_PATH_PREFIXES,
+    SDKCONFIG_DEFAULTS_PREFIX,
+)
 from helpers import (
     CPP_FILE_EXTENSIONS,
     ESPHOME_TESTS_COMPONENTS_PATH,
+    INTEGRATION_TESTS_PATH,
     PYTHON_FILE_EXTENSIONS,
+    all_integration_test_files,
+    base_python_changed,
     changed_files,
     core_changed,
-    filter_component_and_test_cpp_files,
     filter_component_and_test_files,
     get_changed_components,
     get_component_from_path,
@@ -78,6 +87,8 @@ from helpers import (
     get_target_branch,
     git_ls_files,
     is_validate_only_file,
+    load_integration_durations,
+    lpt_partition,
     root_path,
 )
 from split_components_for_ci import create_intelligent_batches
@@ -91,24 +102,24 @@ CLANG_TIDY_SPLIT_THRESHOLD = 65
 # Isolated components count as 10x, groupable components count as 1x
 COMPONENT_TEST_BATCH_SIZE = 40
 
-# Integration test bucketing: when more than the threshold tests are scheduled,
-# fan out across this many parallel jobs. Below the threshold, a single job runs.
+# Above the threshold, fan out across up to this many jobs, balanced by the
+# recorded per-file durations. The target is serial junit-time weight per
+# bucket, not wall time (calibrated with the conftest compile cap); it
+# sizes the bucket count for small subsets.
 INTEGRATION_TESTS_SPLIT_THRESHOLD = 10
-INTEGRATION_TESTS_SPLIT_BUCKETS = 3
+INTEGRATION_TESTS_SPLIT_BUCKETS = 5
+INTEGRATION_TESTS_TARGET_BUCKET_WEIGHT = 360.0
 
-
-def _split_list(items: list[str], n: int) -> list[list[str]]:
-    """Split a list into n roughly-equal contiguous parts (matches script/clang-tidy)."""
-    k, m = divmod(len(items), n)
-    return [items[i * k + min(i, m) : (i + 1) * k + min(i + 1, m)] for i in range(n)]
-
-
-def _all_integration_test_files() -> list[str]:
-    """Return all integration test file paths, sorted, relative to repo root."""
-    return sorted(
-        str(p.relative_to(root_path))
-        for p in (Path(root_path) / "tests" / "integration").glob("test_*.py")
-    )
+# platformio and aioesphomeapi (requirements.txt), the pytest stack
+# (requirements_test.txt) and the fixture every session compiles; a change
+# to any runs the full matrix
+INTEGRATION_TESTS_TRIGGER_FILES = frozenset(
+    {
+        "requirements.txt",
+        "requirements_test.txt",
+        "tests/integration/fixtures/cache_init.yaml",
+    }
+)
 
 
 def _compute_integration_test_buckets(
@@ -117,7 +128,7 @@ def _compute_integration_test_buckets(
 ) -> tuple[bool, list[dict[str, Any]]]:
     """Compute (run_integration, buckets) from the determine_integration_tests result.
 
-    Pure function for unit testing — no I/O beyond `_all_integration_test_files`
+    Pure function for unit testing — no I/O beyond `all_integration_test_files`
     when `integration_run_all` is set.
 
     `buckets` is a list of `{name, tests}` dicts where `tests` is a JSON-friendly
@@ -125,7 +136,7 @@ def _compute_integration_test_buckets(
     shell word-splitting / glob hazards.
     """
     if integration_run_all:
-        files = _all_integration_test_files()
+        files = all_integration_test_files()
     else:
         files = sorted(integration_test_files)
 
@@ -136,12 +147,23 @@ def _compute_integration_test_buckets(
         return False, []
 
     if len(files) > INTEGRATION_TESTS_SPLIT_THRESHOLD:
-        parts = [
-            part for part in _split_list(files, INTEGRATION_TESTS_SPLIT_BUCKETS) if part
-        ]
+        durations = load_integration_durations()
+        # Unrecorded files weigh the recording's median; with no recording a
+        # file weighs a whole bucket, which keeps the full fan-out
+        default = (
+            statistics.median(durations.values())
+            if durations
+            else INTEGRATION_TESTS_TARGET_BUCKET_WEIGHT
+        )
+        weights = {f: durations.get(f, default) for f in files}
+        count = min(
+            INTEGRATION_TESTS_SPLIT_BUCKETS,
+            math.ceil(sum(weights.values()) / INTEGRATION_TESTS_TARGET_BUCKET_WEIGHT),
+        )
+        # count <= SPLIT_BUCKETS < threshold < len(files): no group is empty
+        parts = [sorted(part) for part in lpt_partition(files, weights, count)]
         buckets = [
-            {"name": f"{i + 1}/{len(parts)}", "tests": part}
-            for i, part in enumerate(parts)
+            {"name": f"{i + 1}/{count}", "tests": part} for i, part in enumerate(parts)
         ]
     else:
         buckets = [{"name": "1/1", "tests": files}]
@@ -216,12 +238,15 @@ def determine_integration_tests(branch: str | None = None) -> tuple[bool, list[s
     3. Integration test infrastructure files changed
        - conftest.py, types.py, const.py, entity_utils.py, state_utils.py, etc.
 
+    4. A file in INTEGRATION_TESTS_TRIGGER_FILES changed
+       - The dependency pins and the session init fixture affect every test
+
     Returns (run_all=False, [test_files...]) when:
 
-    4. Specific integration test files changed
+    5. Specific integration test files changed
        - Only those specific test files are returned
 
-    5. Components used by integration tests (or their dependencies) changed
+    6. Components used by integration tests (or their dependencies) changed
        - Only test files whose fixtures use the changed components are returned
 
     Args:
@@ -239,12 +264,15 @@ def determine_integration_tests(branch: str | None = None) -> tuple[bool, list[s
         # If any core files changed, run all integration tests
         return (True, [])
 
+    if any(f in INTEGRATION_TESTS_TRIGGER_FILES for f in files):
+        return (True, [])
+
     # If infrastructure Python files changed (conftest, utils, etc.), run all tests
     # Excludes test files (test_*.py), fixtures, and non-Python files (README.md)
     if any(
-        f.startswith("tests/integration/")
+        f.startswith(INTEGRATION_TESTS_PATH)
         and f.endswith(".py")
-        and not f.startswith("tests/integration/test_")
+        and not f.startswith(f"{INTEGRATION_TESTS_PATH}test_")
         and "/fixtures/" not in f
         for f in files
     ):
@@ -255,9 +283,9 @@ def determine_integration_tests(branch: str | None = None) -> tuple[bool, list[s
     fixture_to_test_files = get_fixture_to_test_files()
 
     for f in files:
-        if f.startswith("tests/integration/test_") and f.endswith(".py"):
+        if f.startswith(f"{INTEGRATION_TESTS_PATH}test_") and f.endswith(".py"):
             test_files.add(f)
-        elif f.startswith("tests/integration/fixtures/"):
+        elif f.startswith(f"{INTEGRATION_TESTS_PATH}fixtures/"):
             if f.endswith(".yaml"):
                 # Fixture YAML changed - add corresponding test file(s)
                 test_files.update(fixture_to_test_files.get(Path(f).stem, ()))
@@ -524,15 +552,6 @@ def _esp32_platformio_path_or_file_trigger(files: list[str]) -> bool:
     return False
 
 
-# ESP-IDF infra: changes under esphome/espidf/ or to the IDF build generator
-# affect every esp32 IDF build (now the default toolchain) but aren't
-# components, so the component matrix wouldn't otherwise force any esp32
-# compile. When they change we fold the `esp32` component into the matrix so
-# the default native-IDF build path is still compiled on an infra-only PR.
-ESP_IDF_INFRA_TRIGGER_PATH_PREFIXES = ("esphome/espidf/",)
-ESP_IDF_INFRA_TRIGGER_FILES = frozenset({"esphome/build_gen/espidf.py"})
-
-
 def _esp_idf_infra_changed(files: list[str]) -> bool:
     """Whether any changed file is ESP-IDF build/runner infrastructure."""
     for file in files:
@@ -619,12 +638,17 @@ def determine_cpp_unit_tests(
 
     C++ unit tests will run when any of the following conditions are met:
 
-    1. Any C++ core source files changed (esphome/core/*), in which case
+    1. Any core C++ or Python files changed (esphome/core/*), in which case
        all cpp unit tests run.
     2. A test file for a component changed, which triggers tests for that
        component.
     3. The code for a component changed, which triggers tests for that
-       component and all components that depend on it.
+       component and all components that depend on it. Python files count
+       too: a component's Python decides which sources and defines go into
+       the host test build, so a Python-only change can break the link.
+
+    Components without C++ test sources are dropped from the list, so the
+    job is only scheduled when there is something to build.
 
     Args:
         branch: Branch to compare against. If None, uses default.
@@ -638,9 +662,7 @@ def determine_cpp_unit_tests(
     if core_changed(files):
         return (True, [])
 
-    # Filter to only C++ files
-    cpp_files = list(filter(filter_component_and_test_cpp_files, files))
-    return (False, get_cpp_changed_components(cpp_files))
+    return (False, get_cpp_changed_components(files))
 
 
 # Paths within tests/benchmarks/ that contain component benchmark files
@@ -657,16 +679,20 @@ BENCHMARK_INFRASTRUCTURE_FILES = frozenset(
 
 
 def should_run_benchmarks(branch: str | None = None) -> bool:
-    """Determine if C++ benchmarks should run based on changed files.
+    """Determine if benchmarks (C++ and Python) should run based on changed files.
 
     Benchmarks run when any of the following conditions are met:
 
-    1. Core C++ files changed (esphome/core/*)
-    2. The host platform changed (esphome/components/host/*) — benchmarks
+    1. Core files changed (esphome/core/*, C++ or Python)
+    2. Top-level Python files changed (esphome/*.py and esphome/*.pyi) —
+       the Python benchmarks exercise config loading (config.py,
+       yaml_util.py, ...), so a slowdown there is invisible unless the
+       benchmarks job runs
+    3. The host platform changed (esphome/components/host/*) — benchmarks
        are built and run on the host platform, so its implementations of
        ``millis()``/``micros()``/etc. affect every benchmark
-    3. A directly changed component has benchmark files (no dependency expansion)
-    4. Benchmark infrastructure changed (tests/benchmarks/*, script/cpp_benchmark.py,
+    4. A directly changed component has benchmark files (no dependency expansion)
+    5. Benchmark infrastructure changed (tests/benchmarks/*, script/cpp_benchmark.py,
        script/build_helpers.py, script/setup_codspeed_lib.py)
 
     Unlike unit tests, benchmarks do NOT expand to dependent components.
@@ -683,6 +709,11 @@ def should_run_benchmarks(branch: str | None = None) -> bool:
     if core_changed(files):
         return True
 
+    # Top-level esphome/*.py modules are what the Python benchmarks in
+    # tests/benchmarks/python/ exercise
+    if base_python_changed(files):
+        return True
+
     # Host platform supplies the runtime that benchmarks execute on
     if any(f.startswith("esphome/components/host/") for f in files):
         return True
@@ -708,7 +739,7 @@ def should_run_benchmarks(branch: str | None = None) -> bool:
 
 
 # Files / path patterns whose changes alone don't warrant running the
-# unconditional CI jobs (`ci-custom`, `pytest`, `pre-commit-ci-lite`).
+# unconditional CI jobs (`ci-custom`, `pytest`, `lint-format`).
 # Single source of truth for what we treat as "CI-irrelevant" on
 # pull_request events; ci.yml used to encode this in its own
 # `pull_request.paths` filter, but that hid the required `CI Status`
@@ -752,7 +783,7 @@ def _is_ci_irrelevant_path(path: str) -> bool:
 
 
 def should_run_core_ci(branch: str | None = None) -> bool:
-    """Determine if the unconditional CI jobs (ci-custom/pytest/pre-commit-ci-lite) should run.
+    """Determine if the unconditional CI jobs (ci-custom/pytest/lint-format) should run.
 
     Returns False only when every changed file is in the CI-irrelevant set
     above (see ``_is_ci_irrelevant_path``). Empty diffs return True so we
@@ -1177,7 +1208,7 @@ def main() -> None:
 
     # Determine what should run
     # core_ci gates the unconditional jobs in ci.yml (ci-custom, pytest,
-    # pre-commit-ci-lite). Non-pull_request events (push to dev/beta/release
+    # lint-format). Non-pull_request events (push to dev/beta/release
     # and merge_group) always run them so behavior like venv-cache saves on
     # push to dev is preserved.
     event_name = os.environ.get("GITHUB_EVENT_NAME", "")
@@ -1390,6 +1421,7 @@ def main() -> None:
     output: dict[str, Any] = {
         "core_ci": run_core_ci,
         "integration_tests": run_integration,
+        "integration_run_all": integration_run_all,
         "integration_test_buckets": integration_test_buckets,
         "clang_tidy": run_clang_tidy,
         "clang_tidy_mode": clang_tidy_mode,

script/git-hooks/post-checkout

@@ -0,0 +1,72 @@
+#!/bin/sh
+# Prepare the dev environment for a new checkout or worktree.
+#
+# Installed into the git hooks directory by script/setup.py. Deliberately tiny
+# and self-contained: it stays valid on branches where the setup script does not
+# exist, and simply does nothing there.
+
+# $3 is 1 for a branch checkout, 0 for a file checkout.
+[ "$3" = "1" ] || exit 0
+
+top=$(git rev-parse --show-toplevel 2>/dev/null) || exit 0
+
+# This also runs on ordinary branch switches, where there is nothing to do. Both
+# layouts are checked because git for Windows runs hooks under its own bundled
+# shell, where the environment lives in venv/Scripts rather than venv/bin.
+[ -x "$top/venv/bin/python" ] && exit 0
+[ -f "$top/venv/Scripts/python.exe" ] && exit 0
+
+# Branches from before the setup script moved to Python carry only the shell
+# entry point, so whichever one the checked out branch has is used.
+py=
+if [ -f "$top/script/setup.py" ]; then
+  # The interpreter goes by different names across platforms, and on Windows
+  # "python3" is often a stub that opens the app store instead of running
+  # anything, so each candidate is tried before it is used. Doing nothing is the
+  # right outcome when none of them work.
+  for candidate in "python3" "python" "py -3"; do
+    # Unquoted on purpose: the launcher candidate is a command plus a flag.
+    if $candidate -c "" >/dev/null 2>&1; then
+      py=$candidate
+      break
+    fi
+  done
+  [ -n "$py" ] || exit 0
+elif ! [ -x "$top/script/setup" ]; then
+  exit 0
+fi
+
+# Every worktree shares the hooks directory of the checkout it was created
+# from, and the setup script run below is the one from whichever branch was just
+# checked out. Older branches install their own pre-commit hook without checking
+# for a worktree: that moves the shared hook aside as pre-commit.legacy and
+# replaces it with one tied to this worktree's virtual environment, so commits
+# break in every checkout. To rule that out, the hooks directory is copied
+# before the setup script runs and put back exactly as it was afterwards,
+# including removing any file the setup script added.
+hooks=$(git rev-parse --path-format=absolute --git-path hooks 2>/dev/null) || exit 0
+snap=$(mktemp -d "$hooks/.post-checkout.XXXXXX") || exit 0
+cp -p "$hooks"/* "$snap"/ 2>/dev/null
+
+# Clear VIRTUAL_ENV so a checkout made from a shell with an environment already
+# activated still gets its own, rather than having the active one repointed at
+# this working tree.
+unset VIRTUAL_ENV
+if [ -n "$py" ]; then
+  # Unquoted on purpose, as above.
+  $py "$top/script/setup.py"
+else
+  "$top/script/setup"
+fi
+status=$?
+
+for f in "$hooks"/*; do
+  [ -e "$snap/${f##*/}" ] || rm -f "$f"
+done
+# Files are moved rather than copied so a hook that is still running, such as
+# this one, is swapped out atomically instead of being rewritten in place.
+for f in "$snap"/*; do
+  cmp -s "$f" "$hooks/${f##*/}" 2>/dev/null || mv -f "$f" "$hooks/${f##*/}"
+done
+rm -rf "$snap"
+exit $status

script/helpers.py

@@ -43,6 +43,53 @@ ESPHOME_TESTS_COMPONENTS_PATH = "tests/components/"
 # Tuple of component and test paths for efficient startswith checks
 COMPONENT_AND_TESTS_PATHS = (ESPHOME_COMPONENTS_PATH, ESPHOME_TESTS_COMPONENTS_PATH)
 
+# Integration tests path prefix
+INTEGRATION_TESTS_PATH = "tests/integration/"
+
+# Per-file integration test durations from CI junit output; shared by the
+# reader (determine-jobs) and writer (update_integration_test_durations)
+INTEGRATION_TEST_DURATIONS_FILE = "tests/integration/integration_test_durations.json"
+
+
+def all_integration_test_files() -> list[str]:
+    """Return all integration test file paths, sorted, relative to repo root."""
+    return sorted(
+        p.relative_to(root_path).as_posix()
+        for p in (Path(root_path) / "tests" / "integration").glob("test_*.py")
+    )
+
+
+def load_integration_durations() -> dict[str, float]:
+    """Return recorded per-file pytest durations in seconds; empty when unavailable."""
+    try:
+        raw = json.loads(
+            (Path(root_path) / INTEGRATION_TEST_DURATIONS_FILE).read_text()
+        )
+        if not isinstance(raw, dict):
+            print(
+                f"integration durations unavailable: expected an object, "
+                f"got {type(raw).__name__}",
+                file=sys.stderr,
+            )
+            return {}
+    except (OSError, ValueError) as err:
+        # The file ships in the repo; degrade to unweighted bucketing, loudly
+        print(f"integration durations unavailable: {err}", file=sys.stderr)
+        return {}
+    durations = {
+        key: seconds
+        for key, value in raw.items()
+        if isinstance(value, (int, float)) and (seconds := float(value)) > 0
+    }
+    if len(durations) != len(raw):
+        # One bad entry must not discard the whole recording
+        print(
+            f"dropped {len(raw) - len(durations)} invalid duration entries",
+            file=sys.stderr,
+        )
+    return durations
+
+
 # Base bus components - these ARE the bus implementations and should not
 # be flagged as needing migration since they are the platform/base components
 BASE_BUS_COMPONENTS = {
@@ -421,7 +468,10 @@ def _get_github_event_data() -> dict | None:
     """
     github_event_path = os.environ.get("GITHUB_EVENT_PATH")
     if github_event_path and Path(github_event_path).exists():
-        with Path(github_event_path).open() as f:
+        # The event payload is UTF-8 JSON; without an explicit encoding
+        # Windows decodes it as cp1252 and any non ASCII byte (an ellipsis in
+        # a commit title is enough) raises UnicodeDecodeError.
+        with Path(github_event_path).open(encoding="utf-8") as f:
             return json.load(f)
     return None
 
@@ -466,6 +516,77 @@ def get_target_branch() -> str | None:
     return None
 
 
+# Substrings (matched case-insensitively against gh's stderr) that identify
+# transient failures worth retrying: server errors (HTTP 5xx) and dropped or
+# failed connections. Permanent failures (bad auth, missing PR, the 300-file
+# diff limit) never match so callers see them immediately. Phrases are
+# anchored so gh's GraphQL "Could not resolve to a PullRequest" (a missing
+# PR) never classifies as a DNS failure.
+_TRANSIENT_GH_ERROR_RE = re.compile(
+    r"http 5\d\d"
+    r"|timed out|timeout"
+    r"|connection (?:reset|refused|closed)"
+    r"|no such host|could not resolve host"
+    # gh intercepts DNS errors and prints its own "error connecting to
+    # <host>" text; the Go phrases above are kept as a hedge in case a
+    # future gh stops swallowing the underlying error
+    r"|error connecting to"
+    r"|failed to verify certificate"
+    # Go reports a server-closed connection as 'Post "<url>": EOF'; the
+    # quote-and-colon anchor keeps a URL or message body containing the
+    # letters from matching
+    r"|unexpected eof"
+    r'|": eof'
+    r"|network is unreachable"
+    r"|temporary failure"
+)
+
+# Same retry policy as git network commands in esphome/git.py: 3 attempts
+# with 2s/4s backoff.
+_GH_MAX_ATTEMPTS = 3
+
+
+def run_gh_command(
+    args: list[str], *, retry: bool = True
+) -> subprocess.CompletedProcess[str]:
+    """Run a gh CLI command, retrying transient network and server failures.
+
+    Args:
+        args: Full command line, including the leading "gh".
+        retry: Pass False for commands that are not idempotent (e.g. posting
+            a comment), where a retry after a dropped response could repeat
+            a write that already succeeded server-side.
+
+    Returns:
+        CompletedProcess with captured text output.
+
+    Raises:
+        subprocess.CalledProcessError: If the command fails with a permanent
+            error, or is still failing after the retries are exhausted.
+    """
+    attempts = _GH_MAX_ATTEMPTS if retry else 1
+    attempt = 0
+    while True:
+        try:
+            return subprocess.run(
+                args, check=True, capture_output=True, text=True, close_fds=False
+            )
+        except subprocess.CalledProcessError as err:
+            attempt += 1
+            stderr = err.stderr or ""
+            if attempt >= attempts or not _TRANSIENT_GH_ERROR_RE.search(stderr.lower()):
+                raise
+            delay = 2**attempt
+            # Only the leading arguments: comment-update calls carry the
+            # whole multi-KB comment body in the argument list
+            print(
+                f"WARNING: {' '.join(args[:3])} failed: {stderr.strip()}; "
+                f"retrying in {delay}s (attempt {attempt}/{attempts})",
+                file=sys.stderr,
+            )
+            time.sleep(delay)
+
+
 @cache
 def _get_changed_files_github_actions() -> list[str] | None:
     """Get changed files in GitHub Actions environment.
@@ -484,8 +605,9 @@ def _get_changed_files_github_actions() -> list[str] | None:
             try:
                 return _get_changed_files_from_command(cmd)
             except Exception as e:
-                # If it fails due to the 300 file limit, use the API method
-                if "maximum" in str(e) and "files" in str(e):
+                # If it fails due to a diff limit (300 files or 20000 lines),
+                # use the API method which only returns filenames
+                if "diff exceeded the maximum" in str(e):
                     cmd = [
                         "gh",
                         "api",
@@ -539,10 +661,22 @@ def changed_files(branch: str | None = None) -> list[str]:
 
 
 def _get_changed_files_from_command(command: list[str]) -> list[str]:
-    """Run a git command to get changed files and return them as a list."""
-    proc = subprocess.run(command, capture_output=True, text=True, check=False)
-    if proc.returncode != 0:
-        raise Exception(f"Command failed: {' '.join(command)}\nstderr: {proc.stderr}")
+    """Run a git or gh command to get changed files and return them as a list."""
+    if command[0] == "gh":
+        try:
+            proc = run_gh_command(command)
+        except subprocess.CalledProcessError as e:
+            raise Exception(
+                f"Command failed: {' '.join(command)}\nstderr: {e.stderr}"
+            ) from e
+    else:
+        proc = subprocess.run(
+            command, capture_output=True, text=True, check=False, close_fds=False
+        )
+        if proc.returncode != 0:
+            raise Exception(
+                f"Command failed: {' '.join(command)}\nstderr: {proc.stderr}"
+            )
 
     changed_files = splitlines_no_ends(proc.stdout)
     cwd = Path.cwd()
@@ -722,17 +856,14 @@ def load_idedata(environment: str) -> dict[str, Any]:
     start_time = time.time()
     print(f"Loading IDE data for environment '{environment}'...")
 
-    # Reuse the clang-tidy input hash as the cache key: it already covers every
-    # file baked into the generated idedata (platformio.ini, sdkconfig.defaults,
-    # esphome/idf_component.yml), so this can't drift from that file list. A
-    # content hash -- unlike an mtime comparison -- stays correct across git
-    # checkouts, which don't preserve mtimes.
-    from clang_tidy_hash import calculate_clang_tidy_hash
+    # Content hash of the idedata inputs (data files and the generator code); a
+    # content hash, unlike mtimes, stays correct across git checkouts.
+    from clang_tidy_hash import idedata_cache_hash
 
     temp_idedata = Path(temp_folder) / f"idedata-{environment}.json"
     temp_hash = Path(temp_folder) / f"idedata-{environment}.hash"
 
-    cache_key = calculate_clang_tidy_hash()
+    cache_key = idedata_cache_hash(environment)
     changed = (
         not temp_idedata.is_file()
         or not temp_hash.is_file()
@@ -973,6 +1104,10 @@ def get_components_per_integration_fixture() -> dict[str, set[str]]:
 
 
 _TEST_FUNC_RE = re.compile(r"async def (test_\w+)")
+# Any usage form (decorator, pytestmark assignment or list element); only
+# test_*.py files are scanned, so the marker docs elsewhere cannot false-hit
+_SHARED_YAML_USE_RE = re.compile(r"\bmark\.shared_yaml")
+_SHARED_YAML_ARG_RE = re.compile(r"\(\s*[\"'](\w+)[\"']\s*\)")
 
 
 @cache
@@ -992,6 +1127,19 @@ def get_fixture_to_test_files() -> dict[str, frozenset[str]]:
         for func in _TEST_FUNC_RE.findall(content):
             base_name = func.replace("test_", "").partition("[")[0]
             result.setdefault(base_name, set()).add(rel_path)
+        # Shared fixtures are named by marker, not by a test function; each
+        # decorator must carry a string literal or its fixture would silently
+        # map to no tests
+        for use in _SHARED_YAML_USE_RE.finditer(content):
+            arg = _SHARED_YAML_ARG_RE.match(content, use.end())
+            if arg is None:
+                line = content.count("\n", 0, use.start()) + 1
+                raise ValueError(
+                    f"{rel_path}:{line}: shared_yaml marker must take a "
+                    "single-line string literal so CI test selection can map "
+                    "its fixture"
+                )
+            result.setdefault(arg.group(1), set()).add(rel_path)
 
     return {k: frozenset(v) for k, v in result.items()}
 
@@ -1063,17 +1211,41 @@ def filter_component_and_test_files(file_path: str) -> bool:
     )
 
 
-def filter_component_and_test_cpp_files(file_path: str) -> bool:
-    """Check if a file is a C++ source file in component or test directories.
+def filter_cpp_unit_test_files(file_path: str) -> bool:
+    """Check if a file can affect a component's C++ unit test build.
+
+    Besides C++ sources, a component's Python code (defines, source file
+    filters, libraries) and the ``__init__.py`` manifest overrides under
+    ``tests/components/<component>/`` decide what the host test binary
+    compiles and links. Other Python files under ``tests/components/``
+    (pytest conftest.py, fixtures) do not.
 
     Args:
         file_path: Path to check
 
     Returns:
-        True if the file is a C++ source/header file in component or test directories
+        True if the file is a C++ or Python file in a component directory, or
+        a C++ file or ``__init__.py`` in a component test directory
     """
-    return file_path.endswith(CPP_FILE_EXTENSIONS) and file_path.startswith(
-        COMPONENT_AND_TESTS_PATHS
+    if file_path.startswith(ESPHOME_COMPONENTS_PATH):
+        return file_path.endswith(CPP_AND_PYTHON_FILE_EXTENSIONS)
+    if file_path.startswith(ESPHOME_TESTS_COMPONENTS_PATH):
+        return file_path.endswith(CPP_FILE_EXTENSIONS) or file_path.endswith(
+            "/__init__.py"
+        )
+    return False
+
+
+def has_cpp_unit_tests(component: str, tests_dir: Path) -> bool:
+    """Check if a component has C++ test or benchmark sources in ``tests_dir``.
+
+    Shared by CI job selection and the build itself
+    (``build_helpers.filter_components_with_files``) so both agree on
+    which components have something to build.
+    """
+    component_dir = tests_dir / component
+    return component_dir.is_dir() and (
+        any(component_dir.glob("*.cpp")) or any(component_dir.glob("*.h"))
     )
 
 
@@ -1377,42 +1549,81 @@ def core_changed(files: list[str]) -> bool:
     )
 
 
+def base_python_changed(files: list[str]) -> bool:
+    """Check if any Python file directly in esphome/ has changed.
+
+    Matches top-level modules and stubs (.py and .pyi) like esphome/config.py
+    and esphome/yaml_util.py but not files in subdirectories such as
+    esphome/components/ or esphome/dashboard/.
+
+    Args:
+        files: List of file paths to check
+
+    Returns:
+        True if any top-level esphome Python file has changed
+    """
+    return any(
+        f.startswith("esphome/")
+        and f.endswith(PYTHON_FILE_EXTENSIONS)
+        and "/" not in f.removeprefix("esphome/")
+        for f in files
+    )
+
+
 def get_cpp_changed_components(files: list[str]) -> list[str]:
-    """Get components that have changed C++ files or tests.
+    """Get components whose C++ unit tests are affected by changed files.
 
     This function analyzes a list of changed files and determines which components
     are affected. It handles two scenarios:
 
-    1. Test files changed (tests/components/<component>/*.cpp):
+    1. Test files changed (tests/components/<component>/*.cpp or __init__.py):
        - Adds the component to the affected list
        - Only that component needs to be tested
 
-    2. Component C++ files changed (esphome/components/<component>/*):
+    2. Component files changed (esphome/components/<component>/*.cpp or *.py):
        - Adds the component to the affected list
        - Also adds all components that depend on this component (recursively)
        - This ensures that changes propagate to dependent components
 
+    Python files count because a component's Python code decides which
+    sources and defines end up in the host test build. Components without
+    C++ test sources are dropped so CI does not schedule the job for nothing.
+
     Args:
-        files: List of file paths to analyze (should be C++ files)
+        files: List of changed file paths; irrelevant ones are ignored
 
     Returns:
         Sorted list of component names that need C++ unit tests run
     """
     components_graph = create_components_graph()
+    tests_dir = Path(root_path) / ESPHOME_TESTS_COMPONENTS_PATH
     affected: set[str] = set()
     for file in files:
-        if not file.endswith(CPP_FILE_EXTENSIONS):
+        if not filter_cpp_unit_test_files(file):
             continue
-        if file.startswith(ESPHOME_TESTS_COMPONENTS_PATH):
-            parts = file.split("/")
-            if len(parts) >= 4:
-                component_dir = Path(ESPHOME_TESTS_COMPONENTS_PATH) / parts[2]
-                if component_dir.is_dir():
-                    affected.add(parts[2])
-        elif file.startswith(ESPHOME_COMPONENTS_PATH):
-            parts = file.split("/")
-            if len(parts) >= 4:
-                component = parts[2]
-                affected.update(find_children_of_component(components_graph, component))
-                affected.add(component)
-    return sorted(affected)
+        parts = file.split("/")
+        if len(parts) < 4:
+            continue
+        component = parts[2]
+        affected.add(component)
+        if file.startswith(ESPHOME_COMPONENTS_PATH):
+            affected.update(find_children_of_component(components_graph, component))
+    return sorted(c for c in affected if has_cpp_unit_tests(c, tests_dir))
+
+
+def lpt_partition(
+    items: list[str], weights: dict[str, float], count: int
+) -> list[list[str]]:
+    """Partition items into `count` weight-balanced groups (LPT greedy).
+
+    Heaviest item first into the lightest group. Ties keep input order, so
+    pass pre-sorted items for deterministic output. script/clang-tidy's
+    split_list is the unweighted contiguous sibling.
+    """
+    groups: list[list[str]] = [[] for _ in range(count)]
+    group_weights = [0.0] * count
+    for item in sorted(items, key=lambda i: -weights[i]):
+        lightest = min(range(count), key=group_weights.__getitem__)
+        groups[lightest].append(item)
+        group_weights[lightest] += weights[item]
+    return groups

script/list-components.py

@@ -3,7 +3,6 @@ import argparse
 
 from helpers import (
     changed_files,
-    filter_component_and_test_cpp_files,
     filter_component_and_test_files,
     get_all_component_files,
     get_components_with_dependencies,
@@ -38,7 +37,7 @@ def main():
     parser.add_argument(
         "--cpp-changed",
         action="store_true",
-        help="List components with changed C++ files",
+        help="List components whose C++ unit tests are affected by changed files",
     )
     args = parser.parse_args()
 
@@ -78,9 +77,9 @@ def main():
         #   Returns: Components with code changes + their dependencies (not infrastructure)
         #   Reason: CI needs to test changed components and their dependents
         #
-        # - --cpp-changed: Used by CI to determine if any C++ files changed (script/determine-jobs.py)
-        #   Returns: Only components with changed C++ files
-        #   Reason: Only components with C++ changes need C++ testing
+        # - --cpp-changed: Mirrors the C++ unit test selection in script/determine-jobs.py
+        #   Returns: Components with changed C++ or Python files (plus dependents)
+        #   Reason: Python decides which sources and defines go into the host test build
 
         base_test_changed = any(
             "tests/test_build_components" in file for file in changed
@@ -115,9 +114,7 @@ def main():
         for c in get_components_with_dependencies(files, False):
             print(c)
     elif args.cpp_changed:
-        # Only look at changed cpp files
-        files = list(filter(filter_component_and_test_cpp_files, changed))
-        for c in get_cpp_changed_components(files):
+        for c in get_cpp_changed_components(changed):
             print(c)
     else:
         # Return all changed components (with dependencies) - default behavior

script/platformio_install_deps.py

@@ -3,58 +3,375 @@
 # all platformio libraries in the global storage
 
 import argparse
+from concurrent.futures import ThreadPoolExecutor
 import configparser
+from contextlib import suppress
+import os
+from pathlib import Path
+import queue
 import subprocess
+import threading
+import traceback
 
-config = configparser.ConfigParser(inline_comment_prefixes=(";",))
+# esphome is not installed at this docker layer; pio's fs.rmtree is the
+# same chmod-on-readonly shape its own installer uses
+try:
+    from platformio import fs
+    from platformio.cache import ContentCache
+    from platformio.package.manager.base import BasePackageManager
+    from platformio.package.manager.library import LibraryPackageManager
+    from platformio.package.manager.tool import ToolPackageManager
+    from platformio.package.meta import PackageCompatibility
 
-parser = argparse.ArgumentParser(description="")
-parser.add_argument("file", help="Path to platformio.ini", nargs=1)
-parser.add_argument("-l", "--libraries", help="Install libraries", action="store_true")
-parser.add_argument("-p", "--platforms", help="Install platforms", action="store_true")
-parser.add_argument("-t", "--tools", help="Install tools", action="store_true")
+    PARALLEL_AVAILABLE = True
+except ImportError as err:  # pragma: no cover
+    # A moved pio module must degrade to the serial pass, not kill the
+    # image build; the tripwire test makes the drift loud in CI
+    PARALLEL_AVAILABLE = False
+    IMPORT_ERROR = repr(err)
 
-args = parser.parse_args()
-
-config.read(args.file)
+# Network-bound downloads release the GIL, so the pool oversubscribes
+# the cores. This bypasses pio's 500ms registry throttle and races its
+# self-unlinking cache LockFiles; both are cache-only and self-healing.
+MAX_WORKERS = 16
 
 
-libs = []
-tools = []
-platforms = []
-# Extract from every lib_deps key in all sections
-for section in config.sections():
-    conf = config[section]
-    if "lib_deps" in conf and args.libraries:
-        for lib_dep in conf["lib_deps"].splitlines():
-            if not lib_dep:
-                # Empty line or comment
-                continue
-            if lib_dep.startswith("${"):
-                # Extending from another section
-                continue
-            if "@" not in lib_dep:
-                # No version pinned, this is an internal lib
-                continue
-            libs.append("-l")
-            libs.append(lib_dep)
-    if "platform" in conf and args.platforms:
-        platforms.append("-p")
-        platforms.append(conf["platform"])
-    if "platform_packages" in conf and args.tools:
-        for tool in conf["platform_packages"].splitlines():
-            if not tool:
-                # Empty line or comment
-                continue
-            if tool.startswith("${"):
-                # Extending from another section
-                continue
-            if tool.find("https://github.com") != -1:
-                split = tool.find("@")
-                tool = tool[split + 1 :]
-            tools.append("-t")
-            tools.append(tool)
+class CleanupError(RuntimeError):
+    """A torn destination could not be removed; the serial pass would
+    trust it, so the build must fail rather than bake a corrupt image."""
 
-subprocess.check_call(
-    ["platformio", "pkg", "install", "-g", *libs, *platforms, *tools], close_fds=False
-)
+
+class LockReleaseError(RuntimeError):
+    """The manager lock could not be released; the serial pass would
+    block on it, so the build must fail with the cause named."""
+
+
+def parse_specs(path: str, args: argparse.Namespace) -> tuple[list, list, list]:
+    """Extract lib/platform/tool specs from every section of a platformio.ini."""
+    config = configparser.ConfigParser(inline_comment_prefixes=(";",))
+    if not config.read(path):
+        # ConfigParser silently ignores unreadable files; an empty spec
+        # list would build an image with no dependencies at all
+        raise SystemExit(f"Could not read {path}")
+    libs = []
+    tools = []
+    platforms = []
+    for section in config.sections():
+        conf = config[section]
+        if "lib_deps" in conf and args.libraries:
+            for lib_dep in conf["lib_deps"].splitlines():
+                if not lib_dep:
+                    # Empty line or comment
+                    continue
+                if lib_dep.startswith("${"):
+                    # Extending from another section
+                    continue
+                if "@" not in lib_dep:
+                    # No version pinned, this is an internal lib
+                    continue
+                libs.append(lib_dep)
+        if "platform" in conf and args.platforms:
+            platforms.append(conf["platform"])
+        if "platform_packages" in conf and args.tools:
+            for tool in conf["platform_packages"].splitlines():
+                if not tool:
+                    # Empty line or comment
+                    continue
+                if tool.startswith("${"):
+                    # Extending from another section
+                    continue
+                if tool.find("https://github.com") != -1:
+                    split = tool.find("@")
+                    tool = tool[split + 1 :]
+                tools.append(tool)
+    # Exact-string dedupe only: name-level dedupe would change which
+    # version conflicts the pkg install pass reconciles
+    return (
+        list(dict.fromkeys(libs)),
+        list(dict.fromkeys(platforms)),
+        list(dict.fromkeys(tools)),
+    )
+
+
+def piopm_matches(package_dir: str, spec) -> list[Path]:
+    """Dirs whose .piopm metadata names this spec; a positive match beats
+    guessing the manifest-derived dirname from the registry name."""
+    want = (BasePackageManager.ensure_spec(spec).name or "").lower()
+    matches: list[Path] = []
+    if not want:
+        return matches
+    try:
+        entries = list(Path(package_dir).iterdir())
+    except FileNotFoundError:
+        return matches
+    for d in entries:
+        if not d.is_dir():
+            continue  # pio's get_installed skips files and *.pio-link too
+        try:
+            meta = fs.load_json(str(d / ".piopm"))
+        except FileNotFoundError:
+            continue  # no metadata means pio does not trust it either
+        except (OSError, ValueError):
+            if d.name.lower() == want:
+                # A corrupt .piopm under this spec's own name would crash
+                # pio's whole storage scan; remove it
+                matches.append(d)
+            continue
+        mspec = meta.get("spec") or {}
+        if (mspec.get("name") or meta.get("name") or "").lower() == want:
+            matches.append(d)
+    return matches
+
+
+def remove_dir(spec, dest: Path) -> None:
+    # fs.rmtree never raises (errors go to a printing onexc handler);
+    # only the destination's absence proves the cleanup worked
+    fs.rmtree(str(dest))
+    if dest.exists():
+        # Failing the build beats baking a corrupt image
+        raise CleanupError(
+            f"could not remove the failed pre-install of {spec} at {dest}"
+        )
+    print(f"Removed torn destination {dest}", flush=True)
+
+
+def cleanup_or_die(mgr, spec) -> None:
+    """Cleanup that did not demonstrably succeed must fail the build."""
+    try:
+        clean_torn(mgr, spec)
+    except CleanupError:
+        raise
+    except Exception as err:  # noqa: BLE001
+        raise CleanupError(f"cleanup failed for {spec}: {err!r}") from err
+
+
+def clean_torn(mgr, spec) -> None:
+    """Remove a torn destination so the serial pass cannot trust it."""
+    pkg = None
+    with suppress(Exception):
+        # get_package memoizes a pre-install snapshot; reset to see the
+        # torn dir. It also recognizes manifest-only legacy dirs pio's
+        # storage scan would trust, which the .piopm fallback cannot see.
+        mgr.memcache_reset()
+        pkg = mgr.get_package(spec)
+    if pkg is not None:
+        remove_dir(spec, Path(pkg.path))
+    elif dests := piopm_matches(mgr.package_dir, spec):
+        # A .piopm naming this spec is the exact shape the serial pass
+        # trusts; a dir without one is overwritten by pio's own install
+        for dest in dests:
+            remove_dir(spec, dest)
+    else:
+        print(f"No resolvable destination to clean for {spec}", flush=True)
+
+
+def spec_key(spec) -> str | None:
+    """The destination identity of a spec: PlatformIO installs by package
+    name, so two specs sharing a name share a directory. ``None`` means
+    the name could not be derived; such a spec must stay out of the wave
+    (a raw-string key would break the one-per-destination guarantee)."""
+    name = BasePackageManager.ensure_spec(spec).name
+    return name.lower() if name else None
+
+
+def dependency_specs(manager, specs: list) -> list:
+    """``(spec, compatibility)`` registry dependencies of installed
+    packages, from local manifest reads. Name-only dependencies
+    (platform-bundled libs like SPI) stay with the ``pkg install`` pass;
+    the compatibility qualifiers mirror pio's install_dependency, so a
+    qualified dep resolves to the same package the serial pass picks."""
+    return [
+        (manager.dependency_to_spec(dep), PackageCompatibility.from_dependency(dep))
+        for spec in specs
+        if (pkg := manager.get_package(spec)) is not None
+        for dep in manager.get_pkg_dependencies(pkg) or []
+        if dep.get("owner") or dep.get("version")
+    ]
+
+
+def parallel_install(manager_cls, specs: list, prior_names: set | None = None) -> None:
+    """Best-effort parallel top-level install.
+
+    PlatformIO's own installer downloads and unpacks one package at a time
+    on one core. Dependencies are skipped (two packages sharing one must
+    not extract into the same directory from two threads) and failures are
+    only reported: the stock ``pkg install`` pass afterwards installs
+    whatever is missing and is the authority on the final state.
+    """
+    if not specs:
+        return
+    manager = manager_cls(None)
+    # One spec per destination: two threads must not extract into the
+    # same directory. Second versions of a name and URL specs (their dir
+    # comes from the archive manifest) stay with the pkg install pass.
+    seen_names: set = prior_names if prior_names is not None else set()
+    # Wave-1 items are strings; dependency waves carry (spec, compatibility)
+    pairs = [item if isinstance(item, tuple) else (item, None) for item in specs]
+    unique = {}
+    for spec, compat in pairs:
+        # Normalize once: a dependency's URL version surfaces as spec.uri
+        parsed = BasePackageManager.ensure_spec(spec)
+        if parsed.uri:
+            continue
+        if (key := spec_key(parsed)) is None:
+            # No name, no destination identity; leave it to the serial pass
+            print(f"Skipping unresolvable spec {spec!r} in the wave", flush=True)
+            continue
+        unique.setdefault(key, (spec, compat))  # first-wins, like pio's walk
+    pending = [
+        (spec, compat)
+        for spec, compat in unique.values()
+        if not manager.get_package(spec)
+    ]
+    if not pending:
+        # Nothing to install, but a warm store's dependencies must still
+        # feed the next wave (a transitive dep may be missing)
+        _next_wave(manager_cls, manager, unique, seen_names)
+        return
+    workers = min(len(pending), MAX_WORKERS)
+    # One manager per worker (_install mutates instance state); built
+    # serially because construction rewires the shared manager logger
+    managers: queue.SimpleQueue = queue.SimpleQueue()
+    for _ in range(workers):
+        managers.put(manager_cls(None))
+    local = threading.local()
+
+    def install_one(item) -> bool:
+        spec, compat = item
+        if (mgr := getattr(local, "mgr", None)) is None:
+            mgr = local.mgr = managers.get_nowait()
+        try:
+            mgr._install(  # noqa: SLF001
+                spec, skip_dependencies=True, compatibility=compat
+            )
+            return True
+        except Exception as err:  # noqa: BLE001
+            print(f"Pre-install of {spec} failed ({err!r})", flush=True)
+            cleanup_or_die(mgr, spec)
+            return False
+        except BaseException:
+            # A worker SystemExit (main() guards against it) must not skip
+            # the cleanup and leave a torn dir the serial pass trusts
+            cleanup_or_die(mgr, spec)
+            raise
+
+    print(f"Preinstalling {len(pending)} package(s) with {workers} workers", flush=True)
+    # The serial getter calls create pio's lazy dirs (made without
+    # exist_ok) before cold-cache workers can race the creation
+    manager.get_download_dir()
+    manager.get_tmp_dir()
+    ContentCache("http")
+    cwd = Path.cwd()
+    manager.lock()
+    try:
+        with ThreadPoolExecutor(max_workers=workers) as ex:
+            futures = [ex.submit(install_one, item) for item in pending]
+        # The with-block joined every future; drain them all so a
+        # concurrent CleanupError is never dropped
+        errors = [err for f in futures if (err := f.exception()) is not None]
+        for err in errors:
+            # Every failure is on the record; the raised one is a summary
+            print(f"Wave failure: {err!r}", flush=True)
+        if errors:
+            raise next((e for e in errors if isinstance(e, CleanupError)), errors[0])
+        results = [f.result() for f in futures]
+    finally:
+        try:
+            manager.unlock()
+        except Exception as unlock_err:  # noqa: BLE001
+            # A held flock would hang the serial pass in another process;
+            # failing loudly beats an unexplained stuck docker build. Any
+            # in-flight error stays attached as the context.
+            raise LockReleaseError(
+                f"could not release the manager lock: {unlock_err!r}"
+            ) from unlock_err
+        # Worker postinstall scripts chdir process-wide (pio's fs.cd);
+        # restore between waves. The serial pass pins its own cwd.
+        with suppress(OSError):
+            os.chdir(cwd)
+    if failures := len(results) - sum(results):
+        # The stock pass retries CLI specs and re-walks installed
+        # packages' dependencies, so failed deps retry too
+        print(
+            f"Pre-install failed for {failures} of {len(results)} package(s); "
+            "pkg install retries them serially",
+            flush=True,
+        )
+
+    # Waves skip dependencies (a shared one must not extract from two
+    # threads); the installed manifests feed the next wave
+    _next_wave(manager_cls, manager, unique, seen_names)
+
+
+def _next_wave(manager_cls, manager, unique: dict, seen_names: set) -> None:
+    """Queue the dependency wave for every requested spec, installed or
+    freshly waved; a warm store can still be missing a transitive dep.
+    Terminates without a cap: each wave admits only never-seen names."""
+    seen_names.update(unique)
+    # The pre-wave get_package calls memoized an empty storage snapshot
+    manager.memcache_reset()
+    next_specs = [
+        item
+        for item in dependency_specs(manager, [spec for spec, _ in unique.values()])
+        if spec_key(item[0]) not in seen_names
+    ]
+    if next_specs:
+        parallel_install(manager_cls, next_specs, seen_names)
+
+
+def build_cli_args(libs: list, platforms: list, tools: list) -> list:
+    return [
+        arg
+        for flag, specs in (("-l", libs), ("-p", platforms), ("-t", tools))
+        for spec in specs
+        for arg in (flag, spec)
+    ]
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description="")
+    parser.add_argument("file", help="Path to platformio.ini", nargs=1)
+    parser.add_argument(
+        "-l", "--libraries", help="Install libraries", action="store_true"
+    )
+    parser.add_argument(
+        "-p", "--platforms", help="Install platforms", action="store_true"
+    )
+    parser.add_argument("-t", "--tools", help="Install tools", action="store_true")
+    args = parser.parse_args()
+    start_cwd = Path.cwd()
+    libs, platforms, tools = parse_specs(args.file[0], args)
+
+    # Platforms stay serial: PlatformPackageManager.install runs an
+    # on_installed hook the private _install path would skip
+    if PARALLEL_AVAILABLE:
+        wave_groups = [(ToolPackageManager, tools), (LibraryPackageManager, libs)]
+    else:  # pragma: no cover
+        wave_groups = []
+        print(
+            f"PlatformIO layout changed ({IMPORT_ERROR}); serial install only",
+            flush=True,
+        )
+    for manager_cls, specs in wave_groups:
+        try:
+            parallel_install(manager_cls, specs)
+        except (CleanupError, LockReleaseError, KeyboardInterrupt):
+            # A torn package or a held lock must fail the build
+            raise
+        except BaseException:  # noqa: BLE001
+            # BaseException: a worker postinstall's SystemExit must not
+            # skip the authoritative serial pass (partial deps, exit 0)
+            print("Parallel preinstall failed, falling back to serial", flush=True)
+            traceback.print_exc()
+
+    # Postinstall scripts chdir process-wide (pio's fs.cd captures its
+    # restore path at construction); pin the authoritative pass's cwd
+    subprocess.check_call(
+        ["platformio", "pkg", "install", "-g", *build_cli_args(libs, platforms, tools)],
+        close_fds=False,
+        cwd=start_cwd,
+    )
+
+
+if __name__ == "__main__":
+    main()

script/setup

@@ -1,40 +1,7 @@
 #!/usr/bin/env bash
-# Set up ESPHome dev environment
+# Set up ESPHome dev environment.
+#
+# The work is done by setup.py, which script/setup.bat also runs, so the Unix
+# and Windows entry points share one implementation.
 
-set -e
-
-cd "$(dirname "$0")/.."
-if [ -n "$VIRTUAL_ENV" ]; then
-  # A virtual environment is already active (e.g. the devcontainer's pre-provisioned
-  # esphome-venv). Install into it rather than creating a ./venv in the workspace.
-  created_venv=false
-else
-  created_venv=true
-  if [ -x "$(command -v uv)" ]; then
-    uv venv --seed venv
-  else
-    python3 -m venv venv
-  fi
-  source venv/bin/activate
-fi
-
-if ! [ -x "$(command -v uv)" ]; then
-  python3 -m pip install uv
-fi
-
-uv pip install setuptools wheel
-uv pip install -e ".[dev,test]" --config-settings editable_mode=compat
-
-pre-commit install
-
-mkdir -p .temp
-
-echo
-echo
-if [ "$created_venv" = true ]; then
-  echo "Virtual environment created at ./venv. Run 'source venv/bin/activate' to use it."
-else
-  echo "Dependencies installed into the active virtual environment:"
-  echo "  $VIRTUAL_ENV"
-  echo "It is already active in this shell, so no 'source venv/bin/activate' is needed."
-fi
+exec python3 "$(dirname "$0")/setup.py" "$@"

script/setup.bat

@@ -1,24 +1 @@
-@echo off
-
-if defined VIRTUAL_ENV goto :install
-
-echo Starting the Virtual Environment
-python -m venv venv
-call venv/Scripts/activate
-echo Running the Virtual Environment
-
-:install
-
-echo Installing required packages...
-
-python.exe -m pip install --upgrade pip
-
-pip3 install -r requirements.txt -r requirements_test.txt -r requirements_dev.txt
-pip3 install setuptools wheel
-pip3 install -e ".[dev,test]" --config-settings editable_mode=compat
-
-pre-commit install
-
-echo .
-echo .
-echo Virtual environment created. Run 'venv/Scripts/activate' to use it.
+@python "%~dp0setup.py" %*

script/setup.py

@@ -0,0 +1,222 @@
+#!/usr/bin/env python3
+"""Set up the ESPHome development environment.
+
+Shared implementation behind script/setup and script/setup.bat, so the Unix and
+Windows entry points cannot drift apart. Uses only the standard library: it runs
+before any dependency has been installed.
+"""
+
+import os
+from pathlib import Path
+import shutil
+import subprocess
+import sys
+import sysconfig
+
+MIN_PYTHON = (3, 12)
+
+ROOT = Path(__file__).resolve().parent.parent
+DEFAULT_VENV = ROOT / "venv"
+POST_CHECKOUT_HOOK = ROOT / "script" / "git-hooks" / "post-checkout"
+
+# State of the environment the dependencies end up in, used for the closing
+# message.
+VENV_ACTIVE = "active"
+VENV_REUSED = "reused"
+VENV_CREATED = "created"
+
+
+def bin_dir(venv: Path) -> Path:
+    """Return the directory holding a virtual environment's executables.
+
+    The "venv" scheme resolves to bin on Unix and Scripts on Windows, so the
+    layout does not have to be hardcoded here.
+    """
+    base = str(venv)
+    return Path(
+        sysconfig.get_path("scripts", "venv", vars={"base": base, "platbase": base})
+    )
+
+
+def venv_python(venv: Path) -> Path:
+    """Return the path to a virtual environment's interpreter."""
+    name = "python.exe" if os.name == "nt" else "python"
+    return bin_dir(venv) / name
+
+
+def run(command: list[str], env: dict[str, str] | None = None) -> None:
+    """Run a command, aborting the whole script if it fails."""
+    print(f"+ {' '.join(command)}", flush=True)
+    result = subprocess.run(command, cwd=ROOT, env=env, check=False)
+    if result.returncode != 0:
+        # Some tools fail without printing anything, so name the step that broke.
+        print(
+            f"Failed with exit code {result.returncode}: {command[0]}", file=sys.stderr
+        )
+        raise SystemExit(result.returncode)
+
+
+def git_output(*args: str) -> str:
+    """Return the trimmed output of a git command, or "" if it cannot be run."""
+    try:
+        result = subprocess.run(
+            ["git", *args], cwd=ROOT, capture_output=True, text=True, check=False
+        )
+    except OSError:
+        # Git is not required to install the dependencies, only to install hooks.
+        return ""
+    if result.returncode != 0:
+        return ""
+    return result.stdout.strip()
+
+
+def create_venv(venv: Path) -> None:
+    """Create a virtual environment, replacing anything already at the path."""
+    # --clear replaces a partial environment left behind by an interrupted run.
+    if (uv := shutil.which("uv")) is not None:
+        run([uv, "venv", "--clear", "--seed", str(venv)])
+    else:
+        run([sys.executable, "-m", "venv", "--clear", str(venv)])
+
+
+def venv_environment(venv: Path) -> dict[str, str]:
+    """Return the environment child processes need to target a virtual env.
+
+    Equivalent to sourcing the environment's activate script: tools such as uv
+    and prek pick the environment up from VIRTUAL_ENV and PATH.
+    """
+    env = dict(os.environ)
+    env["VIRTUAL_ENV"] = str(venv)
+    env.pop("PYTHONHOME", None)
+    path = str(bin_dir(venv))
+    # An empty entry would be appended if PATH is unset, and on Unix that means
+    # the working directory is searched for executables.
+    if existing := env.get("PATH"):
+        path = os.pathsep.join([path, existing])
+    env["PATH"] = path
+    return env
+
+
+def find_uv(venv: Path, env: dict[str, str]) -> str:
+    """Return the path to uv, installing it into the environment if needed."""
+    if (uv := shutil.which("uv", path=env["PATH"])) is not None:
+        return uv
+    run([str(venv_python(venv)), "-m", "pip", "install", "uv"], env=env)
+    if (uv := shutil.which("uv", path=env["PATH"])) is not None:
+        return uv
+    raise SystemExit("uv could not be installed, aborting.")
+
+
+def install_dependencies(venv: Path, env: dict[str, str]) -> None:
+    """Install ESPHome and its development dependencies into the environment."""
+    uv = find_uv(venv, env)
+    run([uv, "pip", "install", "setuptools", "wheel"], env=env)
+    # The dev and test extras pull in requirements_dev.txt and
+    # requirements_test.txt, and the package itself pulls in requirements.txt,
+    # so this single install covers every requirements file.
+    run(
+        [
+            uv,
+            "pip",
+            "install",
+            "-e",
+            ".[dev,test]",
+            "--config-settings",
+            "editable_mode=compat",
+        ],
+        env=env,
+    )
+
+
+def install_git_hooks(env: dict[str, str]) -> None:
+    """Install the git hooks, but only when run from the main checkout.
+
+    A worktree shares one git hooks directory with the main checkout it was
+    created from. Installing from a worktree would point the shared hook at that
+    worktree's virtual environment, breaking it for everyone once the worktree is
+    removed.
+    """
+    git_dir = git_output("rev-parse", "--absolute-git-dir")
+    common_dir = git_output("rev-parse", "--path-format=absolute", "--git-common-dir")
+    if not git_dir or not common_dir or Path(git_dir) != Path(common_dir):
+        return
+
+    prek = shutil.which("prek", path=env["PATH"])
+    if prek is None:
+        raise SystemExit("prek was not installed, aborting.")
+    # --overwrite replaces any hook already in place. Without it, prek finds a
+    # previously installed pre-commit hook, moves it aside to
+    # .git/hooks/pre-commit.legacy and keeps calling it, so every commit would
+    # run both tools.
+    run([prek, "install", "--overwrite"], env=env)
+
+    # Prepares the virtual environment for new checkouts and worktrees. Installed
+    # once here, it covers every worktree created from this checkout.
+    hooks_dir = Path(common_dir) / "hooks"
+    if hooks_dir.is_dir():
+        installed = hooks_dir / "post-checkout"
+        shutil.copyfile(POST_CHECKOUT_HOOK, installed)
+        installed.chmod(0o755)
+
+
+def activate_hint() -> str:
+    """Return the command that activates the environment this script creates."""
+    activate = bin_dir(DEFAULT_VENV).relative_to(ROOT) / "activate"
+    if os.name == "nt":
+        return str(activate)
+    return f"source {activate.as_posix()}"
+
+
+def report(state: str, venv: Path) -> None:
+    """Print the closing message for the environment that was set up."""
+    location = f"./{DEFAULT_VENV.name}"
+    print()
+    print()
+    if state == VENV_ACTIVE:
+        print("Dependencies installed into the active virtual environment:")
+        print(f"  {venv}")
+        print(
+            f"It is already active in this shell, so no '{activate_hint()}' is needed."
+        )
+    elif state == VENV_REUSED:
+        print(
+            f"Dependencies updated in the existing {location}. "
+            f"Run '{activate_hint()}' to use it."
+        )
+    else:
+        print(
+            f"Virtual environment created at {location}. "
+            f"Run '{activate_hint()}' to use it."
+        )
+
+
+def main() -> None:
+    """Set up the development environment."""
+    if sys.version_info < MIN_PYTHON:
+        raise SystemExit(
+            f"ESPHome needs Python {MIN_PYTHON[0]}.{MIN_PYTHON[1]} or newer, "
+            f"but this is Python {sys.version.split()[0]}."
+        )
+
+    # A virtual environment that is already active (for example the
+    # devcontainer's pre-provisioned esphome-venv) is installed into rather than
+    # creating a ./venv in the workspace.
+    if active := os.environ.get("VIRTUAL_ENV"):
+        state, venv = VENV_ACTIVE, Path(active)
+    elif venv_python(DEFAULT_VENV).is_file():
+        # Reuse the environment from an earlier run, so this script can be run
+        # again at any time to pick up dependency changes.
+        state, venv = VENV_REUSED, DEFAULT_VENV
+    else:
+        state, venv = VENV_CREATED, DEFAULT_VENV
+        create_venv(venv)
+
+    env = venv_environment(venv)
+    install_dependencies(venv, env)
+    install_git_hooks(env)
+    (ROOT / ".temp").mkdir(exist_ok=True)
+    report(state, venv)
+
+
+if __name__ == "__main__":
+    main()

script/setup_codspeed_lib.py

@@ -84,7 +84,7 @@ def _read_codspeed_version(cmake_path: Path) -> str:
     """Extract CODSPEED_VERSION from core/CMakeLists.txt."""
     if not cmake_path.exists():
         return "0.0.0"
-    for line in cmake_path.read_text().splitlines():
+    for line in cmake_path.read_text(encoding="utf-8").splitlines():
         if line.startswith("set(CODSPEED_VERSION"):
             return line.split()[1].rstrip(")")
     return "0.0.0"

script/sync_dependency_versions.py

@@ -0,0 +1,164 @@
+#!/usr/bin/env python3
+"""Keep pre-commit hook revs in sync with the requirements files.
+
+Dependabot only bumps the ``package==version`` pins in ``requirements*.txt``.
+Some of those tools are pinned a second time as hook ``rev`` values in
+``.pre-commit-config.yaml``. This script treats the requirements files as
+the source of truth and rewrites the revs to match, editing the config
+through yamlrocks so comments and layout survive.
+
+Run without arguments to apply the changes in place, or with ``--check`` to
+only report drift (exit status 1 when anything is out of sync).
+"""
+
+from __future__ import annotations
+
+import argparse
+from dataclasses import dataclass
+from pathlib import Path
+import re
+import sys
+from typing import Any
+
+import yamlrocks
+
+REPO_ROOT = Path(__file__).resolve().parent.parent
+PRECOMMIT_CONFIG = ".pre-commit-config.yaml"
+
+
+class SyncError(Exception):
+    """A pin could not be located in a requirements file or the config."""
+
+
+@dataclass(frozen=True)
+class SyncTarget:
+    """A requirements pin and the pre-commit repo whose rev mirrors it."""
+
+    package: str
+    requirements_file: str
+    repo: str
+
+
+SYNC_TARGETS: tuple[SyncTarget, ...] = (
+    SyncTarget(
+        "ruff", "requirements_test.txt", "https://github.com/astral-sh/ruff-pre-commit"
+    ),
+    SyncTarget("flake8", "requirements_test.txt", "https://github.com/PyCQA/flake8"),
+    SyncTarget(
+        "pyupgrade", "requirements_test.txt", "https://github.com/asottile/pyupgrade"
+    ),
+    SyncTarget(
+        "clang-format",
+        "requirements_dev.txt",
+        "https://github.com/pre-commit/mirrors-clang-format",
+    ),
+    SyncTarget(
+        "yamllint",
+        "requirements_dev.txt",
+        "https://github.com/adrienverge/yamllint.git",
+    ),
+)
+
+
+def read_requirement_version(requirements: str, package: str) -> str | None:
+    """Return the ``==`` pin for ``package`` or None when it is not pinned."""
+    pattern = re.compile(
+        rf"^{re.escape(package)}==(?P<version>[^\s#]+)",
+        re.MULTILINE | re.IGNORECASE,
+    )
+    match = pattern.search(requirements)
+    return match.group("version") if match else None
+
+
+def find_repo_entry(doc: Any, repo: str) -> Any:
+    """Return the single ``- repo:`` block for ``repo`` in a pre-commit doc."""
+    try:
+        entries = [entry for entry in doc["repos"] if entry["repo"] == repo]
+    except KeyError as err:
+        raise SyncError(f"malformed pre-commit config, missing key {err}") from None
+    if len(entries) != 1:
+        raise SyncError(
+            f"expected exactly one block for repo {repo}, found {len(entries)}"
+        )
+    return entries[0]
+
+
+def current_rev(entry: Any, repo: str) -> tuple[str, str]:
+    """Split the block's rev into its tag prefix (``v`` or empty) and version."""
+    if "rev" not in entry:
+        raise SyncError(f"repo {repo} has no rev")
+    rev = entry["rev"]
+    if not isinstance(rev, str):
+        # A rev such as ``1.0`` parses as a number and cannot be compared or
+        # rewritten safely; quote it in the config instead.
+        raise SyncError(f"rev of repo {repo} is not a string: {rev!r}")
+    prefix = "v" if rev.startswith("v") else ""
+    return prefix, rev.removeprefix("v")
+
+
+def sync(root: Path, *, write: bool) -> list[str]:
+    """Bring every hook rev in line with its requirements pin.
+
+    Returns one description per rev that was (or, when ``write`` is False,
+    would be) changed. Raises SyncError when a pin cannot be found, which
+    means SYNC_TARGETS has gone stale and needs updating by hand.
+    """
+    config_path = root / PRECOMMIT_CONFIG
+    doc = yamlrocks.loads(config_path.read_bytes(), option=yamlrocks.OPT_ROUND_TRIP)
+    requirements: dict[str, str] = {}
+    changes: list[str] = []
+    for target in SYNC_TARGETS:
+        if target.requirements_file not in requirements:
+            requirements[target.requirements_file] = (
+                root / target.requirements_file
+            ).read_text()
+        version = read_requirement_version(
+            requirements[target.requirements_file], target.package
+        )
+        if version is None:
+            raise SyncError(
+                f"{target.requirements_file}: no '{target.package}==' pin found"
+            )
+
+        entry = find_repo_entry(doc, target.repo)
+        prefix, current = current_rev(entry, target.repo)
+        if current == version:
+            continue
+        changes.append(f"{target.package}: {current} -> {version}")
+        entry["rev"] = f"{prefix}{version}"
+
+    if changes and write:
+        config_path.write_bytes(doc.to_yaml())
+    return changes
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
+    parser.add_argument(
+        "--check",
+        action="store_true",
+        help="report drift without modifying any file; exit 1 if out of sync",
+    )
+    parser.add_argument(
+        "--root",
+        type=Path,
+        default=REPO_ROOT,
+        help="repository checkout to operate on (default: this checkout)",
+    )
+    args = parser.parse_args(argv)
+
+    try:
+        changes = sync(args.root, write=not args.check)
+    except SyncError as err:
+        print(f"error: {err}", file=sys.stderr)
+        return 1
+
+    for change in changes:
+        print(change)
+    if args.check and changes:
+        return 1
+    return 0
+
+
+if __name__ == "__main__":  # pragma: no cover
+    sys.exit(main())

script/test_build_components.py

@@ -367,7 +367,7 @@ def run_esphome_test(
     output_file = build_dir / f"{component}.{test_name}.{platform_with_version}.yaml"
 
     # Copy base file and substitute component test file reference
-    base_content = base_file.read_text()
+    base_content = base_file.read_text(encoding="utf-8")
     # Get relative path from build dir to test file
     repo_root = Path(__file__).parent.parent
     component_test_ref = f"../../{test_file.relative_to(repo_root / 'tests')}"
@@ -524,7 +524,7 @@ def run_grouped_test(
 
     # Create test file that includes merged config
     output_file = build_dir / f"test_{group_name}.{platform_with_version}.yaml"
-    base_content = base_file.read_text()
+    base_content = base_file.read_text(encoding="utf-8")
     merged_ref = merged_config_file.name
     output_content = base_content.replace("$component_test_file", merged_ref)
     output_file.write_text(output_content)

script/update_integration_test_durations.py

@@ -0,0 +1,119 @@
+#!/usr/bin/env python3
+"""Merge CI junit output into tests/integration/integration_test_durations.json.
+
+The integration-tests CI job uploads one junit XML artifact per bucket on
+full matrix dev runs. Download a run's artifacts and merge the per file
+durations into the recording used by script/determine-jobs.py:
+
+    gh run download <run-id> --repo esphome/esphome -p "junit-integration-*" -D /tmp/junit
+    script/update_integration_test_durations.py /tmp/junit
+
+Missing files keep their previous recording and deleted files drop out; a
+run covering under 90% of the test files aborts unless --allow-partial.
+"""
+
+from __future__ import annotations
+
+import argparse
+from collections import defaultdict
+import json
+from pathlib import Path
+import sys
+import xml.etree.ElementTree as ET
+
+from helpers import (
+    INTEGRATION_TEST_DURATIONS_FILE,
+    INTEGRATION_TESTS_PATH,
+    all_integration_test_files,
+    load_integration_durations,
+    root_path,
+)
+
+DURATIONS_FILE = Path(root_path) / INTEGRATION_TEST_DURATIONS_FILE
+MIN_COVERAGE = 0.9
+# Exit code for the expected "run covers too few files" refusal, so the
+# refresh workflow can move on to the next candidate run
+EXIT_LOW_COVERAGE = 3
+
+
+def collect_durations(junit_dir: Path, known_files: set[str]) -> dict[str, float]:
+    """Sum junit testcase times per integration test file, in seconds."""
+    durations: defaultdict[str, float] = defaultdict(float)
+    unmatched = 0
+    xml_files = sorted(junit_dir.rglob("*.xml"))
+    if not xml_files:
+        raise SystemExit(f"no junit XML files found under {junit_dir}")
+    for xml_file in xml_files:
+        for testcase in ET.parse(xml_file).getroot().iter("testcase"):
+            # Skipped/errored testcases carry time="0"; recording them would
+            # overwrite a good previous duration
+            if any(
+                testcase.find(tag) is not None
+                for tag in ("skipped", "error", "failure")
+            ):
+                continue
+            # classname is the dotted module plus any test class, e.g.
+            # tests.integration.test_x or tests.integration.test_x.TestFoo
+            parts = testcase.get("classname", "").split(".")
+            if parts[:2] != ["tests", "integration"] or len(parts) < 3:
+                unmatched += 1
+                continue
+            path = f"{INTEGRATION_TESTS_PATH}{parts[2]}.py"
+            if path not in known_files:
+                print(f"skipping unknown test module {path}", file=sys.stderr)
+                continue
+            durations[path] += float(testcase.get("time", "0"))
+    if unmatched:
+        # A junit naming change would otherwise shrink the recording silently
+        raise SystemExit(
+            f"{unmatched} testcases with unexpected classnames; the junit layout changed"
+        )
+    # An all-skipped file totals 0.0; let the merge keep its previous entry
+    return {k: v for k, v in durations.items() if v > 0}
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument(
+        "junit_dir", type=Path, help="directory containing downloaded junit XML files"
+    )
+    parser.add_argument(
+        "--allow-partial",
+        action="store_true",
+        help="merge a run covering under 90%% of the test files",
+    )
+    args = parser.parse_args()
+
+    on_disk = set(all_integration_test_files())
+    if not on_disk:
+        raise SystemExit("no integration test files found; wrong checkout root?")
+    collected = collect_durations(args.junit_dir, on_disk)
+    coverage = len(collected.keys() & on_disk) / len(on_disk)
+    if coverage < MIN_COVERAGE and not args.allow_partial:
+        print(
+            f"artifacts cover only {coverage:.0%} of {len(on_disk)} test files; "
+            "use a full matrix run or pass --allow-partial to merge anyway",
+            file=sys.stderr,
+        )
+        return EXIT_LOW_COVERAGE
+
+    # Validated load: a bad previous entry cannot survive the round trip, and
+    # an unreadable file aborts rather than being overwritten
+    previous = load_integration_durations()
+    if DURATIONS_FILE.is_file() and not previous:
+        raise SystemExit(f"{DURATIONS_FILE} is unreadable; refusing to overwrite it")
+    # New recordings win, absent files keep theirs, deleted files drop out
+    merged = {
+        path: collected.get(path, previous.get(path))
+        for path in sorted(on_disk)
+        if path in collected or path in previous
+    }
+    DURATIONS_FILE.write_text(
+        json.dumps({k: round(v, 2) for k, v in merged.items()}, indent=2) + "\n"
+    )
+    print(f"wrote {len(merged)} entries to {DURATIONS_FILE} ({coverage:.0%} fresh)")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())