Merge remote-tracking branch 'origin/binary-sensor-lazy-integration-imports' into sensor-lazy-integration-imports

script/analyze_component_buses.py

@@ -98,6 +98,7 @@ ISOLATED_COMPONENTS = {
     "neopixelbus": "RMT type conflict with ESP32 Arduino/ESP-IDF headers (enum vs struct rmt_channel_t)",
     "packages": "cannot merge packages",
     "tinyusb": "Conflicts with usb_host component - cannot be used together",
+    "tinyusb_keyboard": "Depends on tinyusb which conflicts with usb_host",
     "uart_mux": "Depends on tinyusb which conflicts with usb_host",
     "usb_cdc_acm": "Depends on tinyusb which conflicts with usb_host",
 }

script/ci-custom.py

@@ -1269,11 +1269,15 @@ def lint_no_std_nothrow(fname, match):
     )
 
 
+# Every C++ source and header; plain C files cannot use the macro
+LOG_TAG_INCLUDE = ["esphome/*.cpp", "esphome/*.h"]
+
+
 @lint_re_check(
-    r"^\s*(?:(?:static|constexpr|inline)\s+)*(?:const\s+)?char\s*(?:\*\s*(?:const\s+)?TAG|\s(?:const\s+)?TAG\s*\[\w*\])"
-    r"\s*(?:=\s*\{?|[{(])\s*\"",
+    r"^\s*(?:(?:static|constexpr|inline)\s+)*(?:const\s+)?char\s*(?:\*\s*(?:const\s+)?(?:\w+::)*TAG|\s(?:const\s+)?(?:\w+::)*TAG\s*\[\w*\])"
+    r"\s*(?:=\s*\{?|[{(])\s*\"" + CPP_RE_EOL,
     prefilter="TAG",
-    include=["esphome/components/*.cpp", "esphome/components/**/*.cpp"],
+    include=LOG_TAG_INCLUDE,
 )
 def lint_log_tag_macro(fname, match):
     return (
@@ -1281,6 +1285,8 @@ def lint_log_tag_macro(fname, match):
         f"ESP8266.\n"
         f"  Before: {highlight('static const char *const TAG = "name";')}\n"
         f"  After:  {highlight('ESPHOME_LOG_TAG(TAG, "name");')}\n"
+        f"A class member tag cannot use the macro; declare {highlight('static const char TAG[];')} in the class "
+        f"and define {highlight('const char Foo::TAG[] PROGMEM = "name";')} in the .cpp.\n"
         f"Never pass such a TAG to set_timeout/set_interval names or string functions.\n"
         f"(If strictly necessary, add `// NOLINT` to the end of the line)"
     )
@@ -1288,11 +1294,11 @@ def lint_log_tag_macro(fname, match):
 
 @lint_re_check(
     r"(?:\b(?:set_timeout|set_interval|set_retry|cancel_timeout|cancel_interval|cancel_retry|defer)"
-    r"|\b(?:strcmp|strncmp|strcasecmp|strlen|strcpy|strncpy)|std::string)\s*\((?:(?:[^();]|\([^()]*\))*,)?\s*TAG\b"
-    r"|\bstd::string\s+\w+\s*[={(]\s*TAG\b",
+    r"|\b(?:strcmp|strncmp|strcasecmp|strlen|strcpy|strncpy)|std::string)\s*\((?:(?:[^();]|\([^()]*\))*,)?\s*(?:\w+::)*TAG\b"
+    r"|\bstd::string\s+\w+\s*[={(]\s*(?:\w+::)*TAG\b",
     mask=True,
     prefilter="TAG",
-    include=["esphome/components/*.cpp", "esphome/components/**/*.cpp"],
+    include=LOG_TAG_INCLUDE,
 )
 def lint_log_tag_as_string(fname, match):
     return (

script/clang-tidy

@@ -68,6 +68,9 @@ def clang_options(idedata, environment):
 
     # extract target architecture from triplet in g++ filename
     triplet = Path(idedata["cxx_path"]).name[:-4]
+    # env:host compiles with the system g++, so there is no triplet and the
+    # host's own target and headers are the right ones.
+    native = environment.startswith("host")
     if triplet.startswith("xtensa-"):
         # clang has an Xtensa frontend, but only a generic core -- the esp32 IDF
         # toolchain headers (xtruntime, xtensa/config) need the GCC core config
@@ -83,6 +86,8 @@ def clang_options(idedata, environment):
         cmd.append("-D__XTENSA__")
         cmd.append("-D__XTENSA_EL__")
         cmd.append("-D_LIBC")
+    elif native:
+        cmd.append("-Qunused-arguments")
     else:
         # RISC-V (and other non-Xtensa targets) have a real clang backend, so
         # compile for the actual triplet. Espressif's RISC-V GCC -march adds
@@ -118,7 +123,7 @@ def clang_options(idedata, environment):
             "-mfp16-format=ieee",
             "--param=min-pagesize=0",
         )
-    else:
+    elif not native:
         cmd.extend(
             [
                 # disable built-in include directories from the host
@@ -127,10 +132,11 @@ def clang_options(idedata, environment):
         )
 
     # set flags
+    if not native:
+        # disable built-in include directories from the host
+        cmd.append("-nostdinc")
     cmd.extend(
         [
-            # disable built-in include directories from the host
-            "-nostdinc",
             # allow to condition code on the presence of clang-tidy
             "-DCLANG_TIDY",
             # (esp-idf) Fix __once_callable in some libstdc++ headers
@@ -314,11 +320,17 @@ def run_tidy(
         if args.environment.startswith(("rp2", "bk72xx", "ln882h", "rtl87xx")):
             # MMIO peripheral access on these bare-metal platforms is all
             # fixed-address.
+            disabled_checks.append("-clang-analyzer-core.FixedAddressDereference")
+        if args.environment.startswith("host"):
+            # glibc's assert() expands to a call the check mistakes for a
+            # constant condition.
+            disabled_checks += ["-misc-static-assert", "-cert-dcl03-c"]
+        if args.environment.startswith(("rp2", "bk72xx", "ln882h", "rtl87xx", "host")):
             # bugprone-pointer-arithmetic-on-polymorphic-object (and its
             # cert-ctr56-cpp alias) crashes clang-tidy 22 with infinite matcher
-            # recursion on lvgl_esphome.h under the RP2 defines.
+            # recursion on lvgl_esphome.h under the RP2 defines and on the
+            # all-include file under the host defines.
             disabled_checks += [
-                "-clang-analyzer-core.FixedAddressDereference",
                 "-bugprone-pointer-arithmetic-on-polymorphic-object",
                 "-cert-ctr56-cpp",
             ]
@@ -327,6 +339,8 @@ def run_tidy(
         if disabled_checks:
             invocation.append("--checks=" + ",".join(disabled_checks))
         invocation.append(f"--header-filter={Path(basepath).resolve()}/.*")
+        # Vendored third-party headers are kept as upstream wrote them.
+        invocation.append("--exclude-header-filter=.*/http_request/httplib\\.h")
         invocation.append(str(Path(path).resolve()))
         invocation.append("--")
         invocation.extend(options)
@@ -448,6 +462,11 @@ def main():
 
     if args.grep:
         files = filter_grep(files, args.grep)
+        # Not every tidy environment carries googletest (LibreTiny does not),
+        # so a platform grep that matched a unit test would fail on
+        # <gtest/gtest.h>. The tests are analyzed by the full ESP32 IDF run,
+        # which has no --grep.
+        files = [f for f in files if not f.startswith("tests/")]
 
     if args.exclude_grep:
         excluded = set(filter_grep(files, args.exclude_grep))
@@ -464,11 +483,16 @@ def main():
 
     # Add header file for checking (before early exit check)
     if args.all_headers and args.split_at in (None, 1):
+        all_headers = [
+            os.path.relpath(p, cwd) for p in git_ls_files(["esphome/**/*.h"])
+        ]
+        # The path filter applies to headers too: a component left out because
+        # it does not build in this environment must not come back via the
+        # all-include file.
+        if args.files:
+            all_headers = [p for p in all_headers if file_name_re.search(p)]
         # When --changed is used, only include changed headers instead of all headers
         if args.changed:
-            all_headers = [
-                os.path.relpath(p, cwd) for p in git_ls_files(["esphome/**/*.h"])
-            ]
             changed_headers = filter_changed(all_headers)
             if changed_headers:
                 build_all_include(changed_headers)
@@ -476,7 +500,7 @@ def main():
             else:
                 print("No changed headers to check")
         else:
-            build_all_include()
+            build_all_include(all_headers)
             files.insert(0, temp_header_file)
             print(f"Added all-include header file, new total: {len(files)}")
 

script/determine-jobs.py

@@ -49,6 +49,7 @@ Options:
 from __future__ import annotations
 
 import argparse
+import ast
 from collections import Counter
 from collections.abc import Callable
 from enum import StrEnum
@@ -58,6 +59,7 @@ import math
 import os
 from pathlib import Path
 import statistics
+import subprocess
 import sys
 from typing import Any
 
@@ -69,6 +71,7 @@ from clang_tidy_hash import (
 )
 from helpers import (
     CPP_FILE_EXTENSIONS,
+    ESPHOME_COMPONENTS_PATH,
     ESPHOME_TESTS_COMPONENTS_PATH,
     INTEGRATION_TESTS_PATH,
     PYTHON_FILE_EXTENSIONS,
@@ -805,6 +808,9 @@ def determine_cpp_unit_tests(
 # Paths within tests/benchmarks/ that contain component benchmark files
 BENCHMARKS_COMPONENTS_PATH = "tests/benchmarks/components"
 
+# The Python benchmark that imports components; its COMPONENTS tuple says which ones
+PYTHON_IMPORT_BENCHMARK_FILE = "tests/benchmarks/python/test_component_import_bench.py"
+
 # Files that, when changed, should trigger benchmark runs
 BENCHMARK_INFRASTRUCTURE_FILES = frozenset(
     {
@@ -815,6 +821,53 @@ BENCHMARK_INFRASTRUCTURE_FILES = frozenset(
 )
 
 
+def _benchmark_components_tuple(path: Path) -> frozenset[str]:
+    """Return the benchmark file's ``COMPONENTS`` tuple.
+
+    Raises when the file has none, so renaming it fails CI instead of quietly
+    turning the trigger off.
+    """
+    for node in ast.parse(path.read_text(encoding="utf-8")).body:
+        if isinstance(node, ast.Assign) and any(
+            isinstance(t, ast.Name) and t.id == "COMPONENTS" for t in node.targets
+        ):
+            return frozenset(ast.literal_eval(node.value))
+    raise RuntimeError(f"{PYTHON_IMPORT_BENCHMARK_FILE} has no COMPONENTS tuple")
+
+
+# Imports the given components and prints every component whose module got loaded
+_IMPORT_PROBE = (
+    "import importlib, sys\n"
+    "for c in sys.argv[1:]: importlib.import_module('esphome.components.' + c)\n"
+    "print(*{n.split('.')[2] for n in sys.modules if n.startswith('esphome.components.')})"
+)
+
+
+def python_import_benchmarked_components() -> frozenset[str] | None:
+    """Return the components loaded by importing the benchmark's ``COMPONENTS``.
+
+    Imported in a fresh interpreter from the checkout, so the set is exactly what
+    the benchmark times. None when the import fails.
+    """
+    path = Path(root_path) / PYTHON_IMPORT_BENCHMARK_FILE
+    if not path.is_file():
+        return frozenset()
+    result = subprocess.run(
+        [sys.executable, "-c", _IMPORT_PROBE, *_benchmark_components_tuple(path)],
+        cwd=root_path,
+        capture_output=True,
+        text=True,
+        check=False,
+    )
+    if result.returncode != 0:
+        print(
+            f"Benchmark import probe failed, running benchmarks:\n{result.stderr}",
+            file=sys.stderr,
+        )
+        return None
+    return frozenset(result.stdout.split())
+
+
 def should_run_benchmarks(branch: str | None = None) -> bool:
     """Determine if benchmarks (C++ and Python) should run based on changed files.
 
@@ -831,6 +884,8 @@ def should_run_benchmarks(branch: str | None = None) -> bool:
     4. A directly changed component has benchmark files (no dependency expansion)
     5. Benchmark infrastructure changed (tests/benchmarks/*, script/cpp_benchmark.py,
        script/build_helpers.py, script/setup_codspeed_lib.py)
+    6. A Python file changed in a component that importing the Python import
+       benchmark's ``COMPONENTS`` loads, even if it has no C++ benchmark
 
     Unlike unit tests, benchmarks do NOT expand to dependent components.
     Changing ``sensor`` does not trigger ``api`` benchmarks just because
@@ -862,6 +917,19 @@ def should_run_benchmarks(branch: str | None = None) -> bool:
     ):
         return True
 
+    # The Python import benchmark times importing its components and what they pull
+    # in, so their Python sources move its numbers even without a C++ benchmark.
+    # A failed import runs the benchmarks rather than risk skipping them.
+    changed_python = {
+        get_component_from_path(f)
+        for f in files
+        if f.startswith(ESPHOME_COMPONENTS_PATH) and f.endswith(PYTHON_FILE_EXTENSIONS)
+    }
+    if changed_python:
+        loaded = python_import_benchmarked_components()
+        if loaded is None or changed_python & loaded:
+            return True
+
     # Check if any directly changed component has benchmarks
     benchmarks_dir = Path(root_path) / BENCHMARKS_COMPONENTS_PATH
     if not benchmarks_dir.is_dir():