Merge remote-tracking branch 'origin/dev' into integration

script/analyze_component_buses.py

@@ -90,6 +90,7 @@ ISOLATED_COMPONENTS = {
     "openthread_info": "Conflicts with wifi: used by most components",
     "matrix_keypad": "Needs isolation due to keypad",
     "microphone": "Defines PDM microphone requiring I2S port 0 - conflicts with micro_wake_word PDM mic when merged",
+    "mipi_rgb": "RGB display occupies many GPIOs (including ones used by the shared i2c bus) that conflict when merged with other bus components",
     "modbus_controller": "Defines multiple modbus buses for testing client/server functionality - conflicts with package modbus bus",
     "neopixelbus": "RMT type conflict with ESP32 Arduino/ESP-IDF headers (enum vs struct rmt_channel_t)",
     "packages": "cannot merge packages",

script/build_codeowners.py

@@ -61,6 +61,13 @@ for path in components_dir.iterdir():
     codeowners[f"esphome/components/{name}/*"].extend(comp.codeowners)
 
     for platform_path in path.iterdir():
+        if platform_path.name == "__init__.py":
+            # `import pkg.__init__` is valid but distinct from `import pkg`: it re-executes
+            # the component's __init__.py as a second, separate module. That's harmless for
+            # components whose top-level code is idempotent, but not guaranteed in general
+            # (e.g. code that registers into a global registry with a duplicate check), so
+            # never treat __init__.py itself as a platform candidate.
+            continue
         platform_name = platform_path.stem
         platform = get_platform(platform_name, name)
         if platform is None:

script/build_language_schema.py

@@ -390,16 +390,6 @@ def fix_mapping():
     output["mapping"][S_SCHEMAS][S_CONFIG_SCHEMA] = config
 
 
-def fix_image():
-    if "image" not in output:
-        return
-    from esphome.components.image import IMAGE_SCHEMA
-
-    config = convert_config(IMAGE_SCHEMA, "image/CONFIG_SCHEMA")
-    config["is_list"] = True
-    output["image"][S_SCHEMAS][S_CONFIG_SCHEMA] = config
-
-
 def fix_menu():
     if "display_menu_base" not in output:
         return
@@ -763,7 +753,6 @@ def build_schema():
     fix_font()
     fix_globals()
     fix_mapping()
-    fix_image()
     add_logger_tags()
     shrink()
     fix_menu()

script/ci_memory_impact_extract.py

@@ -33,6 +33,11 @@ sys.path.insert(0, str(Path(__file__).parent.parent))
 
 # pylint: disable=wrong-import-position
 from esphome.analyze_memory import MemoryAnalyzer
+from esphome.analyze_memory.toolchain import (
+    find_elf_path,
+    find_idedata_path,
+    idedata_candidates,
+)
 from esphome.platformio.toolchain import IDEData
 from script.ci_helpers import write_github_output
 
@@ -130,53 +135,31 @@ def run_detailed_analysis(build_dir: str) -> dict | None:
         print(f"Build directory not found: {build_dir}", file=sys.stderr)
         return None
 
-    # Find firmware.elf (or raw_firmware.elf for LibreTiny)
-    elf_path = None
-    for elf_candidate in [
-        build_path / "firmware.elf",
-        build_path / ".pioenvs" / build_path.name / "firmware.elf",
-        # LibreTiny uses raw_firmware.elf
-        build_path / "raw_firmware.elf",
-        build_path / ".pioenvs" / build_path.name / "raw_firmware.elf",
-    ]:
-        if elf_candidate.exists():
-            elf_path = str(elf_candidate)
-            break
-
+    elf_path = find_elf_path(build_path)
     if not elf_path:
-        print(
-            f"firmware.elf/raw_firmware.elf not found in {build_dir}", file=sys.stderr
-        )
+        print(f"No firmware ELF found in {build_dir}", file=sys.stderr)
         return None
 
-    # Find idedata.json - check multiple locations
-    device_name = build_path.name
-    idedata_candidates = [
-        # In .pioenvs for test builds
-        build_path / ".pioenvs" / device_name / "idedata.json",
-        # In .esphome/idedata for regular builds
-        Path.home() / ".esphome" / "idedata" / f"{device_name}.json",
-        # Check parent directories for .esphome/idedata (for test_build_components)
-        build_path.parent.parent.parent / "idedata" / f"{device_name}.json",
-    ]
-
     idedata = None
-    for idedata_path in idedata_candidates:
-        if not idedata_path.exists():
-            continue
+    if idedata_path := find_idedata_path(build_path):
         try:
             with idedata_path.open(encoding="utf-8") as f:
                 raw_data = json.load(f)
             idedata = IDEData(raw_data)
             print(f"Loaded idedata from: {idedata_path}", file=sys.stderr)
-            break
         except (json.JSONDecodeError, OSError) as e:
             print(
                 f"Warning: Failed to load idedata from {idedata_path}: {e}",
                 file=sys.stderr,
             )
+    else:
+        # Without idedata the analyzer falls back to whatever binutils are on
+        # PATH, which are the wrong architecture for a cross build, so say where
+        # we looked rather than let the results quietly get worse.
+        searched = "\n  ".join(str(p) for p in idedata_candidates(build_path))
+        print(f"Warning: idedata not found, searched:\n  {searched}", file=sys.stderr)
 
-    analyzer = MemoryAnalyzer(elf_path, idedata=idedata)
+    analyzer = MemoryAnalyzer(str(elf_path), idedata=idedata)
     components = analyzer.analyze()
 
     # Convert to JSON-serializable format
@@ -320,6 +303,19 @@ def main() -> int:
     else:
         print(f"{ram_bytes},{flash_bytes}")
 
+    # The build produced usable totals, so a missing detailed analysis means the
+    # build layout moved out from under this script rather than a broken build.
+    # Fail loudly: the comment would otherwise silently drop the component
+    # breakdown and the symbol tables, which is easy to miss for a long time.
+    if detailed_analysis is None:
+        print(
+            "::error::Detailed memory analysis unavailable even though the build "
+            f"succeeded (build directory: {build_dir or 'not detected'}). The PR "
+            "comment would be missing its component breakdown and symbol changes.",
+            file=sys.stderr,
+        )
+        return 1
+
     return 0
 
 

script/clang-tidy

@@ -29,7 +29,7 @@ from helpers import (
 )
 
 
-def clang_options(idedata):
+def clang_options(idedata, environment):
     cmd = []
 
     # extract target architecture from triplet in g++ filename
@@ -74,6 +74,8 @@ def clang_options(idedata):
         "-fno-jump-tables",
         "-fno-shrink-wrap",
         "-mno-target-align",
+        # GCC-only flag emitted by the LibreTiny build
+        "-mthumb-interwork",
     )
 
     if "zephyr" in triplet:
@@ -95,30 +97,59 @@ def clang_options(idedata):
         [
             # disable built-in include directories from the host
             "-nostdinc",
-            # replace pgmspace.h, as it uses GNU extensions clang doesn't support
-            # https://github.com/earlephilhower/newlib-xtensa/pull/18
-            "-D_PGMSPACE_H_",
-            "-Dpgm_read_byte(s)=(*(const uint8_t *)(s))",
-            "-Dpgm_read_byte_near(s)=(*(const uint8_t *)(s))",
-            "-Dpgm_read_word(s)=(*(const uint16_t *)(s))",
-            "-Dpgm_read_dword(s)=(*(const uint32_t *)(s))",
-            "-Dpgm_read_ptr(s)=(*(const void *const *)(s))",
-            "-DPROGMEM=",
-            "-DPGM_P=const char *",
-            "-DPSTR(s)=(s)",
-            # this next one is also needed with upstream pgmspace.h
-            # suppress warning about identifier naming in expansion of this macro
-            "-DPSTRN(s, n)=(s)",
-            # suppress warning about attribute cannot be applied to type
-            # https://github.com/esp8266/Arduino/pull/8258
-            "-Ddeprecated(x)=",
             # allow to condition code on the presence of clang-tidy
             "-DCLANG_TIDY",
             # (esp-idf) Fix __once_callable in some libstdc++ headers
             "-D_GLIBCXX_HAVE_TLS",
+            # suppress warning about attribute cannot be applied to type
+            # https://github.com/esp8266/Arduino/pull/8258
+            # also keeps deprecation diagnostics consistent across environments
+            "-Ddeprecated(x)=",
         ]
     )
 
+    if environment.startswith("rp2"):
+        # clang's ARM backend doesn't know GCC's long_call attribute (IRAM_ATTR)
+        cmd.append("-Wno-unknown-attributes")
+    elif environment.startswith(("bk72xx", "ln882h", "rtl87xx")):
+        cmd.extend(
+            [
+                # GCC on arm-none-eabi types (u)int32_t as (unsigned) long; clang
+                # types it as (unsigned) int, clashing with LibreTiny's lwip
+                # port typedefs. Match the GCC type model.
+                "-U__UINT32_TYPE__",
+                "-D__UINT32_TYPE__=long unsigned int",
+                "-U__INT32_TYPE__",
+                "-D__INT32_TYPE__=long int",
+                # newlib's machine/endian.h macroizes __bswap16 into
+                # __builtin_bswap16; the beken BDK then defines __bswap16 as a
+                # function, which GCC tolerates as a builtin redeclaration but
+                # clang rejects
+                "-D__MACHINE_ENDIAN_H__",
+            ]
+        )
+    else:
+        # replace pgmspace.h, as it uses GNU extensions clang doesn't support
+        # https://github.com/earlephilhower/newlib-xtensa/pull/18
+        # arduino-pico ships clang-parseable pgmspace inline functions, so the
+        # replacements are skipped there (they clash with those definitions).
+        cmd.extend(
+            [
+                "-D_PGMSPACE_H_",
+                "-Dpgm_read_byte(s)=(*(const uint8_t *)(s))",
+                "-Dpgm_read_byte_near(s)=(*(const uint8_t *)(s))",
+                "-Dpgm_read_word(s)=(*(const uint16_t *)(s))",
+                "-Dpgm_read_dword(s)=(*(const uint32_t *)(s))",
+                "-Dpgm_read_ptr(s)=(*(const void *const *)(s))",
+                "-DPROGMEM=",
+                "-DPGM_P=const char *",
+                "-DPSTR(s)=(s)",
+                # this next one is also needed with upstream pgmspace.h
+                # suppress warning about identifier naming in expansion of this macro
+                "-DPSTRN(s, n)=(s)",
+            ]
+        )
+
     # Copy compiler flags, dropping: ones clang doesn't understand; -Werror*
     # (clang-tidy enforces .clang-tidy's WarningsAsErrors, and a build -Werror
     # would bypass the -clang-diagnostic-* suppressions); and -std= (the native
@@ -142,8 +173,32 @@ def clang_options(idedata):
     )
     cmd.append("-std=gnu++20")
 
-    # defines
-    cmd.extend(f"-D{define}" for define in idedata["defines"])
+    if environment.startswith(("bk72xx", "ln882h", "rtl87xx")):
+        # LibreTiny leaves function-like macro values unparenthesized
+        # (bugprone-macro-parentheses); its SDK-internal FAL_PART_TABLE macro trips the same
+        # check. Assumes define values are always expressions (never type- or char-literal),
+        # which holds for the current LibreTiny idedata.
+        def sanitize_define(define):
+            name, sep, value = define.partition("=")
+            if (
+                sep
+                and value
+                and not value.startswith(("(", '"'))
+                and ("(" in name or not re.fullmatch(r"[\w.]+", value))
+            ):
+                value = f"({value})"
+            return f"-D{name}{sep}{value}"
+
+        # FAL_PART_TABLE and the delay() remap are library-scope LibreTiny flags that the real
+        # build never applies to esphome sources. Strip LibreTiny's shell quoting from define
+        # names first so the skip list matches regardless of which names it happens to quote.
+        for define in idedata["defines"]:
+            define = define.replace("'", "")
+            if define.startswith(("FAL_PART_TABLE", "delay(")):
+                continue
+            cmd.append(sanitize_define(define))
+    else:
+        cmd.extend(f"-D{define}" for define in idedata["defines"])
 
     # toolchain include directories, using -isystem to suppress their errors
     # idedata contains include directories for all toolchains of this platform, only use those from the one in use
@@ -207,6 +262,16 @@ def run_tidy(executable, args, options, tmpdir, path_queue, lock, failed_files):
         if sys.stdout.isatty():
             invocation.append("--use-color")
 
+        if args.environment.startswith(("rp2", "bk72xx", "ln882h", "rtl87xx")):
+            # MMIO peripheral access on these bare-metal platforms is all
+            # fixed-address.
+            # bugprone-pointer-arithmetic-on-polymorphic-object (and its
+            # cert-ctr56-cpp alias) crashes clang-tidy 22 with infinite matcher
+            # recursion on lvgl_esphome.h under the RP2 defines.
+            invocation.append(
+                "--checks=-clang-analyzer-core.FixedAddressDereference,"
+                "-bugprone-pointer-arithmetic-on-polymorphic-object,-cert-ctr56-cpp"
+            )
         invocation.append(f"--header-filter={Path(basepath).resolve()}/.*")
         invocation.append(str(Path(path).resolve()))
         invocation.append("--")
@@ -351,7 +416,7 @@ def main():
 
     # Load idedata and options only if we have files to check
     idedata = load_idedata(args.environment)
-    options = clang_options(idedata)
+    options = clang_options(idedata, args.environment)
 
     tmpdir = None
     if args.fix:
@@ -418,7 +483,9 @@ def main():
             print("Error applying fixes.\n", file=sys.stderr)
             raise
 
-    return len(failed_files)
+    # Cap at 255: shells truncate exit codes to one byte, so 256 failures
+    # would otherwise report success
+    return min(len(failed_files), 255)
 
 
 if __name__ == "__main__":

script/merge_component_configs.py

@@ -263,22 +263,39 @@ def prepare_component_body(comp_data: dict, comp_name: str, comp_dir: Path) -> d
         else {}
     )
 
-    packages_value = comp_data.get("packages")
-    if isinstance(packages_value, dict):
-        common_bus_packages = get_common_bus_packages()
-        for pkg_name, pkg_value in list(packages_value.items()):
-            if pkg_name in common_bus_packages:
-                continue
-            if isinstance(pkg_value, yaml_util.IncludeFile):
-                pkg_value = pkg_value.load()
-            if isinstance(pkg_value, dict):
-                comp_data = merge_config(comp_data, pkg_value)
-    elif isinstance(packages_value, list):
-        for pkg_value in packages_value:
-            if isinstance(pkg_value, yaml_util.IncludeFile):
-                pkg_value = pkg_value.load()
-            if isinstance(pkg_value, dict):
-                comp_data = merge_config(comp_data, pkg_value)
+    # Expand component-specific package includes inline. A package include may
+    # itself pull in further component-specific packages (e.g. web_server's test
+    # includes common_v2, which includes common with the wifi/network config), so
+    # keep expanding until only common bus packages remain -- otherwise the nested
+    # includes are silently dropped when the packages key is removed below.
+    common_bus_packages = get_common_bus_packages()
+    while True:
+        packages_value = comp_data.get("packages")
+        expanded = False
+        if isinstance(packages_value, dict):
+            for pkg_name, pkg_value in list(packages_value.items()):
+                if pkg_name in common_bus_packages:
+                    continue
+                # Drop before merging so a nested packages dict introduced by the
+                # include does not re-add this same key on the next iteration.
+                del packages_value[pkg_name]
+                if isinstance(pkg_value, yaml_util.IncludeFile):
+                    pkg_value = pkg_value.load()
+                if isinstance(pkg_value, dict):
+                    comp_data = merge_config(comp_data, pkg_value)
+                expanded = True
+        elif isinstance(packages_value, list):
+            # List-style packages never contain common bus packages, so expand
+            # them all and drop the key entirely.
+            comp_data.pop("packages", None)
+            for pkg_value in packages_value:
+                if isinstance(pkg_value, yaml_util.IncludeFile):
+                    pkg_value = pkg_value.load()
+                if isinstance(pkg_value, dict):
+                    comp_data = merge_config(comp_data, pkg_value)
+                    expanded = True
+        if not expanded:
+            break
     # Common bus packages are re-added once by the caller; drop them here.
     comp_data.pop("packages", None)
 

script/test_build_components.py

@@ -88,6 +88,38 @@ def show_disk_space_if_ci(esphome_command: str) -> None:
     sys.stdout.flush()
 
 
+def start_log_group(title: str) -> None:
+    """Begin a collapsible log group in the GitHub Actions log viewer.
+
+    Everything printed until the matching :func:`end_log_group` is folded away
+    by default, so the full ``esphome config``/``compile`` dump for one
+    configuration no longer pushes the pass/fail result thousands of lines down
+    the log. Outside CI this is a no-op so local runs stay plain.
+
+    Args:
+        title: Text shown on the (collapsed) group header line.
+    """
+    if not os.environ.get("GITHUB_ACTIONS"):
+        return
+    # Flush so the marker is ordered correctly relative to the child process
+    # output that follows (the subprocess writes straight to our stdout).
+    sys.stdout.flush()
+    print(f"::group::{title}")
+    sys.stdout.flush()
+
+
+def end_log_group() -> None:
+    """Close the collapsible log group opened by :func:`start_log_group`.
+
+    Outside CI this is a no-op.
+    """
+    if not os.environ.get("GITHUB_ACTIONS"):
+        return
+    sys.stdout.flush()
+    print("::endgroup::")
+    sys.stdout.flush()
+
+
 def find_component_tests(
     components_dir: Path,
     component_pattern: str = "*",
@@ -383,54 +415,48 @@ def run_esphome_test(
     # Build command string for display/logging
     cmd_str = " ".join(cmd)
 
-    # Run command
-    print(f"> [{component}] [{test_name}] [{platform_with_version}]")
+    # Run command inside a collapsible CI log group so the full esphome output
+    # for this configuration can be folded away by default.
+    group_title = f"[{component}] [{test_name}] [{platform_with_version}]"
+    start_log_group(group_title)
+    print(f"> {group_title}")
     if use_testing_mode:
         print("  (using --testing-mode)")
 
     start_time = time.time()
     test_id = f"{component}.{test_name}.{platform_with_version}"
 
+    # Always close the group, even if the subprocess or disk-space reporting
+    # raises, so later output is never folded into the wrong CI log section.
     try:
         result = subprocess.run(cmd, check=False)
-        success = result.returncode == 0
-        duration = time.time() - start_time
-
         # Show disk space after build in CI during compile
         show_disk_space_if_ci(esphome_command)
+    finally:
+        end_log_group()
 
-        if not success and not continue_on_fail:
-            # Print command immediately for failed tests
-            print(f"\n{'=' * 80}")
-            print("FAILED - Command to reproduce:")
-            print(f"{'=' * 80}")
-            print(cmd_str)
-            print()
-            raise subprocess.CalledProcessError(result.returncode, cmd)
+    success = result.returncode == 0
+    duration = time.time() - start_time
 
-        return TestResult(
-            test_id=test_id,
-            components=[component],
-            platform=platform_with_version,
-            success=success,
-            duration=duration,
-            command=cmd_str,
-            test_type=esphome_command,
-        )
-    except subprocess.CalledProcessError:
-        duration = time.time() - start_time
-        # Re-raise if we're not continuing on fail
-        if not continue_on_fail:
-            raise
-        return TestResult(
-            test_id=test_id,
-            components=[component],
-            platform=platform_with_version,
-            success=False,
-            duration=duration,
-            command=cmd_str,
-            test_type=esphome_command,
-        )
+    if not success and not continue_on_fail:
+        # Print command immediately for failed tests. The group is already
+        # closed, so the failure and reproduce command stay visible.
+        print(f"\n{'=' * 80}")
+        print("FAILED - Command to reproduce:")
+        print(f"{'=' * 80}")
+        print(cmd_str)
+        print()
+        raise subprocess.CalledProcessError(result.returncode, cmd)
+
+    return TestResult(
+        test_id=test_id,
+        components=[component],
+        platform=platform_with_version,
+        success=success,
+        duration=duration,
+        command=cmd_str,
+        test_type=esphome_command,
+    )
 
 
 def run_grouped_test(
@@ -534,54 +560,48 @@ def run_grouped_test(
     # Build command string for display/logging
     cmd_str = " ".join(cmd)
 
-    # Run command
+    # Run command inside a collapsible CI log group so the full esphome output
+    # for this grouped configuration can be folded away by default.
     components_str = ", ".join(components)
-    print(f"> [GROUPED: {components_str}] [{platform_with_version}]")
+    group_title = f"[GROUPED: {components_str}] [{platform_with_version}]"
+    start_log_group(group_title)
+    print(f"> {group_title}")
     print("  (using --testing-mode)")
 
     start_time = time.time()
     test_id = f"GROUPED[{','.join(components)}].{platform_with_version}"
 
+    # Always close the group, even if the subprocess or disk-space reporting
+    # raises, so later output is never folded into the wrong CI log section.
     try:
         result = subprocess.run(cmd, check=False)
-        success = result.returncode == 0
-        duration = time.time() - start_time
-
         # Show disk space after build in CI during compile
         show_disk_space_if_ci(esphome_command)
+    finally:
+        end_log_group()
 
-        if not success and not continue_on_fail:
-            # Print command immediately for failed tests
-            print(f"\n{'=' * 80}")
-            print("FAILED - Command to reproduce:")
-            print(f"{'=' * 80}")
-            print(cmd_str)
-            print()
-            raise subprocess.CalledProcessError(result.returncode, cmd)
+    success = result.returncode == 0
+    duration = time.time() - start_time
 
-        return TestResult(
-            test_id=test_id,
-            components=components,
-            platform=platform_with_version,
-            success=success,
-            duration=duration,
-            command=cmd_str,
-            test_type=esphome_command,
-        )
-    except subprocess.CalledProcessError:
-        duration = time.time() - start_time
-        # Re-raise if we're not continuing on fail
-        if not continue_on_fail:
-            raise
-        return TestResult(
-            test_id=test_id,
-            components=components,
-            platform=platform_with_version,
-            success=False,
-            duration=duration,
-            command=cmd_str,
-            test_type=esphome_command,
-        )
+    if not success and not continue_on_fail:
+        # Print command immediately for failed tests. The group is already
+        # closed, so the failure and reproduce command stay visible.
+        print(f"\n{'=' * 80}")
+        print("FAILED - Command to reproduce:")
+        print(f"{'=' * 80}")
+        print(cmd_str)
+        print()
+        raise subprocess.CalledProcessError(result.returncode, cmd)
+
+    return TestResult(
+        test_id=test_id,
+        components=components,
+        platform=platform_with_version,
+        success=success,
+        duration=duration,
+        command=cmd_str,
+        test_type=esphome_command,
+    )
 
 
 def run_grouped_component_tests(