diff --git a/script/helpers.py b/script/helpers.py index fc2a3607fbd..0086a00e858 100644 --- a/script/helpers.py +++ b/script/helpers.py @@ -238,6 +238,72 @@ class _ConflictWalk: rejects: set[str] +@cache +def _get_test_config_components(component: str, platform: str) -> frozenset[str]: + """Return the components referenced by a component's test config for a platform. + + Loads ``tests/components//test..yaml`` and extracts the + top-level component keys (and list ``platform:`` values). This lets the + conflict splitter see components that are only pulled in via a test config + (e.g. nRF52 ``network`` tests that also enable ``openthread``), which a + purely static AUTO_LOAD/CONFLICTS_WITH parse cannot discover -- notably for + components like ``api`` whose ``AUTO_LOAD`` is a callable. + + Failures (missing file, parse error) are treated as empty so the splitter + never crashes on a malformed or absent test config. + """ + from esphome import yaml_util + + test_file = ( + Path(root_path) / "tests" / "components" / component / f"test.{platform}.yaml" + ) + if not test_file.exists(): + return frozenset() + try: + config = yaml_util.load_yaml(test_file) + except Exception: # noqa: BLE001 - never let a bad test config crash grouping + # Matches analyze_component_buses, which loads these same files and + # silently tolerates parse failures; surfacing it only here would be + # inconsistent and noisy. + return frozenset() + if not isinstance(config, dict): + return frozenset() + return frozenset(_extract_components_from_yaml(config)) + + +@cache +def _conflict_walk(comp: str, platform: str) -> _ConflictWalk: + """Build the platform-aware conflict walk for a single component. + + Seeds the walk with the component itself plus any components pulled in via + its ``test..yaml`` config, then folds in each seed's static + AUTO_LOAD closure and CONFLICTS_WITH declarations. Cached per + ``(component, platform)`` since the test-config seeds are platform-specific. + """ + seeds = {comp} | set(_get_test_config_components(comp, platform)) + walk = _ConflictWalk(loaded=set(seeds), rejects=set()) + stack = list(seeds) + while stack: + metadata = parse_component_metadata(stack.pop()) + walk.rejects |= metadata.conflicts_with + new = metadata.auto_load - walk.loaded + walk.loaded |= new + stack.extend(new) + return walk + + +def components_conflict(a: str, b: str, platform: str) -> bool: + """Return True if components ``a`` and ``b`` cannot share a build on ``platform``. + + Uses the same platform-aware conflict walk as :func:`split_conflicting_groups` + so callers (e.g. the no-bus redistribution in ``test_build_components.py``) + agree with how groups were originally split. The conflict relation is + symmetric even when only one side declares CONFLICTS_WITH. + """ + wa, wb = _conflict_walk(a, platform), _conflict_walk(b, platform) + return not wa.rejects.isdisjoint(wb.loaded) or not wb.rejects.isdisjoint(wa.loaded) + + def split_conflicting_groups( grouped_components: dict[tuple[str, str], list[str]], ) -> dict[tuple[str, str], list[str]]: @@ -250,33 +316,24 @@ def split_conflicting_groups( conflict relation is treated as symmetric even when only one side declares it (e.g. ethernet rejects wifi but wifi does not declare the reverse). + + The walk is platform-aware: in addition to the static AUTO_LOAD closure, + each ``(component, platform)`` walk is seeded with the components found in + that component's ``test..yaml`` config. This catches conflicts + that only exist on a given platform and are expressed through the test + config rather than static metadata -- e.g. on nRF52 the ``network``/``api`` + test configs also enable ``openthread``, which ``zigbee`` declares a + conflict with, so ``api`` and ``zigbee`` end up split there. On ESP32 those + test configs have no ``openthread``, so the components still group together. """ - batch = {c for comps in grouped_components.values() for c in comps} - - walks: dict[str, _ConflictWalk] = {} - for comp in batch: - walk = _ConflictWalk(loaded={comp}, rejects=set()) - stack = [comp] - while stack: - metadata = parse_component_metadata(stack.pop()) - walk.rejects |= metadata.conflicts_with - new = metadata.auto_load - walk.loaded - walk.loaded |= new - stack.extend(new) - walks[comp] = walk - - def conflicts(a: str, b: str) -> bool: - wa, wb = walks[a], walks[b] - return not wa.rejects.isdisjoint(wb.loaded) or not wb.rejects.isdisjoint( - wa.loaded - ) - result: dict[tuple[str, str], list[str]] = {} for (platform, signature), components in grouped_components.items(): buckets: list[list[str]] = [] for comp in components: for bucket in buckets: - if not any(conflicts(comp, other) for other in bucket): + if not any( + components_conflict(comp, other, platform) for other in bucket + ): bucket.append(comp) break else: diff --git a/script/test_build_components.py b/script/test_build_components.py index 651268609e1..ce2a35add35 100755 --- a/script/test_build_components.py +++ b/script/test_build_components.py @@ -40,6 +40,7 @@ from script.analyze_component_buses import ( uses_local_file_references, ) from script.helpers import ( + components_conflict, get_component_test_files, is_validate_only_file, parse_test_filename, @@ -788,14 +789,35 @@ def run_grouped_component_tests( if plat == platform and sig != NO_BUSES_SIGNATURE ] - if platform_groups: - # Distribute no_buses components round-robin across existing groups - for i, comp in enumerate(no_buses_comps): - sig, _ = platform_groups[i % len(platform_groups)] - grouped_components[(platform, sig)].append(comp) - else: - # No other groups for this platform - keep no_buses components together - grouped_components[(platform, NO_BUSES_SIGNATURE)] = no_buses_comps + # Distribute no_buses components round-robin across existing groups, + # but never place a component into a group it conflicts with. Conflict + # splitting (split_conflicting_groups) may have created sibling groups + # like "no_buses__conflict1" precisely to keep incompatible components + # apart (e.g. on nRF52, network pulls in openthread which zigbee + # conflicts with); redistribution must not silently undo that split. + leftover: list[str] = [] + for i, comp in enumerate(no_buses_comps): + placed = False + # Try groups starting at the round-robin offset to keep the spread. + for offset in range(len(platform_groups)): + sig, comps = platform_groups[(i + offset) % len(platform_groups)] + if any(components_conflict(comp, other, platform) for other in comps): + continue + # comps is the same list object stored in grouped_components, so + # this also extends the group in grouped_components. + comps.append(comp) + placed = True + break + if not placed: + leftover.append(comp) + + if leftover: + # Components that conflict with every existing group stay together in + # their own no_buses group (they were grouped before, so they don't + # conflict with each other). + grouped_components.setdefault((platform, NO_BUSES_SIGNATURE), []).extend( + leftover + ) groups_to_test = [] individual_tests = set() # Use set to avoid duplicates diff --git a/tests/components/api/test.nrf52-adafruit.yaml b/tests/components/api/test.nrf52-adafruit.yaml index 9229d68aa3a..18bf23d7106 100644 --- a/tests/components/api/test.nrf52-adafruit.yaml +++ b/tests/components/api/test.nrf52-adafruit.yaml @@ -1,4 +1,7 @@ network: enable_ipv6: true +openthread: + tlv: 0E080000000000010000 + api: diff --git a/tests/components/mdns/test.nrf52-adafruit.yaml b/tests/components/mdns/test.nrf52-adafruit.yaml index 6aff688ff49..c24d0a19087 100644 --- a/tests/components/mdns/test.nrf52-adafruit.yaml +++ b/tests/components/mdns/test.nrf52-adafruit.yaml @@ -1,4 +1,7 @@ network: enable_ipv6: true +openthread: + tlv: 0E080000000000010000 + mdns: diff --git a/tests/components/network/test.nrf52-adafruit.yaml b/tests/components/network/test.nrf52-adafruit.yaml index 61889b0361b..ac2fe63739c 100644 --- a/tests/components/network/test.nrf52-adafruit.yaml +++ b/tests/components/network/test.nrf52-adafruit.yaml @@ -1 +1,5 @@ network: + enable_ipv6: true + +openthread: + tlv: 0E080000000000010000 diff --git a/tests/components/network/test.nrf52-mcumgr.yaml b/tests/components/network/test.nrf52-mcumgr.yaml index 61889b0361b..ac2fe63739c 100644 --- a/tests/components/network/test.nrf52-mcumgr.yaml +++ b/tests/components/network/test.nrf52-mcumgr.yaml @@ -1 +1,5 @@ network: + enable_ipv6: true + +openthread: + tlv: 0E080000000000010000 diff --git a/tests/components/network/test.nrf52-xiao-ble.yaml b/tests/components/network/test.nrf52-xiao-ble.yaml index 61889b0361b..ac2fe63739c 100644 --- a/tests/components/network/test.nrf52-xiao-ble.yaml +++ b/tests/components/network/test.nrf52-xiao-ble.yaml @@ -1 +1,5 @@ network: + enable_ipv6: true + +openthread: + tlv: 0E080000000000010000 diff --git a/tests/script/test_helpers.py b/tests/script/test_helpers.py index 82ff5e14112..886d413ccff 100644 --- a/tests/script/test_helpers.py +++ b/tests/script/test_helpers.py @@ -35,6 +35,8 @@ def clear_helpers_cache() -> None: helpers._get_github_event_data.cache_clear() helpers._get_changed_files_github_actions.cache_clear() helpers.get_components_per_integration_fixture.cache_clear() + helpers._get_test_config_components.cache_clear() + helpers._conflict_walk.cache_clear() @pytest.mark.parametrize( @@ -1504,6 +1506,8 @@ def fake_components(tmp_path: Path) -> Path: write("callable_auto", "def AUTO_LOAD():\n return ['beta']\n") write("broken", "this is not valid python !!!") helpers.parse_component_metadata.cache_clear() + helpers._get_test_config_components.cache_clear() + helpers._conflict_walk.cache_clear() return tmp_path @@ -1624,6 +1628,47 @@ def test_split_conflicting_groups_preserves_original_signature_for_first_bucket( assert signature.startswith("i2c__conflict") +def test_split_conflicting_groups_seeds_from_test_config( + fake_components: Path, monkeypatch: MonkeyPatch +) -> None: + """A conflict reachable only via a component's test config splits the group. + + ``host_user`` declares no static conflict with ``beta``, but its + ``test..yaml`` pulls in ``beta_variant`` (which AUTO_LOADs + ``beta``). On that platform the group must split; on another platform + (no such test config) it must stay together. + """ + monkeypatch.setattr(helpers, "root_path", str(fake_components)) + + # host_user has no static metadata, but its esp32 test config references + # beta_variant -> AUTO_LOAD beta, which conflicts with alpha. + tests_dir = fake_components / "tests" / "components" / "host_user" + tests_dir.mkdir(parents=True) + (tests_dir / "test.esp32.yaml").write_text("beta_variant:\n") + (fake_components / "esphome" / "components" / "host_user").mkdir() + ( + fake_components / "esphome" / "components" / "host_user" / "__init__.py" + ).write_text("") + + helpers.parse_component_metadata.cache_clear() + helpers._get_test_config_components.cache_clear() + helpers._conflict_walk.cache_clear() + + # On esp32, host_user pulls in beta (via its test config) -> conflicts with alpha. + result = helpers.split_conflicting_groups( + {("esp32", "no_buses"): ["alpha", "host_user"]} + ) + buckets = list(result.values()) + for bucket in buckets: + assert not ({"alpha", "host_user"} <= set(bucket)) + + # On a platform without that test config, they stay grouped together. + result_other = helpers.split_conflicting_groups( + {("rp2040", "no_buses"): ["alpha", "host_user"]} + ) + assert result_other == {("rp2040", "no_buses"): ["alpha", "host_user"]} + + # --------------------------------------------------------------------------- # get_component_test_files / is_validate_only_file # ---------------------------------------------------------------------------