From 22062d79a2b7d82bf9f62f9dc92f37d8f8e91686 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 13 Mar 2026 13:20:17 -1000 Subject: [PATCH 1/6] [analyze-memory] Add function call frequency analysis (#14779) --- esphome/analyze_memory/__init__.py | 56 ++++++++++++++- esphome/analyze_memory/cli.py | 109 +++++++++++++++++++++++++++++ 2 files changed, 162 insertions(+), 3 deletions(-) diff --git a/esphome/analyze_memory/__init__.py b/esphome/analyze_memory/__init__.py index bf1bcbfa05..7954c22822 100644 --- a/esphome/analyze_memory/__init__.py +++ b/esphome/analyze_memory/__init__.py @@ -1,6 +1,6 @@ """Memory usage analyzer for ESPHome compiled binaries.""" -from collections import defaultdict +from collections import Counter, defaultdict from dataclasses import dataclass, field import logging from pathlib import Path @@ -40,6 +40,15 @@ _READELF_SECTION_PATTERN = re.compile( r"\s*\[\s*\d+\]\s+([\.\w]+)\s+\w+\s+[\da-fA-F]+\s+[\da-fA-F]+\s+([\da-fA-F]+)" ) +# Regex for extracting call targets from objdump disassembly +# Matches direct call instructions across architectures: +# Xtensa: call0/call4/call8/call12/callx0/callx4/callx8/callx12 +# ARM: bl/blx +# Captures the mangled symbol name inside angle brackets. +_CALL_TARGET_PATTERN = re.compile( + r"\t(?:call(?:0|4|8|12)|callx(?:0|4|8|12)|blx?)\s+[\da-fA-F]+ <([^>]+)>" +) + # Component category prefixes _COMPONENT_PREFIX_ESPHOME = "[esphome]" _COMPONENT_PREFIX_EXTERNAL = "[external]" @@ -197,6 +206,8 @@ class MemoryAnalyzer: self._lib_hash_to_name: dict[str, str] = {} # Heuristic category to library redirect: "mdns_lib" -> "[lib]mdns" self._heuristic_to_lib: dict[str, str] = {} + # Function call counts: mangled_name -> call_count + self._function_call_counts: Counter[str] = Counter() def analyze(self) -> dict[str, ComponentMemory]: """Analyze the ELF file and return component memory usage.""" @@ -206,6 +217,7 @@ class MemoryAnalyzer: self._categorize_symbols() self._analyze_cswtch_symbols() self._analyze_sdk_libraries() + self._analyze_function_calls() return dict(self.components) def _parse_sections(self) -> None: @@ -384,8 +396,9 @@ class MemoryAnalyzer: return _LOGGER.info("Demangling %d symbols", len(symbols)) - self._demangle_cache = batch_demangle(symbols, objdump_path=self.objdump_path) - _LOGGER.info("Successfully demangled %d symbols", len(self._demangle_cache)) + demangled = batch_demangle(symbols, objdump_path=self.objdump_path) + self._demangle_cache.update(demangled) + _LOGGER.info("Successfully demangled %d symbols", len(demangled)) def _demangle_symbol(self, symbol: str) -> str: """Get demangled C++ symbol name from cache.""" @@ -1011,6 +1024,43 @@ class MemoryAnalyzer: total_size, ) + def _analyze_function_calls(self) -> None: + """Count function call sites by parsing disassembly output. + + Parses direct call instructions (call0/call8/bl/blx) from objdump -d + to count how many times each function is called. This helps identify + inlining candidates — frequently called small functions benefit most + from inlining. + """ + result = run_tool( + [self.objdump_path, "-d", str(self.elf_path)], + timeout=60, + ) + if result is None or result.returncode != 0: + _LOGGER.debug("Failed to disassemble ELF for function call analysis") + return + + self._function_call_counts = Counter( + match.group(1) + for line in result.stdout.splitlines() + if (match := _CALL_TARGET_PATTERN.search(line)) + ) + + # Demangle any call targets not already in the cache + missing = [ + name + for name in self._function_call_counts + if name not in self._demangle_cache + ] + if missing: + self._batch_demangle_symbols(missing) + + _LOGGER.debug( + "Function call analysis: %d unique targets, %d total calls", + len(self._function_call_counts), + sum(self._function_call_counts.values()), + ) + def get_unattributed_ram(self) -> tuple[int, int, int]: """Get unattributed RAM sizes (SDK/framework overhead). diff --git a/esphome/analyze_memory/cli.py b/esphome/analyze_memory/cli.py index dbc19c6b89..acaf5f4562 100644 --- a/esphome/analyze_memory/cli.py +++ b/esphome/analyze_memory/cli.py @@ -231,6 +231,110 @@ class MemoryAnalyzerCLI(MemoryAnalyzer): lines.append(f" {size:>6,} B {sym_name}") lines.append("") + # Number of top called functions to show + TOP_CALLS_LIMIT: int = 50 + # Number of inlining candidates to show + INLINE_CANDIDATES_LIMIT: int = 25 + # Maximum function size in bytes to consider for inlining + INLINE_SIZE_THRESHOLD: int = 16 + + def _build_symbol_sizes(self) -> dict[str, int]: + """Build a size lookup from all component symbols: mangled_name -> size.""" + return { + symbol: size + for symbols in self._component_symbols.values() + for symbol, _, size, _ in symbols + } + + def _format_call_row( + self, index: int, mangled: str, count: int, symbol_sizes: dict[str, int] + ) -> str: + """Format a single row for call frequency tables.""" + demangled = self._demangle_cache.get(mangled, mangled) + if len(demangled) > 80: + demangled = f"{demangled[:77]}..." + size = symbol_sizes.get(mangled) + size_str = f"{size:>5,} B" if size is not None else " ?" + return f"{index:>3} {count:>5} {size_str} {demangled}" + + def _add_call_table_header(self, lines: list[str]) -> None: + """Add the header row for call frequency tables.""" + lines.append(f"{'#':>3} {'Calls':>5} {'Size':>7} Function") + lines.append(f"{'---':>3} {'-----':>5} {'-------':>7} {'-' * 60}") + + def _add_function_call_analysis(self, lines: list[str]) -> None: + """Add function call frequency analysis section. + + Shows the most frequently called functions by call site count. + """ + self._add_section_header(lines, "Top Called Functions") + + symbol_sizes = self._build_symbol_sizes() + + # Sort by call count descending + sorted_calls = sorted( + self._function_call_counts.items(), key=lambda x: x[1], reverse=True + ) + + self._add_call_table_header(lines) + + for i, (mangled, count) in enumerate(sorted_calls[: self.TOP_CALLS_LIMIT]): + lines.append(self._format_call_row(i + 1, mangled, count, symbol_sizes)) + + total_calls = sum(self._function_call_counts.values()) + lines.append("") + lines.append( + f"Total: {len(self._function_call_counts)} unique targets, " + f"{total_calls:,} call sites" + ) + lines.append("") + + def _add_inline_candidates(self, lines: list[str]) -> None: + """Add inlining candidates section. + + Shows frequently called functions that are small enough to benefit + from inlining (< 16 bytes). These are the best candidates for + reducing call overhead. + """ + self._add_section_header( + lines, + f"Inlining Candidates (<{self.INLINE_SIZE_THRESHOLD} B, by call count)", + ) + + symbol_sizes = self._build_symbol_sizes() + + # Filter to small functions with known size, sort by call count + candidates = sorted( + ( + (mangled, count) + for mangled, count in self._function_call_counts.items() + if mangled in symbol_sizes + and symbol_sizes[mangled] < self.INLINE_SIZE_THRESHOLD + ), + key=lambda x: x[1], + reverse=True, + ) + + if not candidates: + lines.append("No candidates found.") + lines.append("") + return + + self._add_call_table_header(lines) + + for i, (mangled, count) in enumerate( + candidates[: self.INLINE_CANDIDATES_LIMIT] + ): + lines.append(self._format_call_row(i + 1, mangled, count, symbol_sizes)) + + lines.append("") + lines.append( + f"Showing top {min(len(candidates), self.INLINE_CANDIDATES_LIMIT)} " + f"of {len(candidates)} functions under " + f"{self.INLINE_SIZE_THRESHOLD} B" + ) + lines.append("") + def generate_report(self, detailed: bool = False) -> str: """Generate a formatted memory report.""" components = sorted( @@ -533,6 +637,11 @@ class MemoryAnalyzerCLI(MemoryAnalyzer): if self._cswtch_symbols: self._add_cswtch_analysis(lines) + # Function call frequency analysis + if self._function_call_counts: + self._add_function_call_analysis(lines) + self._add_inline_candidates(lines) + lines.append( "Note: This analysis covers symbols in the ELF file. Some runtime allocations may not be included." ) From 56f7b3e61b0bb6defd652092d9f4c13e4dabd593 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 13 Mar 2026 13:20:35 -1000 Subject: [PATCH 2/6] [ci] Only run integration tests for changed components (#14776) --- .github/workflows/ci.yml | 17 ++- script/determine-jobs.py | 104 ++++++++----- script/helpers.py | 132 +++++++++++++++-- tests/script/test_determine_jobs.py | 217 ++++++++++++++++++++++------ tests/script/test_helpers.py | 30 +++- 5 files changed, 400 insertions(+), 100 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 461e676c4e..fedfebf393 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -170,6 +170,8 @@ jobs: - common outputs: integration-tests: ${{ steps.determine.outputs.integration-tests }} + integration-tests-run-all: ${{ steps.determine.outputs.integration-tests-run-all }} + integration-test-files: ${{ steps.determine.outputs.integration-test-files }} clang-tidy: ${{ steps.determine.outputs.clang-tidy }} clang-tidy-mode: ${{ steps.determine.outputs.clang-tidy-mode }} python-linters: ${{ steps.determine.outputs.python-linters }} @@ -210,6 +212,8 @@ jobs: # Extract individual fields echo "integration-tests=$(echo "$output" | jq -r '.integration_tests')" >> $GITHUB_OUTPUT + echo "integration-tests-run-all=$(echo "$output" | jq -r '.integration_tests_run_all')" >> $GITHUB_OUTPUT + echo "integration-test-files=$(echo "$output" | jq -c '.integration_test_files')" >> $GITHUB_OUTPUT echo "clang-tidy=$(echo "$output" | jq -r '.clang_tidy')" >> $GITHUB_OUTPUT echo "clang-tidy-mode=$(echo "$output" | jq -r '.clang_tidy_mode')" >> $GITHUB_OUTPUT echo "python-linters=$(echo "$output" | jq -r '.python_linters')" >> $GITHUB_OUTPUT @@ -261,9 +265,20 @@ jobs: - name: Register matcher run: echo "::add-matcher::.github/workflows/matchers/pytest.json" - name: Run integration tests + env: + INTEGRATION_TEST_FILES: ${{ needs.determine-jobs.outputs.integration-test-files }} + INTEGRATION_TESTS_RUN_ALL: ${{ needs.determine-jobs.outputs.integration-tests-run-all }} run: | . venv/bin/activate - pytest -vv --no-cov --tb=native -n auto tests/integration/ + if [[ "$INTEGRATION_TESTS_RUN_ALL" == "true" ]]; then + echo "Running all integration tests" + pytest -vv --no-cov --tb=native -n auto tests/integration/ + else + # Parse JSON array into bash array to avoid shell expansion issues + mapfile -t test_files < <(echo "$INTEGRATION_TEST_FILES" | jq -r '.[]') + echo "Running ${#test_files[@]} specific integration tests" + pytest -vv --no-cov --tb=native -n auto "${test_files[@]}" + fi cpp-unit-tests: name: Run C++ unit tests diff --git a/script/determine-jobs.py b/script/determine-jobs.py index 318ac04a7d..6808a3cf6c 100755 --- a/script/determine-jobs.py +++ b/script/determine-jobs.py @@ -6,6 +6,8 @@ what files have changed. It outputs JSON with the following structure: { "integration_tests": true/false, + "integration_tests_run_all": true/false, + "integration_test_files": ["tests/integration/test_foo.py", ...], "clang_tidy": true/false, "clang_format": true/false, "python_linters": true/false, @@ -56,13 +58,13 @@ from helpers import ( core_changed, filter_component_and_test_cpp_files, filter_component_and_test_files, - get_all_dependencies, get_changed_components, get_component_from_path, get_component_test_files, - get_components_from_integration_fixtures, get_components_with_dependencies, get_cpp_changed_components, + get_fixture_to_test_files, + get_integration_test_files_for_components, get_target_branch, git_ls_files, parse_test_filename, @@ -143,65 +145,88 @@ MEMORY_IMPACT_PLATFORM_PREFERENCE = [ ] -def should_run_integration_tests(branch: str | None = None) -> bool: - """Determine if integration tests should run based on changed files. +def determine_integration_tests(branch: str | None = None) -> tuple[bool, list[str]]: + """Determine which integration tests should run based on changed files. - This function is used by the CI workflow to intelligently skip integration tests when they're - not needed, saving significant CI time and resources. + This function is used by the CI workflow to intelligently skip or filter + integration tests, saving significant CI time and resources. - Integration tests will run when ANY of the following conditions are met: + Returns (run_all=True, []) when ANY of the following conditions are met: 1. Core C++ files changed (esphome/core/*) - Any .cpp, .h, .tcc files in the core directory - These files contain fundamental functionality used throughout ESPHome - - Examples: esphome/core/component.cpp, esphome/core/application.h 2. Core Python files changed (esphome/core/*.py) - Only .py files in the esphome/core/ directory - These are core Python files that affect the entire system - - Examples: esphome/core/config.py, esphome/core/__init__.py - - NOT included: esphome/*.py, esphome/dashboard/*.py, esphome/components/*/*.py - 3. Integration test files changed - - Any file in tests/integration/ directory - - This includes test files themselves and fixture YAML files - - Examples: tests/integration/test_api.py, tests/integration/fixtures/api.yaml + 3. Integration test infrastructure files changed + - conftest.py, types.py, const.py, entity_utils.py, state_utils.py, etc. - 4. Components used by integration tests (or their dependencies) changed - - The function parses all YAML files in tests/integration/fixtures/ - - Extracts which components are used in integration tests - - Recursively finds all dependencies of those components - - If any of these components have changes, tests must run - - Example: If api.yaml uses 'sensor' and 'api' components, and 'api' depends on 'socket', - then changes to sensor/, api/, or socket/ components trigger tests + Returns (run_all=False, [test_files...]) when: + + 4. Specific integration test files changed + - Only those specific test files are returned + + 5. Components used by integration tests (or their dependencies) changed + - Only test files whose fixtures use the changed components are returned Args: branch: Branch to compare against. If None, uses default. Returns: - True if integration tests should run, False otherwise. + Tuple of (run_all, test_files) where: + - run_all: True if all integration tests should run + - test_files: List of specific test file paths to run (empty if run_all + is True, or if no tests need to run) """ files = changed_files(branch) if core_changed(files): - # If any core files changed, run integration tests - return True + # If any core files changed, run all integration tests + return (True, []) - # Check if any integration test files changed - if any("tests/integration" in file for file in files): - return True + # If infrastructure Python files changed (conftest, utils, etc.), run all tests + # Excludes test files (test_*.py), fixtures, and non-Python files (README.md) + if any( + f.startswith("tests/integration/") + and f.endswith(".py") + and not f.startswith("tests/integration/test_") + and "/fixtures/" not in f + for f in files + ): + return (True, []) - # Get all components used in integration tests and their dependencies - fixture_components = get_components_from_integration_fixtures() - all_required_components = get_all_dependencies(fixture_components) + # Collect specific test files that need to run + test_files: set[str] = set() + fixture_to_test_files = get_fixture_to_test_files() - # Check if any required components changed - for file in files: - component = get_component_from_path(file) - if component and component in all_required_components: - return True + for f in files: + if f.startswith("tests/integration/test_") and f.endswith(".py"): + test_files.add(f) + elif f.startswith("tests/integration/fixtures/"): + if f.endswith(".yaml"): + # Fixture YAML changed - add corresponding test file(s) + test_files.update(fixture_to_test_files.get(Path(f).stem, ())) + else: + # Non-YAML fixture file changed (e.g., external_components/) + # Run all tests since we can't determine which tests are affected + return (True, []) - return False + # Find test files whose fixtures use any of the changed components + changed_component_set = { + component for file in files if (component := get_component_from_path(file)) + } + if changed_component_set: + test_files.update( + get_integration_test_files_for_components(changed_component_set) + ) + + if test_files: + return (False, sorted(test_files)) + + return (False, []) @cache @@ -682,7 +707,10 @@ def main() -> None: args = parser.parse_args() # Determine what should run - run_integration = should_run_integration_tests(args.branch) + integration_run_all, integration_test_files = determine_integration_tests( + args.branch + ) + run_integration = integration_run_all or bool(integration_test_files) run_clang_tidy = should_run_clang_tidy(args.branch) run_clang_format = should_run_clang_format(args.branch) run_python_linters = should_run_python_linters(args.branch) @@ -810,6 +838,8 @@ def main() -> None: output: dict[str, Any] = { "integration_tests": run_integration, + "integration_tests_run_all": integration_run_all, + "integration_test_files": integration_test_files, "clang_tidy": run_clang_tidy, "clang_tidy_mode": clang_tidy_mode, "clang_format": run_clang_format, diff --git a/script/helpers.py b/script/helpers.py index 6ee286a657..9665af70ec 100644 --- a/script/helpers.py +++ b/script/helpers.py @@ -700,37 +700,141 @@ def get_all_dependencies( return all_components +def _extract_components_from_yaml(config: dict) -> set[str]: + """Extract component names from a parsed YAML config. + + Args: + config: Parsed YAML configuration dictionary + + Returns: + Set of component names found in the config + """ + components: set[str] = set() + + # Add all top-level component keys (skip YAML anchor keys starting with '.') + components.update(k for k in config if isinstance(k, str) and not k.startswith(".")) + + # Add platform values from list entries (e.g., sensor -> platform: template adds "template") + for value in config.values(): + if isinstance(value, list): + components.update( + item["platform"] + for item in value + if isinstance(item, dict) and "platform" in item + ) + + return components + + def get_components_from_integration_fixtures() -> set[str]: """Extract all components used in integration test fixtures. Returns: Set of component names used in integration test fixtures """ + return { + comp + for components in get_components_per_integration_fixture().values() + for comp in components + } + + +@cache +def get_components_per_integration_fixture() -> dict[str, set[str]]: + """Extract components used in each integration test fixture. + + Returns: + Dictionary mapping fixture name (stem) to set of component names + """ from esphome import yaml_util - components: set[str] = set() + result: dict[str, set[str]] = {} fixtures_dir = Path(__file__).parent.parent / "tests" / "integration" / "fixtures" for yaml_file in fixtures_dir.glob("*.yaml"): - config: dict[str, any] | None = yaml_util.load_yaml(yaml_file) + config: dict[str, Any] | None = yaml_util.load_yaml(yaml_file) if not config: continue - # Add all top-level component keys (skip YAML anchor keys starting with '.') - components.update( - k for k in config if isinstance(k, str) and not k.startswith(".") - ) + result[yaml_file.stem] = _extract_components_from_yaml(config) - # Add platform components (e.g., output.template) - for value in config.values(): - if not isinstance(value, list): - continue + return result - for item in value: - if isinstance(item, dict) and "platform" in item: - components.add(item["platform"]) - return components +_TEST_FUNC_RE = re.compile(r"async def (test_\w+)") + + +@cache +def get_fixture_to_test_files() -> dict[str, frozenset[str]]: + """Map integration test fixture names to the test files that use them. + + Returns: + Dictionary mapping fixture name to frozenset of test file paths + (relative to repo root) + """ + integration_dir = Path(__file__).parent.parent / "tests" / "integration" + result: dict[str, set[str]] = {} + + for test_file in integration_dir.glob("test_*.py"): + content = test_file.read_text(encoding="utf-8") + rel_path = test_file.relative_to(Path(__file__).parent.parent).as_posix() + for func in _TEST_FUNC_RE.findall(content): + base_name = func.replace("test_", "").partition("[")[0] + result.setdefault(base_name, set()).add(rel_path) + + return {k: frozenset(v) for k, v in result.items()} + + +@cache +def _get_component_to_integration_test_files() -> dict[str, frozenset[str]]: + """Build index mapping each component to the test files that depend on it. + + Resolves full dependency trees once per fixture, then inverts the mapping + so lookups are O(1) per component. + + Returns: + Dictionary mapping component name to frozenset of test file paths + """ + fixture_components = get_components_per_integration_fixture() + fixture_to_test_files = get_fixture_to_test_files() + + result: dict[str, set[str]] = {} + for fixture_name, components in fixture_components.items(): + test_files = fixture_to_test_files.get(fixture_name) + if not test_files: + continue + # Get full dependency tree for this fixture's components + all_deps = get_all_dependencies(components) + for dep in all_deps: + result.setdefault(dep, set()).update(test_files) + + return {k: frozenset(v) for k, v in result.items()} + + +def get_integration_test_files_for_components( + changed_components: set[str], +) -> list[str]: + """Get integration test file paths that use any of the given components. + + Uses a precomputed component → test files index for O(C) lookup + where C is the number of changed components. + + Args: + changed_components: Set of component names that have changed + + Returns: + Sorted list of test file paths relative to repo root + (e.g., ["tests/integration/test_api.py", ...]) + """ + component_to_tests = _get_component_to_integration_test_files() + + return sorted( + { + test_file + for component in changed_components + for test_file in component_to_tests.get(component, ()) + } + ) def filter_component_and_test_files(file_path: str) -> bool: diff --git a/tests/script/test_determine_jobs.py b/tests/script/test_determine_jobs.py index 61ef8985df..5c81ad374b 100644 --- a/tests/script/test_determine_jobs.py +++ b/tests/script/test_determine_jobs.py @@ -29,9 +29,9 @@ spec.loader.exec_module(determine_jobs) @pytest.fixture -def mock_should_run_integration_tests() -> Generator[Mock, None, None]: - """Mock should_run_integration_tests from helpers.""" - with patch.object(determine_jobs, "should_run_integration_tests") as mock: +def mock_determine_integration_tests() -> Generator[Mock, None, None]: + """Mock determine_integration_tests.""" + with patch.object(determine_jobs, "determine_integration_tests") as mock: yield mock @@ -87,7 +87,7 @@ def clear_determine_jobs_caches() -> None: def test_main_all_tests_should_run( - mock_should_run_integration_tests: Mock, + mock_determine_integration_tests: Mock, mock_should_run_clang_tidy: Mock, mock_should_run_clang_format: Mock, mock_should_run_python_linters: Mock, @@ -100,7 +100,7 @@ def test_main_all_tests_should_run( # Ensure we're not in GITHUB_ACTIONS mode for this test monkeypatch.delenv("GITHUB_ACTIONS", raising=False) - mock_should_run_integration_tests.return_value = True + mock_determine_integration_tests.return_value = (True, []) mock_should_run_clang_tidy.return_value = True mock_should_run_clang_format.return_value = True mock_should_run_python_linters.return_value = True @@ -152,6 +152,8 @@ def test_main_all_tests_should_run( output = json.loads(captured.out) assert output["integration_tests"] is True + assert output["integration_tests_run_all"] is True + assert output["integration_test_files"] == [] assert output["clang_tidy"] is True assert output["clang_tidy_mode"] in ["nosplit", "split"] assert output["clang_format"] is True @@ -183,7 +185,7 @@ def test_main_all_tests_should_run( def test_main_no_tests_should_run( - mock_should_run_integration_tests: Mock, + mock_determine_integration_tests: Mock, mock_should_run_clang_tidy: Mock, mock_should_run_clang_format: Mock, mock_should_run_python_linters: Mock, @@ -196,7 +198,7 @@ def test_main_no_tests_should_run( # Ensure we're not in GITHUB_ACTIONS mode for this test monkeypatch.delenv("GITHUB_ACTIONS", raising=False) - mock_should_run_integration_tests.return_value = False + mock_determine_integration_tests.return_value = (False, []) mock_should_run_clang_tidy.return_value = False mock_should_run_clang_format.return_value = False mock_should_run_python_linters.return_value = False @@ -233,6 +235,8 @@ def test_main_no_tests_should_run( output = json.loads(captured.out) assert output["integration_tests"] is False + assert output["integration_tests_run_all"] is False + assert output["integration_test_files"] == [] assert output["clang_tidy"] is False assert output["clang_tidy_mode"] == "disabled" assert output["clang_format"] is False @@ -253,7 +257,7 @@ def test_main_no_tests_should_run( def test_main_with_branch_argument( - mock_should_run_integration_tests: Mock, + mock_determine_integration_tests: Mock, mock_should_run_clang_tidy: Mock, mock_should_run_clang_format: Mock, mock_should_run_python_linters: Mock, @@ -266,7 +270,7 @@ def test_main_with_branch_argument( # Ensure we're not in GITHUB_ACTIONS mode for this test monkeypatch.delenv("GITHUB_ACTIONS", raising=False) - mock_should_run_integration_tests.return_value = False + mock_determine_integration_tests.return_value = (False, []) mock_should_run_clang_tidy.return_value = True mock_should_run_clang_format.return_value = False mock_should_run_python_linters.return_value = True @@ -302,7 +306,7 @@ def test_main_with_branch_argument( determine_jobs.main() # Check that functions were called with branch - mock_should_run_integration_tests.assert_called_once_with("main") + mock_determine_integration_tests.assert_called_once_with("main") mock_should_run_clang_tidy.assert_called_once_with("main") mock_should_run_clang_format.assert_called_once_with("main") mock_should_run_python_linters.assert_called_once_with("main") @@ -312,6 +316,8 @@ def test_main_with_branch_argument( output = json.loads(captured.out) assert output["integration_tests"] is False + assert output["integration_tests_run_all"] is False + assert output["integration_test_files"] == [] assert output["clang_tidy"] is True assert output["clang_tidy_mode"] in ["nosplit", "split"] assert output["clang_format"] is False @@ -334,30 +340,33 @@ def test_main_with_branch_argument( assert output["cpp_unit_tests_components"] == ["mqtt"] -def test_should_run_integration_tests( +def test_determine_integration_tests( monkeypatch: pytest.MonkeyPatch, ) -> None: - """Test should_run_integration_tests function.""" - # Core C++ files trigger tests + """Test determine_integration_tests function.""" + # Core C++ files trigger run_all with patch.object( determine_jobs, "changed_files", return_value=["esphome/core/component.cpp"] ): - result = determine_jobs.should_run_integration_tests() - assert result is True + run_all, test_files = determine_jobs.determine_integration_tests() + assert run_all is True + assert test_files == [] - # Core Python files trigger tests + # Core Python files trigger run_all with patch.object( determine_jobs, "changed_files", return_value=["esphome/core/config.py"] ): - result = determine_jobs.should_run_integration_tests() - assert result is True + run_all, test_files = determine_jobs.determine_integration_tests() + assert run_all is True + assert test_files == [] # Python files directly in esphome/ do NOT trigger tests with patch.object( determine_jobs, "changed_files", return_value=["esphome/config.py"] ): - result = determine_jobs.should_run_integration_tests() - assert result is False + run_all, test_files = determine_jobs.determine_integration_tests() + assert run_all is False + assert test_files == [] # Python files in subdirectories (not core) do NOT trigger tests with patch.object( @@ -365,35 +374,151 @@ def test_should_run_integration_tests( "changed_files", return_value=["esphome/dashboard/web_server.py"], ): - result = determine_jobs.should_run_integration_tests() - assert result is False + run_all, test_files = determine_jobs.determine_integration_tests() + assert run_all is False + assert test_files == [] -def test_should_run_integration_tests_with_branch() -> None: - """Test should_run_integration_tests with branch argument.""" +def test_determine_integration_tests_with_branch() -> None: + """Test determine_integration_tests with branch argument.""" with patch.object(determine_jobs, "changed_files") as mock_changed: mock_changed.return_value = [] - determine_jobs.should_run_integration_tests("release") + run_all, test_files = determine_jobs.determine_integration_tests("release") mock_changed.assert_called_once_with("release") + assert run_all is False + assert test_files == [] -def test_should_run_integration_tests_component_dependency() -> None: - """Test that integration tests run when components used in fixtures change.""" +def test_determine_integration_tests_component_dependency() -> None: + """Test that integration tests return specific test files when components used in fixtures change.""" with ( patch.object( determine_jobs, "changed_files", return_value=["esphome/components/api/api.cpp"], ), + patch.object(determine_jobs, "get_fixture_to_test_files") as mock_fixture_map, patch.object( - determine_jobs, "get_components_from_integration_fixtures" - ) as mock_fixtures, + determine_jobs, "get_integration_test_files_for_components" + ) as mock_test_files, ): - mock_fixtures.return_value = {"api", "sensor"} - with patch.object(determine_jobs, "get_all_dependencies") as mock_deps: - mock_deps.return_value = {"api", "sensor", "network"} - result = determine_jobs.should_run_integration_tests() - assert result is True + mock_fixture_map.return_value = {} + mock_test_files.return_value = [ + "tests/integration/test_api.py", + "tests/integration/test_sensor.py", + ] + run_all, test_files = determine_jobs.determine_integration_tests() + assert run_all is False + assert test_files == [ + "tests/integration/test_api.py", + "tests/integration/test_sensor.py", + ] + + +def test_determine_integration_tests_component_only_affected_tests() -> None: + """Test that only tests using the changed component are returned.""" + with ( + patch.object( + determine_jobs, + "changed_files", + return_value=["esphome/components/modbus/modbus.cpp"], + ), + patch.object(determine_jobs, "get_fixture_to_test_files", return_value={}), + patch.object( + determine_jobs, "get_integration_test_files_for_components" + ) as mock_test_files, + ): + mock_test_files.return_value = [ + "tests/integration/test_uart_mock_modbus.py", + ] + run_all, test_files = determine_jobs.determine_integration_tests() + assert run_all is False + assert test_files == ["tests/integration/test_uart_mock_modbus.py"] + # Verify it was called with the right component + mock_test_files.assert_called_once_with({"modbus"}) + + +def test_determine_integration_tests_infra_file_runs_all() -> None: + """Test that changing infrastructure files (conftest.py, etc.) runs all tests.""" + with patch.object( + determine_jobs, + "changed_files", + return_value=["tests/integration/conftest.py"], + ): + run_all, test_files = determine_jobs.determine_integration_tests() + assert run_all is True + assert test_files == [] + + +def test_determine_integration_tests_readme_does_not_run_all() -> None: + """Test that changing README.md does not trigger integration tests.""" + with patch.object( + determine_jobs, + "changed_files", + return_value=["tests/integration/README.md"], + ): + run_all, test_files = determine_jobs.determine_integration_tests() + assert run_all is False + assert test_files == [] + + +def test_determine_integration_tests_changed_test_file() -> None: + """Test that changing a specific test file only runs that test.""" + with ( + patch.object( + determine_jobs, + "changed_files", + return_value=["tests/integration/test_syslog.py"], + ), + patch.object(determine_jobs, "get_fixture_to_test_files", return_value={}), + patch.object( + determine_jobs, + "get_integration_test_files_for_components", + return_value=[], + ), + ): + run_all, test_files = determine_jobs.determine_integration_tests() + assert run_all is False + assert test_files == ["tests/integration/test_syslog.py"] + + +def test_determine_integration_tests_changed_fixture_yaml() -> None: + """Test that changing a fixture YAML runs the corresponding test file.""" + with ( + patch.object( + determine_jobs, + "changed_files", + return_value=["tests/integration/fixtures/uart_mock_modbus.yaml"], + ), + patch.object(determine_jobs, "get_fixture_to_test_files") as mock_fixture_map, + patch.object( + determine_jobs, + "get_integration_test_files_for_components", + return_value=[], + ), + ): + mock_fixture_map.return_value = { + "uart_mock_modbus": frozenset( + {"tests/integration/test_uart_mock_modbus.py"} + ), + } + run_all, test_files = determine_jobs.determine_integration_tests() + assert run_all is False + assert test_files == ["tests/integration/test_uart_mock_modbus.py"] + + +def test_determine_integration_tests_non_yaml_fixture_runs_all() -> None: + """Test that non-YAML changes under fixtures/ (e.g., external_components) run all tests.""" + with patch.object( + determine_jobs, + "changed_files", + return_value=[ + "tests/integration/fixtures/external_components/test_component/__init__.py" + ], + ): + run_all, test_files = determine_jobs.determine_integration_tests() + assert run_all is True + assert test_files == [] @pytest.mark.parametrize( @@ -538,7 +663,7 @@ def test_count_changed_cpp_files_with_branch() -> None: def test_main_filters_components_without_tests( - mock_should_run_integration_tests: Mock, + mock_determine_integration_tests: Mock, mock_should_run_clang_tidy: Mock, mock_should_run_clang_format: Mock, mock_should_run_python_linters: Mock, @@ -551,7 +676,7 @@ def test_main_filters_components_without_tests( # Ensure we're not in GITHUB_ACTIONS mode for this test monkeypatch.delenv("GITHUB_ACTIONS", raising=False) - mock_should_run_integration_tests.return_value = False + mock_determine_integration_tests.return_value = (False, []) mock_should_run_clang_tidy.return_value = False mock_should_run_clang_format.return_value = False mock_should_run_python_linters.return_value = False @@ -631,7 +756,7 @@ def test_main_filters_components_without_tests( def test_main_detects_components_with_variant_tests( - mock_should_run_integration_tests: Mock, + mock_determine_integration_tests: Mock, mock_should_run_clang_tidy: Mock, mock_should_run_clang_format: Mock, mock_should_run_python_linters: Mock, @@ -649,7 +774,7 @@ def test_main_detects_components_with_variant_tests( # Ensure we're not in GITHUB_ACTIONS mode for this test monkeypatch.delenv("GITHUB_ACTIONS", raising=False) - mock_should_run_integration_tests.return_value = False + mock_determine_integration_tests.return_value = (False, []) mock_should_run_clang_tidy.return_value = False mock_should_run_clang_format.return_value = False mock_should_run_python_linters.return_value = False @@ -999,7 +1124,7 @@ def test_detect_memory_impact_config_with_variant_tests(tmp_path: Path) -> None: def test_clang_tidy_mode_full_scan( - mock_should_run_integration_tests: Mock, + mock_determine_integration_tests: Mock, mock_should_run_clang_tidy: Mock, mock_should_run_clang_format: Mock, mock_should_run_python_linters: Mock, @@ -1010,7 +1135,7 @@ def test_clang_tidy_mode_full_scan( """Test that full scan (hash changed) always uses split mode.""" monkeypatch.delenv("GITHUB_ACTIONS", raising=False) - mock_should_run_integration_tests.return_value = False + mock_determine_integration_tests.return_value = (False, []) mock_should_run_clang_tidy.return_value = True mock_should_run_clang_format.return_value = False mock_should_run_python_linters.return_value = False @@ -1065,7 +1190,7 @@ def test_clang_tidy_mode_targeted_scan( component_count: int, files_per_component: int, expected_mode: str, - mock_should_run_integration_tests: Mock, + mock_determine_integration_tests: Mock, mock_should_run_clang_tidy: Mock, mock_should_run_clang_format: Mock, mock_should_run_python_linters: Mock, @@ -1076,7 +1201,7 @@ def test_clang_tidy_mode_targeted_scan( """Test clang-tidy mode selection based on files_to_check count.""" monkeypatch.delenv("GITHUB_ACTIONS", raising=False) - mock_should_run_integration_tests.return_value = False + mock_determine_integration_tests.return_value = (False, []) mock_should_run_clang_tidy.return_value = True mock_should_run_clang_format.return_value = False mock_should_run_python_linters.return_value = False @@ -1123,7 +1248,7 @@ def test_clang_tidy_mode_targeted_scan( def test_main_core_files_changed_still_detects_components( - mock_should_run_integration_tests: Mock, + mock_determine_integration_tests: Mock, mock_should_run_clang_tidy: Mock, mock_should_run_clang_format: Mock, mock_should_run_python_linters: Mock, @@ -1135,7 +1260,7 @@ def test_main_core_files_changed_still_detects_components( """Test that component changes are detected even when core files change.""" monkeypatch.delenv("GITHUB_ACTIONS", raising=False) - mock_should_run_integration_tests.return_value = True + mock_determine_integration_tests.return_value = (True, []) mock_should_run_clang_tidy.return_value = True mock_should_run_clang_format.return_value = True mock_should_run_python_linters.return_value = True @@ -1604,7 +1729,7 @@ def test_detect_platform_hint_from_filename_case_insensitive( def test_component_batching_beta_branch_40_per_batch( tmp_path: Path, - mock_should_run_integration_tests: Mock, + mock_determine_integration_tests: Mock, mock_should_run_clang_tidy: Mock, mock_should_run_clang_format: Mock, mock_should_run_python_linters: Mock, @@ -1628,7 +1753,7 @@ def test_component_batching_beta_branch_40_per_batch( (comp_dir / "test.esp32-idf.yaml").write_text(f"# Test for {comp}") # Setup mocks - mock_should_run_integration_tests.return_value = False + mock_determine_integration_tests.return_value = (False, []) mock_should_run_clang_tidy.return_value = False mock_should_run_clang_format.return_value = False mock_should_run_python_linters.return_value = False diff --git a/tests/script/test_helpers.py b/tests/script/test_helpers.py index 781054eb3b..e3802d2d51 100644 --- a/tests/script/test_helpers.py +++ b/tests/script/test_helpers.py @@ -36,6 +36,7 @@ def clear_helpers_cache() -> None: """Clear cached functions before each test.""" helpers._get_github_event_data.cache_clear() helpers._get_changed_files_github_actions.cache_clear() + helpers.get_components_per_integration_fixture.cache_clear() @pytest.mark.parametrize( @@ -1111,7 +1112,7 @@ def test_get_components_from_integration_fixtures() -> None: "gpio", } - mock_yaml_file = Mock() + mock_yaml_file = Mock(stem="test_fixture") with ( patch("pathlib.Path.glob") as mock_glob, @@ -1133,7 +1134,7 @@ def test_get_components_from_integration_fixtures_skips_yaml_anchors() -> None: ".binary_filters": {"filters": [{"settle": "50ms"}]}, } - mock_yaml_file = Mock() + mock_yaml_file = Mock(stem="test_fixture") with ( patch("pathlib.Path.glob") as mock_glob, @@ -1148,6 +1149,31 @@ def test_get_components_from_integration_fixtures_skips_yaml_anchors() -> None: assert components == {"sensor", "esphome", "template"} +def test_get_integration_test_files_for_components_real_fixtures() -> None: + """Test that component changes map to the correct real integration test files. + + This test uses real fixtures to verify the mapping stays correct + as new tests are added. + """ + # modbus should include at least the modbus test + modbus_tests = helpers.get_integration_test_files_for_components({"modbus"}) + assert "tests/integration/test_uart_mock_modbus.py" in modbus_tests + + # ld2410 should include at least the ld2410 test + ld2410_tests = helpers.get_integration_test_files_for_components({"ld2410"}) + assert "tests/integration/test_uart_mock_ld2410.py" in ld2410_tests + + # syslog should include at least the syslog test + syslog_tests = helpers.get_integration_test_files_for_components({"syslog"}) + assert "tests/integration/test_syslog.py" in syslog_tests + + # A component not used by any fixture should return nothing + fake_tests = helpers.get_integration_test_files_for_components( + {"nonexistent_component_xyz"} + ) + assert fake_tests == [] + + @pytest.mark.parametrize( "output,expected", [ From 7cceb72cc310e784acd249ff1108793abe36293f Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 13 Mar 2026 13:23:41 -1000 Subject: [PATCH 3/6] [api] Inline force-variant ProtoSize calc methods (#14781) --- esphome/components/api/proto.h | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/esphome/components/api/proto.h b/esphome/components/api/proto.h index d1c955b1fb..814a3f4456 100644 --- a/esphome/components/api/proto.h +++ b/esphome/components/api/proto.h @@ -602,7 +602,7 @@ class ProtoSize { static constexpr uint32_t calc_sint32(uint32_t field_id_size, int32_t value) { return value ? field_id_size + varint(encode_zigzag32(value)) : 0; } - static constexpr uint32_t calc_sint32_force(uint32_t field_id_size, int32_t value) { + static constexpr inline uint32_t ESPHOME_ALWAYS_INLINE calc_sint32_force(uint32_t field_id_size, int32_t value) { return field_id_size + varint(encode_zigzag32(value)); } static constexpr uint32_t calc_int64(uint32_t field_id_size, int64_t value) { @@ -614,13 +614,13 @@ class ProtoSize { static constexpr uint32_t calc_uint64(uint32_t field_id_size, uint64_t value) { return value ? field_id_size + varint(value) : 0; } - static constexpr uint32_t calc_uint64_force(uint32_t field_id_size, uint64_t value) { + static constexpr inline uint32_t ESPHOME_ALWAYS_INLINE calc_uint64_force(uint32_t field_id_size, uint64_t value) { return field_id_size + varint(value); } static constexpr uint32_t calc_length(uint32_t field_id_size, size_t len) { return len ? field_id_size + varint(static_cast(len)) + static_cast(len) : 0; } - static constexpr uint32_t calc_length_force(uint32_t field_id_size, size_t len) { + static constexpr inline uint32_t ESPHOME_ALWAYS_INLINE calc_length_force(uint32_t field_id_size, size_t len) { return field_id_size + varint(static_cast(len)) + static_cast(len); } static constexpr uint32_t calc_sint64(uint32_t field_id_size, int64_t value) { @@ -638,7 +638,8 @@ class ProtoSize { static constexpr uint32_t calc_message(uint32_t field_id_size, uint32_t nested_size) { return nested_size ? field_id_size + varint(nested_size) + nested_size : 0; } - static constexpr uint32_t calc_message_force(uint32_t field_id_size, uint32_t nested_size) { + static constexpr inline uint32_t ESPHOME_ALWAYS_INLINE calc_message_force(uint32_t field_id_size, + uint32_t nested_size) { return field_id_size + varint(nested_size) + nested_size; } }; From 86b79330815c75cb3cd980afecfe80eb87b1026a Mon Sep 17 00:00:00 2001 From: Jonathan Swoboda <154711427+swoboda1337@users.noreply.github.com> Date: Fri, 13 Mar 2026 19:24:41 -0400 Subject: [PATCH 4/6] [esp32_rmt_led_strip][remote_transmitter][remote_receiver] Fix ESP-IDF 6.0 RMT compatibility (#14783) Co-authored-by: Claude Opus 4.6 --- .../components/esp32_rmt_led_strip/led_strip.cpp | 2 -- .../remote_receiver/remote_receiver_rmt.cpp | 1 - .../remote_transmitter/remote_transmitter_rmt.cpp | 13 +++++++++++-- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/esphome/components/esp32_rmt_led_strip/led_strip.cpp b/esphome/components/esp32_rmt_led_strip/led_strip.cpp index 66b41931aa..ca97a181fd 100644 --- a/esphome/components/esp32_rmt_led_strip/led_strip.cpp +++ b/esphome/components/esp32_rmt_led_strip/led_strip.cpp @@ -99,8 +99,6 @@ void ESP32RMTLEDStripLightOutput::setup() { channel.gpio_num = gpio_num_t(this->pin_); channel.mem_block_symbols = this->rmt_symbols_; channel.trans_queue_depth = 1; - channel.flags.io_loop_back = 0; - channel.flags.io_od_mode = 0; channel.flags.invert_out = this->invert_out_; channel.flags.with_dma = this->use_dma_; channel.intr_priority = 0; diff --git a/esphome/components/remote_receiver/remote_receiver_rmt.cpp b/esphome/components/remote_receiver/remote_receiver_rmt.cpp index 96b23bd0f5..596608a4d0 100644 --- a/esphome/components/remote_receiver/remote_receiver_rmt.cpp +++ b/esphome/components/remote_receiver/remote_receiver_rmt.cpp @@ -44,7 +44,6 @@ void RemoteReceiverComponent::setup() { channel.intr_priority = 0; channel.flags.invert_in = 0; channel.flags.with_dma = this->with_dma_; - channel.flags.io_loop_back = 0; esp_err_t error = rmt_new_rx_channel(&channel, &this->channel_); if (error != ESP_OK) { this->error_code_ = error; diff --git a/esphome/components/remote_transmitter/remote_transmitter_rmt.cpp b/esphome/components/remote_transmitter/remote_transmitter_rmt.cpp index 71773e3ddf..3c9a12d472 100644 --- a/esphome/components/remote_transmitter/remote_transmitter_rmt.cpp +++ b/esphome/components/remote_transmitter/remote_transmitter_rmt.cpp @@ -120,11 +120,13 @@ void RemoteTransmitterComponent::configure_rmt_() { channel.gpio_num = gpio_num_t(this->pin_->get_pin()); channel.mem_block_symbols = this->rmt_symbols_; channel.trans_queue_depth = 1; - channel.flags.io_loop_back = open_drain; - channel.flags.io_od_mode = open_drain; channel.flags.invert_out = 0; channel.flags.with_dma = this->with_dma_; channel.intr_priority = 0; +#if ESP_IDF_VERSION < ESP_IDF_VERSION_VAL(6, 0, 0) + channel.flags.io_loop_back = open_drain; + channel.flags.io_od_mode = open_drain; +#endif error = rmt_new_tx_channel(&channel, &this->channel_); if (error != ESP_OK) { this->error_code_ = error; @@ -136,6 +138,13 @@ void RemoteTransmitterComponent::configure_rmt_() { this->mark_failed(); return; } +#if ESP_IDF_VERSION >= ESP_IDF_VERSION_VAL(6, 0, 0) + if (open_drain) { + gpio_num_t gpio = gpio_num_t(this->pin_->get_pin()); + gpio_od_enable(gpio); + gpio_input_enable(gpio); + } +#endif if (this->pin_->get_flags() & gpio::FLAG_PULLUP) { gpio_pullup_en(gpio_num_t(this->pin_->get_pin())); } else { From d6d3bbbad8f6e14a78c2f6c7505bdbfcbd3dad2d Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 13 Mar 2026 13:28:34 -1000 Subject: [PATCH 5/6] [scheduler] Use integer math for interval offset calculation (#14755) --- esphome/core/scheduler.cpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/esphome/core/scheduler.cpp b/esphome/core/scheduler.cpp index 63e1006b03..72b183384e 100644 --- a/esphome/core/scheduler.cpp +++ b/esphome/core/scheduler.cpp @@ -105,10 +105,11 @@ static void validate_static_string(const char *name) { // avoid the main thread modifying the list while it is being accessed. // Calculate random offset for interval timers -// Extracted from set_timer_common_ to reduce code size - float math + random_float() -// only needed for intervals, not timeouts +// Extracted from set_timer_common_ to reduce code size - only needed for intervals, not timeouts uint32_t Scheduler::calculate_interval_offset_(uint32_t delay) { - return static_cast(std::min(delay / 2, MAX_INTERVAL_DELAY) * random_float()); + uint32_t max_offset = std::min(delay / 2, MAX_INTERVAL_DELAY); + // Multiply-and-shift: uniform random in [0, max_offset) without floating point + return static_cast((static_cast(random_uint32()) * max_offset) >> 32); } // Check if a retry was already cancelled in items_ or to_add_ From 5e3c44d48fc5e7870725e5572accf0d0b0cd9de0 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 13 Mar 2026 13:28:55 -1000 Subject: [PATCH 6/6] [rp2040] Add CI check for boards.py freshness (#14754) Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- .github/workflows/ci.yml | 1 + esphome/components/rp2040/generate_boards.py | 12 +++- script/generate-rp2040-boards.py | 61 ++++++++++++++++++++ 3 files changed, 73 insertions(+), 1 deletion(-) create mode 100755 script/generate-rp2040-boards.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fedfebf393..f7710589c5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -106,6 +106,7 @@ jobs: script/build_codeowners.py --check script/build_language_schema.py --check script/generate-esp32-boards.py --check + script/generate-rp2040-boards.py --check pytest: name: Run pytest diff --git a/esphome/components/rp2040/generate_boards.py b/esphome/components/rp2040/generate_boards.py index a0e3699f37..7ea02d185e 100644 --- a/esphome/components/rp2040/generate_boards.py +++ b/esphome/components/rp2040/generate_boards.py @@ -6,6 +6,7 @@ Usage: python esphome/components/rp2040/generate_boards.py import json from pathlib import Path import re +import subprocess import sys from jinja2 import Environment, FileSystemLoader @@ -157,7 +158,7 @@ def generate(arduino_pico_path: Path) -> str: board_pins, boards = load_boards(arduino_pico_path) template = _jinja_env.get_template("boards.jinja2") - return template.render( + content = template.render( cyw43_gpio_offset=CYW43_GPIO_OFFSET, cyw43_max_gpio=CYW43_GPIO_OFFSET + CYW43_GPIO_COUNT - 1, default_max_pin=DEFAULT_MAX_PIN, @@ -165,6 +166,15 @@ def generate(arduino_pico_path: Path) -> str: boards=sorted(boards.items()), ) + # Format output to match pre-commit ruff formatting + result = subprocess.run( + [sys.executable, "-m", "ruff", "format", "--stdin-filename", "boards.py"], + input=content.encode(), + capture_output=True, + check=True, + ) + return result.stdout.decode() + def main(): if len(sys.argv) < 2: diff --git a/script/generate-rp2040-boards.py b/script/generate-rp2040-boards.py new file mode 100755 index 0000000000..1b4846fd2b --- /dev/null +++ b/script/generate-rp2040-boards.py @@ -0,0 +1,61 @@ +#!/usr/bin/env python3 + +from __future__ import annotations + +import argparse +from pathlib import Path +import subprocess +import sys +import tempfile + +from esphome.components.rp2040 import RECOMMENDED_ARDUINO_FRAMEWORK_VERSION +from esphome.components.rp2040.generate_boards import generate +from esphome.helpers import write_file_if_changed + +ver = RECOMMENDED_ARDUINO_FRAMEWORK_VERSION +version_tag: str = f"{ver.major}.{ver.minor}.{ver.patch}" +root: Path = Path(__file__).parent.parent +boards_file_path: Path = root / "esphome" / "components" / "rp2040" / "boards.py" + + +def main(check: bool) -> None: + with tempfile.TemporaryDirectory() as tempdir: + subprocess.run( + [ + "git", + "clone", + "-q", + "-c", + "advice.detachedHead=false", + "--depth", + "1", + "--branch", + version_tag, + "https://github.com/earlephilhower/arduino-pico", + tempdir, + ], + check=True, + ) + + content: str = generate(Path(tempdir)) + + if check: + existing_content: str = boards_file_path.read_text(encoding="utf-8") + if existing_content != content: + print("esphome/components/rp2040/boards.py is not up to date.") + print("Please run `script/generate-rp2040-boards.py`") + sys.exit(1) + print("esphome/components/rp2040/boards.py is up to date") + elif write_file_if_changed(boards_file_path, content): + print("RP2040 boards updated successfully.") + + +if __name__ == "__main__": + parser: argparse.ArgumentParser = argparse.ArgumentParser() + parser.add_argument( + "--check", + help="Check if the boards.py file is up to date.", + action="store_true", + ) + args: argparse.Namespace = parser.parse_args() + main(args.check)