github/esphome
2
0
Fork 0
mirror of https://github.com/esphome/esphome.git synced 2026-10-01 08:50:22 +00:00
Code Issues Packages Projects Releases Wiki Activity

[ci] Run downstream device-builder tests against PR Python code

Browse Source 
Add a CI job that checks out esphome/device-builder@main, installs it,
overlays the PR's esphome via `pip install -e ./esphome`, and runs its
pytest suite. Gated by determine-jobs.py: only runs when Python files
under esphome/ change or runtime deps (requirements.txt, pyproject.toml)
change -- the surface device-builder consumes when it imports esphome.

This catches downstream breakage at PR time instead of after release,
mirroring the yarl -> aiohttp pattern.
This commit is contained in:
J. Nick Koston
2026-05-03 08:49:59 -05:00
parent 8046ff7e1e
commit 365d93f01b
3 changed files with 139 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/script/test_determine_jobs.py
+61
View File
@@ -63,6 +63,13 @@ def mock_should_run_import_time() -> Generator[Mock, None, None]:
yield mock
@pytest.fixture
def mock_should_run_device_builder() -> Generator[Mock, None, None]:
"""Mock should_run_device_builder from determine_jobs."""
with patch.object(determine_jobs, "should_run_device_builder") as mock:
yield mock
@pytest.fixture
def mock_determine_cpp_unit_tests() -> Generator[Mock, None, None]:
"""Mock determine_cpp_unit_tests from helpers."""
@@ -99,6 +106,7 @@ def test_main_all_tests_should_run(
mock_should_run_clang_format: Mock,
mock_should_run_python_linters: Mock,
mock_should_run_import_time: Mock,
mock_should_run_device_builder: Mock,
mock_changed_files: Mock,
mock_determine_cpp_unit_tests: Mock,
capsys: pytest.CaptureFixture[str],
@@ -113,6 +121,7 @@ def test_main_all_tests_should_run(
mock_should_run_clang_format.return_value = True
mock_should_run_python_linters.return_value = True
mock_should_run_import_time.return_value = True
mock_should_run_device_builder.return_value = True
mock_determine_cpp_unit_tests.return_value = (False, ["wifi", "api", "sensor"])
# Mock changed_files to return non-component files (to avoid memory impact)
@@ -193,6 +202,7 @@ def test_main_all_tests_should_run(
assert output["clang_format"] is True
assert output["python_linters"] is True
assert output["import_time"] is True
assert output["device_builder"] is True
assert output["changed_components"] == ["wifi", "api", "sensor"]
# changed_components_with_tests will only include components that actually have test files
assert "changed_components_with_tests" in output
@@ -225,6 +235,7 @@ def test_main_no_tests_should_run(
mock_should_run_clang_format: Mock,
mock_should_run_python_linters: Mock,
mock_should_run_import_time: Mock,
mock_should_run_device_builder: Mock,
mock_changed_files: Mock,
mock_determine_cpp_unit_tests: Mock,
capsys: pytest.CaptureFixture[str],
@@ -239,6 +250,7 @@ def test_main_no_tests_should_run(
mock_should_run_clang_format.return_value = False
mock_should_run_python_linters.return_value = False
mock_should_run_import_time.return_value = False
mock_should_run_device_builder.return_value = False
mock_determine_cpp_unit_tests.return_value = (False, [])
# Mock changed_files to return no component files
@@ -278,6 +290,7 @@ def test_main_no_tests_should_run(
assert output["clang_format"] is False
assert output["python_linters"] is False
assert output["import_time"] is False
assert output["device_builder"] is False
assert output["changed_components"] == []
assert output["changed_components_with_tests"] == []
assert output["component_test_count"] == 0
@@ -299,6 +312,7 @@ def test_main_with_branch_argument(
mock_should_run_clang_format: Mock,
mock_should_run_python_linters: Mock,
mock_should_run_import_time: Mock,
mock_should_run_device_builder: Mock,
mock_changed_files: Mock,
mock_determine_cpp_unit_tests: Mock,
capsys: pytest.CaptureFixture[str],
@@ -313,6 +327,7 @@ def test_main_with_branch_argument(
mock_should_run_clang_format.return_value = False
mock_should_run_python_linters.return_value = True
mock_should_run_import_time.return_value = True
mock_should_run_device_builder.return_value = True
mock_determine_cpp_unit_tests.return_value = (False, ["mqtt"])
# Mock changed_files to return non-component files (to avoid memory impact)
@@ -350,6 +365,7 @@ def test_main_with_branch_argument(
mock_should_run_clang_format.assert_called_once_with("main")
mock_should_run_python_linters.assert_called_once_with("main")
mock_should_run_import_time.assert_called_once_with("main")
mock_should_run_device_builder.assert_called_once_with("main")
# Check output
captured = capsys.readouterr()
@@ -362,6 +378,7 @@ def test_main_with_branch_argument(
assert output["clang_format"] is False
assert output["python_linters"] is True
assert output["import_time"] is True
assert output["device_builder"] is True
assert output["changed_components"] == ["mqtt"]
# changed_components_with_tests will only include components that actually have test files
assert "changed_components_with_tests" in output
@@ -734,6 +751,50 @@ def test_should_run_import_time_with_branch() -> None:
mock_changed.assert_called_once_with("release")
@pytest.mark.parametrize(
("changed_files", "expected_result"),
[
# esphome Python files trigger downstream device-builder tests
(["esphome/__main__.py"], True),
(["esphome/components/wifi/__init__.py"], True),
(["esphome/core/config.py"], True),
(["esphome/types.pyi"], True),
# Runtime dependency changes trigger
(["requirements.txt"], True),
(["pyproject.toml"], True),
# Mixed: any triggering file is enough
(["docs/README.md", "esphome/config.py"], True),
# Dev/test-only dependency changes don't trigger device-builder
# (they don't affect the importable surface device-builder uses)
(["requirements_dev.txt"], False),
(["requirements_test.txt"], False),
# Python files outside esphome/ don't trigger
(["script/some_other_script.py"], False),
(["tests/script/test_determine_jobs.py"], False),
# Non-Python changes don't trigger
(["esphome/core/component.cpp"], False),
(["tests/components/wifi/test.esp32-idf.yaml"], False),
(["README.md"], False),
([], False),
],
)
def test_should_run_device_builder(
changed_files: list[str], expected_result: bool
) -> None:
"""Test should_run_device_builder function."""
with patch.object(determine_jobs, "changed_files", return_value=changed_files):
result = determine_jobs.should_run_device_builder()
assert result == expected_result
def test_should_run_device_builder_with_branch() -> None:
"""Test should_run_device_builder with branch argument."""
with patch.object(determine_jobs, "changed_files") as mock_changed:
mock_changed.return_value = []
determine_jobs.should_run_device_builder("release")
mock_changed.assert_called_once_with("release")
@pytest.mark.parametrize(
("changed_files", "expected_result"),
[