From a92b607754c3babd607f3e689bb017955485072f Mon Sep 17 00:00:00 2001 From: Jonathan Swoboda <154711427+swoboda1337@users.noreply.github.com> Date: Thu, 14 May 2026 18:54:13 -0400 Subject: [PATCH] [ci] Add ci-run-all label to force full CI matrix (#16421) --- .github/workflows/ci.yml | 39 ++++++-- script/determine-jobs.py | 79 ++++++++++++---- tests/script/test_determine_jobs.py | 139 ++++++++++++++++++++++++++++ 3 files changed, 231 insertions(+), 26 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 819dac926e..abd2d1b3a5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -249,6 +249,7 @@ jobs: integration-test-buckets: ${{ steps.determine.outputs.integration-test-buckets }} clang-tidy: ${{ steps.determine.outputs.clang-tidy }} clang-tidy-mode: ${{ steps.determine.outputs.clang-tidy-mode }} + clang-tidy-full-scan: ${{ steps.determine.outputs.clang-tidy-full-scan }} python-linters: ${{ steps.determine.outputs.python-linters }} import-time: ${{ steps.determine.outputs.import-time }} device-builder: ${{ steps.determine.outputs.device-builder }} @@ -287,7 +288,12 @@ jobs: GH_TOKEN: ${{ github.token }} run: | . venv/bin/activate - output=$(python script/determine-jobs.py) + EXTRA_ARGS="" + if [[ "${{ contains(github.event.pull_request.labels.*.name, 'ci-run-all') }}" == "true" ]]; then + EXTRA_ARGS="--force-all" + echo "::notice::ci-run-all label detected -- forcing every CI job to run" + fi + output=$(python script/determine-jobs.py $EXTRA_ARGS) echo "Test determination output:" echo "$output" | jq @@ -296,6 +302,7 @@ jobs: echo "integration-test-buckets=$(echo "$output" | jq -c '.integration_test_buckets')" >> $GITHUB_OUTPUT echo "clang-tidy=$(echo "$output" | jq -r '.clang_tidy')" >> $GITHUB_OUTPUT echo "clang-tidy-mode=$(echo "$output" | jq -r '.clang_tidy_mode')" >> $GITHUB_OUTPUT + echo "clang-tidy-full-scan=$(echo "$output" | jq -r '.clang_tidy_full_scan')" >> $GITHUB_OUTPUT echo "python-linters=$(echo "$output" | jq -r '.python_linters')" >> $GITHUB_OUTPUT echo "import-time=$(echo "$output" | jq -r '.import_time')" >> $GITHUB_OUTPUT echo "device-builder=$(echo "$output" | jq -r '.device_builder')" >> $GITHUB_OUTPUT @@ -500,7 +507,13 @@ jobs: id: check_full_scan run: | . venv/bin/activate - if python script/clang_tidy_hash.py --check; then + # determine-jobs.clang-tidy-full-scan is true when core C++ changed + # OR the ci-run-all label forced --force-all. Independent of the + # hash check, both must produce a full scan in the job itself. + if [ "${{ needs.determine-jobs.outputs.clang-tidy-full-scan }}" = "true" ]; then + echo "full_scan=true" >> $GITHUB_OUTPUT + echo "reason=determine_jobs" >> $GITHUB_OUTPUT + elif python script/clang_tidy_hash.py --check; then echo "full_scan=true" >> $GITHUB_OUTPUT echo "reason=hash_changed" >> $GITHUB_OUTPUT else @@ -512,7 +525,7 @@ jobs: run: | . venv/bin/activate if [ "${{ steps.check_full_scan.outputs.full_scan }}" = "true" ]; then - echo "Running FULL clang-tidy scan (hash changed)" + echo "Running FULL clang-tidy scan (reason: ${{ steps.check_full_scan.outputs.reason }})" script/clang-tidy --all-headers --fix ${{ matrix.options }} ${{ matrix.ignore_errors && '|| true' || '' }} else echo "Running clang-tidy on changed files only" @@ -572,7 +585,13 @@ jobs: id: check_full_scan run: | . venv/bin/activate - if python script/clang_tidy_hash.py --check; then + # determine-jobs.clang-tidy-full-scan is true when core C++ changed + # OR the ci-run-all label forced --force-all. Independent of the + # hash check, both must produce a full scan in the job itself. + if [ "${{ needs.determine-jobs.outputs.clang-tidy-full-scan }}" = "true" ]; then + echo "full_scan=true" >> $GITHUB_OUTPUT + echo "reason=determine_jobs" >> $GITHUB_OUTPUT + elif python script/clang_tidy_hash.py --check; then echo "full_scan=true" >> $GITHUB_OUTPUT echo "reason=hash_changed" >> $GITHUB_OUTPUT else @@ -584,7 +603,7 @@ jobs: run: | . venv/bin/activate if [ "${{ steps.check_full_scan.outputs.full_scan }}" = "true" ]; then - echo "Running FULL clang-tidy scan (hash changed)" + echo "Running FULL clang-tidy scan (reason: ${{ steps.check_full_scan.outputs.reason }})" script/clang-tidy --all-headers --fix --environment esp32-arduino-tidy else echo "Running clang-tidy on changed files only" @@ -661,7 +680,13 @@ jobs: id: check_full_scan run: | . venv/bin/activate - if python script/clang_tidy_hash.py --check; then + # determine-jobs.clang-tidy-full-scan is true when core C++ changed + # OR the ci-run-all label forced --force-all. Independent of the + # hash check, both must produce a full scan in the job itself. + if [ "${{ needs.determine-jobs.outputs.clang-tidy-full-scan }}" = "true" ]; then + echo "full_scan=true" >> $GITHUB_OUTPUT + echo "reason=determine_jobs" >> $GITHUB_OUTPUT + elif python script/clang_tidy_hash.py --check; then echo "full_scan=true" >> $GITHUB_OUTPUT echo "reason=hash_changed" >> $GITHUB_OUTPUT else @@ -673,7 +698,7 @@ jobs: run: | . venv/bin/activate if [ "${{ steps.check_full_scan.outputs.full_scan }}" = "true" ]; then - echo "Running FULL clang-tidy scan (hash changed)" + echo "Running FULL clang-tidy scan (reason: ${{ steps.check_full_scan.outputs.reason }})" script/clang-tidy --all-headers --fix ${{ matrix.options }} else echo "Running clang-tidy on changed files only" diff --git a/script/determine-jobs.py b/script/determine-jobs.py index 0a55b2a848..3259fb5836 100755 --- a/script/determine-jobs.py +++ b/script/determine-jobs.py @@ -1062,22 +1062,42 @@ def main() -> None: parser.add_argument( "-b", "--branch", help="Branch to compare changed files against" ) + parser.add_argument( + "--force-all", + action="store_true", + help=( + "Force every job to run regardless of what changed. Used by CI " + "when the ci-run-all label is applied to a PR (escape hatch for " + "changes that need full-matrix validation but don't touch enough " + "files to trigger it organically)." + ), + ) args = parser.parse_args() # Determine what should run - integration_run_all, integration_test_files = determine_integration_tests( - args.branch - ) + if args.force_all: + integration_run_all, integration_test_files = True, [] + run_clang_tidy = True + run_clang_format = True + run_python_linters = True + run_import_time = True + run_device_builder = True + native_idf_components = sorted(NATIVE_IDF_TEST_COMPONENTS) + run_native_idf = True + else: + integration_run_all, integration_test_files = determine_integration_tests( + args.branch + ) + run_clang_tidy = should_run_clang_tidy(args.branch) + run_clang_format = should_run_clang_format(args.branch) + run_python_linters = should_run_python_linters(args.branch) + run_import_time = should_run_import_time(args.branch) + run_device_builder = should_run_device_builder(args.branch) + native_idf_components = native_idf_components_to_test(args.branch) + run_native_idf = bool(native_idf_components) run_integration, integration_test_buckets = _compute_integration_test_buckets( integration_run_all, integration_test_files ) - run_clang_tidy = should_run_clang_tidy(args.branch) - run_clang_format = should_run_clang_format(args.branch) - run_python_linters = should_run_python_linters(args.branch) - run_import_time = should_run_import_time(args.branch) - run_device_builder = should_run_device_builder(args.branch) - native_idf_components = native_idf_components_to_test(args.branch) - run_native_idf = bool(native_idf_components) changed_cpp_file_count = count_changed_cpp_files(args.branch) # Get changed components @@ -1106,11 +1126,27 @@ def main() -> None: changed_components = changed_components_result is_core_change = False - # Filter to only components that have test files - # Components without tests shouldn't generate CI test jobs - changed_components_with_tests = [ - component for component in changed_components if _component_has_tests(component) - ] + if args.force_all: + # Force every component with tests into the CI matrix. Each disk entry + # under tests/components/ is treated as a component; filtered + # below by _component_has_tests so components without YAML tests are + # still excluded. + tests_root = Path(root_path) / ESPHOME_TESTS_COMPONENTS_PATH + all_components = sorted(d.name for d in tests_root.iterdir() if d.is_dir()) + changed_components_with_tests = [ + component for component in all_components if _component_has_tests(component) + ] + # Treat as a core change so downstream logic (clang-tidy full scan, + # dep expansion) sees the same world as when esphome/core/ changes. + is_core_change = True + else: + # Filter to only components that have test files + # Components without tests shouldn't generate CI test jobs + changed_components_with_tests = [ + component + for component in changed_components + if _component_has_tests(component) + ] # Get directly changed components with tests (for isolated testing) # These will be tested WITHOUT --testing-mode in CI to enable full validation @@ -1143,8 +1179,10 @@ def main() -> None: memory_impact = detect_memory_impact_config(args.branch) # Determine clang-tidy mode based on actual files that will be checked + is_full_scan = False if run_clang_tidy: # Full scan needed if: hash changed OR core files changed + # (is_core_change is forced True under --force-all) is_full_scan = _is_clang_tidy_full_scan() or is_core_change if is_full_scan: @@ -1177,10 +1215,12 @@ def main() -> None: # Build output # Determine which C++ unit tests to run - cpp_run_all, cpp_components = determine_cpp_unit_tests(args.branch) - - # Determine if benchmarks should run - run_benchmarks = should_run_benchmarks(args.branch) + if args.force_all: + cpp_run_all, cpp_components = True, [] + run_benchmarks = True + else: + cpp_run_all, cpp_components = determine_cpp_unit_tests(args.branch) + run_benchmarks = should_run_benchmarks(args.branch) # Split components into batches for CI testing # This intelligently groups components with similar bus configurations @@ -1219,6 +1259,7 @@ def main() -> None: "integration_test_buckets": integration_test_buckets, "clang_tidy": run_clang_tidy, "clang_tidy_mode": clang_tidy_mode, + "clang_tidy_full_scan": is_full_scan, "clang_format": run_clang_format, "python_linters": run_python_linters, "import_time": run_import_time, diff --git a/tests/script/test_determine_jobs.py b/tests/script/test_determine_jobs.py index 9139c6e095..3fd5eada94 100644 --- a/tests/script/test_determine_jobs.py +++ b/tests/script/test_determine_jobs.py @@ -2602,3 +2602,142 @@ def test_main_validate_only_excludes_transitive_components( # Only foo (directly changed, validate-only). bar is a transitive dep # and still needs compile despite no source change of its own. assert output["validate_only_components"] == ["foo"] + + +def test_main_force_all_overrides_detection( + mock_determine_integration_tests: Mock, + mock_should_run_clang_tidy: Mock, + mock_should_run_clang_format: Mock, + mock_should_run_python_linters: Mock, + mock_should_run_import_time: Mock, + mock_should_run_device_builder: Mock, + mock_native_idf_components_to_test: Mock, + mock_determine_cpp_unit_tests: Mock, + mock_changed_files: Mock, + capsys: pytest.CaptureFixture[str], + monkeypatch: pytest.MonkeyPatch, +) -> None: + """--force-all bypasses per-feature detection and runs every job. + + Detection mocks all return False/empty (which would normally skip + everything) -- the flag must override them. Also verifies clang-tidy + goes to ``split`` (full scan) and the component-test matrix is + populated from disk rather than from changed-files. + """ + monkeypatch.delenv("GITHUB_ACTIONS", raising=False) + + mock_determine_integration_tests.return_value = (False, []) + mock_should_run_clang_tidy.return_value = False + mock_should_run_clang_format.return_value = False + mock_should_run_python_linters.return_value = False + mock_should_run_import_time.return_value = False + mock_should_run_device_builder.return_value = False + mock_native_idf_components_to_test.return_value = [] + mock_determine_cpp_unit_tests.return_value = (False, []) + mock_changed_files.return_value = [] + + with ( + patch("sys.argv", ["determine-jobs.py", "--force-all"]), + patch.object(determine_jobs, "get_changed_components", return_value=[]), + patch.object( + determine_jobs, "filter_component_and_test_files", return_value=False + ), + patch.object( + determine_jobs, "get_components_with_dependencies", return_value=[] + ), + patch.object( + determine_jobs, + "detect_memory_impact_config", + return_value={"should_run": "false"}, + ), + patch.object(determine_jobs, "should_run_benchmarks", return_value=False), + ): + determine_jobs.main() + + output = json.loads(capsys.readouterr().out) + + assert output["integration_tests"] is True + assert output["clang_tidy"] is True + assert output["clang_tidy_mode"] == "split" + assert output["clang_tidy_full_scan"] is True + assert output["clang_format"] is True + assert output["python_linters"] is True + assert output["import_time"] is True + assert output["device_builder"] is True + assert output["native_idf"] is True + # native_idf_components is a CSV of NATIVE_IDF_TEST_COMPONENTS + assert "esp32" in output["native_idf_components"].split(",") + assert output["cpp_unit_tests_run_all"] is True + assert output["cpp_unit_tests_components"] == [] + assert output["benchmarks"] is True + # Detection helpers must not be consulted when --force-all is set + mock_determine_integration_tests.assert_not_called() + mock_should_run_clang_tidy.assert_not_called() + mock_should_run_clang_format.assert_not_called() + mock_should_run_python_linters.assert_not_called() + mock_should_run_import_time.assert_not_called() + mock_should_run_device_builder.assert_not_called() + mock_native_idf_components_to_test.assert_not_called() + mock_determine_cpp_unit_tests.assert_not_called() + # Component matrix is populated from disk (tests/components/ in the repo) + assert output["component_test_count"] > 0 + assert len(output["component_test_batches"]) > 0 + + +def test_main_force_all_off_uses_detection( + mock_determine_integration_tests: Mock, + mock_should_run_clang_tidy: Mock, + mock_should_run_clang_format: Mock, + mock_should_run_python_linters: Mock, + mock_should_run_import_time: Mock, + mock_should_run_device_builder: Mock, + mock_native_idf_components_to_test: Mock, + mock_determine_cpp_unit_tests: Mock, + mock_changed_files: Mock, + capsys: pytest.CaptureFixture[str], + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Without --force-all, detection helpers drive the decision (regression guard).""" + monkeypatch.delenv("GITHUB_ACTIONS", raising=False) + + mock_determine_integration_tests.return_value = (False, []) + mock_should_run_clang_tidy.return_value = False + mock_should_run_clang_format.return_value = False + mock_should_run_python_linters.return_value = False + mock_should_run_import_time.return_value = False + mock_should_run_device_builder.return_value = False + mock_native_idf_components_to_test.return_value = [] + mock_determine_cpp_unit_tests.return_value = (False, []) + mock_changed_files.return_value = [] + + with ( + patch("sys.argv", ["determine-jobs.py"]), + patch.object(determine_jobs, "get_changed_components", return_value=[]), + patch.object( + determine_jobs, "filter_component_and_test_files", return_value=False + ), + patch.object( + determine_jobs, "get_components_with_dependencies", return_value=[] + ), + patch.object( + determine_jobs, + "detect_memory_impact_config", + return_value={"should_run": "false"}, + ), + patch.object( + determine_jobs, "create_intelligent_batches", return_value=([], {}) + ), + patch.object(determine_jobs, "should_run_benchmarks", return_value=False), + ): + determine_jobs.main() + + output = json.loads(capsys.readouterr().out) + + assert output["integration_tests"] is False + assert output["clang_tidy"] is False + assert output["clang_format"] is False + assert output["python_linters"] is False + assert output["native_idf"] is False + assert output["component_test_count"] == 0 + mock_determine_integration_tests.assert_called_once() + mock_should_run_clang_tidy.assert_called_once()