diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 57053c3645..3af1709774 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -199,8 +199,7 @@ jobs: - common outputs: integration-tests: ${{ steps.determine.outputs.integration-tests }} - integration-tests-run-all: ${{ steps.determine.outputs.integration-tests-run-all }} - integration-test-files: ${{ steps.determine.outputs.integration-test-files }} + integration-test-buckets: ${{ steps.determine.outputs.integration-test-buckets }} clang-tidy: ${{ steps.determine.outputs.clang-tidy }} clang-tidy-mode: ${{ steps.determine.outputs.clang-tidy-mode }} python-linters: ${{ steps.determine.outputs.python-linters }} @@ -243,8 +242,7 @@ jobs: # Extract individual fields echo "integration-tests=$(echo "$output" | jq -r '.integration_tests')" >> $GITHUB_OUTPUT - echo "integration-tests-run-all=$(echo "$output" | jq -r '.integration_tests_run_all')" >> $GITHUB_OUTPUT - echo "integration-test-files=$(echo "$output" | jq -c '.integration_test_files')" >> $GITHUB_OUTPUT + echo "integration-test-buckets=$(echo "$output" | jq -c '.integration_test_buckets')" >> $GITHUB_OUTPUT echo "clang-tidy=$(echo "$output" | jq -r '.clang_tidy')" >> $GITHUB_OUTPUT echo "clang-tidy-mode=$(echo "$output" | jq -r '.clang_tidy_mode')" >> $GITHUB_OUTPUT echo "python-linters=$(echo "$output" | jq -r '.python_linters')" >> $GITHUB_OUTPUT @@ -267,12 +265,16 @@ jobs: key: components-graph-${{ hashFiles('esphome/components/**/*.py') }} integration-tests: - name: Run integration tests + name: Run integration tests (${{ matrix.bucket.name }}) runs-on: ubuntu-latest needs: - common - determine-jobs if: needs.determine-jobs.outputs.integration-tests == 'true' + strategy: + fail-fast: false + matrix: + bucket: ${{ fromJson(needs.determine-jobs.outputs.integration-test-buckets) }} steps: - name: Check out code from GitHub uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 @@ -299,19 +301,14 @@ jobs: run: echo "::add-matcher::.github/workflows/matchers/pytest.json" - name: Run integration tests env: - INTEGRATION_TEST_FILES: ${{ needs.determine-jobs.outputs.integration-test-files }} - INTEGRATION_TESTS_RUN_ALL: ${{ needs.determine-jobs.outputs.integration-tests-run-all }} + # JSON array of test paths; parsed into a bash array below to avoid + # shell word-splitting / glob hazards. + BUCKET_TESTS: ${{ toJson(matrix.bucket.tests) }} run: | . venv/bin/activate - if [[ "$INTEGRATION_TESTS_RUN_ALL" == "true" ]]; then - echo "Running all integration tests" - pytest -vv --no-cov --tb=native -n auto tests/integration/ - else - # Parse JSON array into bash array to avoid shell expansion issues - mapfile -t test_files < <(echo "$INTEGRATION_TEST_FILES" | jq -r '.[]') - echo "Running ${#test_files[@]} specific integration tests" - pytest -vv --no-cov --tb=native -n auto "${test_files[@]}" - fi + mapfile -t test_files < <(echo "$BUCKET_TESTS" | jq -r '.[]') + echo "Bucket ${{ matrix.bucket.name }}: running ${#test_files[@]} integration tests" + pytest -vv --no-cov --tb=native -n auto "${test_files[@]}" cpp-unit-tests: name: Run C++ unit tests diff --git a/esphome/components/beken_spi_led_strip/light.py b/esphome/components/beken_spi_led_strip/light.py index 31572cd800..9093b08b62 100644 --- a/esphome/components/beken_spi_led_strip/light.py +++ b/esphome/components/beken_spi_led_strip/light.py @@ -62,6 +62,7 @@ CONF_IS_WRGB = "is_wrgb" SUPPORTED_PINS = { libretiny.const.FAMILY_BK7231N: [16], libretiny.const.FAMILY_BK7231T: [16], + libretiny.const.FAMILY_BK7238: [16], libretiny.const.FAMILY_BK7251: [16], } diff --git a/esphome/components/esp32/printf_stubs.cpp b/esphome/components/esp32/printf_stubs.cpp index 386fbbd79d..908b4023ea 100644 --- a/esphome/components/esp32/printf_stubs.cpp +++ b/esphome/components/esp32/printf_stubs.cpp @@ -13,14 +13,21 @@ * and printf() calls in SDK components are only in debug/assert paths * (gpio_dump_io_configuration, ringbuf diagnostics) that are either * GC'd or never called. Crash backtraces and panic output are - * unaffected — they use esp_rom_printf() which is a ROM function + * unaffected; they use esp_rom_printf() which is a ROM function * and does not go through libc. * - * These stubs redirect through vsnprintf() (which uses _svfprintf_r - * already in the binary) and fwrite(), allowing the linker to - * dead-code eliminate _vfprintf_r. + * On picolibc (default for IDF >= 5 on RISC-V, IDF >= 6 everywhere) we + * route output through a stack-allocated cookie FILE that forwards each + * byte to the real target stream via fputc(). Picolibc's tinystdio + * vfprintf walks the FILE::put callback one character at a time, so this + * costs ~32 bytes of stack for the cookie struct vs. a 512-byte format + * buffer. The buffered path overflows the loopTask stack on IDF 6. * - * Saves ~11 KB of flash. + * On newlib (IDF <= 5 on Xtensa) we keep the original snprintf-then-fwrite + * path because that loopTask stack budget has plenty of headroom for the + * 512-byte buffer; the picolibc-only crash above does not affect it. + * + * Saves ~11 KB of flash on newlib, ~2.8 KB on picolibc. * * To disable these wraps, set enable_full_printf: true in the esp32 * advanced config section. @@ -30,10 +37,55 @@ #include #include +#ifndef __PICOLIBC__ #include "esp_system.h" +#endif namespace esphome::esp32 {} +// NOLINTBEGIN(bugprone-reserved-identifier,cert-dcl37-c,cert-dcl51-cpp,readability-identifier-naming) +extern "C" { + +#ifdef __PICOLIBC__ + +#include +#include + +extern int __real_vfprintf(FILE *stream, const char *fmt, va_list ap); + +namespace { + +struct CookieFile { + FILE base; + FILE *target; +}; + +// cookie_put() recovers CookieFile* from FILE* via reinterpret_cast, which is +// only well-defined when FILE is the first member at offset 0 and CookieFile +// is standard-layout. +static_assert(offsetof(CookieFile, base) == 0, "FILE must be the first member of CookieFile"); +static_assert(std::is_standard_layout::value, "CookieFile must be standard-layout"); + +int cookie_put(char c, FILE *stream) { + auto *cookie = reinterpret_cast(stream); + return fputc(static_cast(c), cookie->target); +} + +const FILE COOKIE_FILE_TEMPLATE = FDEV_SETUP_STREAM(cookie_put, nullptr, nullptr, _FDEV_SETUP_WRITE); + +} // namespace + +int __wrap_vfprintf(FILE *stream, const char *fmt, va_list ap) { + CookieFile cookie; + cookie.base = COOKIE_FILE_TEMPLATE; + cookie.target = stream; + return __real_vfprintf(&cookie.base, fmt, ap); +} + +int __wrap_vprintf(const char *fmt, va_list ap) { return __wrap_vfprintf(stdout, fmt, ap); } + +#else // !__PICOLIBC__ + static constexpr size_t PRINTF_BUFFER_SIZE = 512; // These stubs are essentially dead code at runtime — ESPHome replaces the @@ -55,14 +107,18 @@ static int write_printf_buffer(FILE *stream, char *buf, int len) { return len; } -// NOLINTBEGIN(bugprone-reserved-identifier,cert-dcl37-c,cert-dcl51-cpp,readability-identifier-naming) -extern "C" { - int __wrap_vprintf(const char *fmt, va_list ap) { char buf[PRINTF_BUFFER_SIZE]; return write_printf_buffer(stdout, buf, vsnprintf(buf, sizeof(buf), fmt, ap)); } +int __wrap_vfprintf(FILE *stream, const char *fmt, va_list ap) { + char buf[PRINTF_BUFFER_SIZE]; + return write_printf_buffer(stream, buf, vsnprintf(buf, sizeof(buf), fmt, ap)); +} + +#endif // __PICOLIBC__ + int __wrap_printf(const char *fmt, ...) { va_list ap; va_start(ap, fmt); @@ -71,11 +127,6 @@ int __wrap_printf(const char *fmt, ...) { return len; } -int __wrap_vfprintf(FILE *stream, const char *fmt, va_list ap) { - char buf[PRINTF_BUFFER_SIZE]; - return write_printf_buffer(stream, buf, vsnprintf(buf, sizeof(buf), fmt, ap)); -} - int __wrap_fprintf(FILE *stream, const char *fmt, ...) { va_list ap; va_start(ap, fmt); diff --git a/esphome/components/esp32_hosted/__init__.py b/esphome/components/esp32_hosted/__init__.py index 1619a845d8..eca7c24b10 100644 --- a/esphome/components/esp32_hosted/__init__.py +++ b/esphome/components/esp32_hosted/__init__.py @@ -246,9 +246,10 @@ async def to_code(config): idf_ver = esp32.idf_version() os.environ["ESP_IDF_VERSION"] = f"{idf_ver.major}.{idf_ver.minor}" if idf_ver >= cv.Version(5, 5, 0): - esp32.add_idf_component(name="espressif/esp_wifi_remote", ref="1.4.0") - esp32.add_idf_component(name="espressif/eppp_link", ref="1.1.4") - esp32.add_idf_component(name="espressif/esp_hosted", ref="2.12.1") + esp32.add_idf_component(name="espressif/esp_wifi_remote", ref="1.5.1") + esp32.add_idf_component(name="espressif/wifi_remote_over_eppp", ref="0.3.2") + esp32.add_idf_component(name="espressif/eppp_link", ref="1.1.5") + esp32.add_idf_component(name="espressif/esp_hosted", ref="2.12.6") else: esp32.add_idf_component(name="espressif/esp_wifi_remote", ref="0.13.0") esp32.add_idf_component(name="espressif/eppp_link", ref="0.2.0") diff --git a/esphome/components/host/core.cpp b/esphome/components/host/core.cpp index b067ebbf6e..9123975884 100644 --- a/esphome/components/host/core.cpp +++ b/esphome/components/host/core.cpp @@ -1,74 +1,16 @@ #ifdef USE_HOST #include "esphome/core/application.h" -#include "esphome/core/hal.h" -#include "esphome/core/helpers.h" #include "preferences.h" #include -#include -#include -#include namespace { volatile sig_atomic_t s_signal_received = 0; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) void signal_handler(int signal) { s_signal_received = signal; } } // namespace -namespace esphome { - -void HOT yield() { ::sched_yield(); } -uint32_t IRAM_ATTR HOT millis() { - struct timespec spec; - clock_gettime(CLOCK_MONOTONIC, &spec); - return static_cast(spec.tv_sec * 1000ULL + spec.tv_nsec / 1000000); -} -uint64_t millis_64() { - struct timespec spec; - clock_gettime(CLOCK_MONOTONIC, &spec); - return static_cast(spec.tv_sec) * 1000ULL + static_cast(spec.tv_nsec) / 1000000ULL; -} -void HOT delay(uint32_t ms) { - struct timespec ts; - ts.tv_sec = ms / 1000; - ts.tv_nsec = (ms % 1000) * 1000000; - int res; - do { - res = nanosleep(&ts, &ts); - } while (res != 0 && errno == EINTR); -} -uint32_t IRAM_ATTR HOT micros() { - struct timespec spec; - clock_gettime(CLOCK_MONOTONIC, &spec); - return static_cast(spec.tv_sec * 1000000ULL + spec.tv_nsec / 1000); -} -void IRAM_ATTR HOT delayMicroseconds(uint32_t us) { - struct timespec ts; - ts.tv_sec = us / 1000000U; - ts.tv_nsec = (us % 1000000U) * 1000U; - int res; - do { - res = nanosleep(&ts, &ts); - } while (res != 0 && errno == EINTR); -} -void arch_restart() { exit(0); } -void arch_init() { - // pass -} -void HOT arch_feed_wdt() { - // pass -} - -uint32_t arch_get_cpu_cycle_count() { - struct timespec spec; - clock_gettime(CLOCK_MONOTONIC, &spec); - time_t seconds = spec.tv_sec; - uint32_t us = spec.tv_nsec; - return ((uint32_t) seconds) * 1000000000U + us; -} -uint32_t arch_get_cpu_freq_hz() { return 1000000000U; } - -} // namespace esphome +// HAL functions live in hal.cpp. void setup(); void loop(); diff --git a/esphome/components/host/hal.cpp b/esphome/components/host/hal.cpp new file mode 100644 index 0000000000..256a12ac62 --- /dev/null +++ b/esphome/components/host/hal.cpp @@ -0,0 +1,65 @@ +#ifdef USE_HOST + +#include "esphome/core/hal.h" +#include "esphome/core/helpers.h" + +#include +#include +#include + +// Empty host namespace block to satisfy ci-custom's lint_namespace check. +// HAL functions live in namespace esphome (root) — they are not part of the +// host component's API. +namespace esphome::host {} // namespace esphome::host + +namespace esphome { + +// yield(), arch_init(), arch_feed_wdt(), arch_get_cpu_freq_hz() inlined in +// core/hal/hal_host.h. + +uint32_t IRAM_ATTR HOT millis() { + struct timespec spec; + clock_gettime(CLOCK_MONOTONIC, &spec); + return static_cast(spec.tv_sec * 1000ULL + spec.tv_nsec / 1000000); +} +uint64_t millis_64() { + struct timespec spec; + clock_gettime(CLOCK_MONOTONIC, &spec); + return static_cast(spec.tv_sec) * 1000ULL + static_cast(spec.tv_nsec) / 1000000ULL; +} +void HOT delay(uint32_t ms) { + struct timespec ts; + ts.tv_sec = ms / 1000; + ts.tv_nsec = (ms % 1000) * 1000000; + int res; + do { + res = nanosleep(&ts, &ts); + } while (res != 0 && errno == EINTR); +} +uint32_t IRAM_ATTR HOT micros() { + struct timespec spec; + clock_gettime(CLOCK_MONOTONIC, &spec); + return static_cast(spec.tv_sec * 1000000ULL + spec.tv_nsec / 1000); +} +void IRAM_ATTR HOT delayMicroseconds(uint32_t us) { + struct timespec ts; + ts.tv_sec = us / 1000000U; + ts.tv_nsec = (us % 1000000U) * 1000U; + int res; + do { + res = nanosleep(&ts, &ts); + } while (res != 0 && errno == EINTR); +} +void arch_restart() { exit(0); } + +uint32_t arch_get_cpu_cycle_count() { + struct timespec spec; + clock_gettime(CLOCK_MONOTONIC, &spec); + time_t seconds = spec.tv_sec; + uint32_t ns = static_cast(spec.tv_nsec); + return static_cast(seconds) * 1000000000U + ns; +} + +} // namespace esphome + +#endif // USE_HOST diff --git a/esphome/components/libretiny/__init__.py b/esphome/components/libretiny/__init__.py index 40b8c8dc6c..74ac51d200 100644 --- a/esphome/components/libretiny/__init__.py +++ b/esphome/components/libretiny/__init__.py @@ -37,6 +37,7 @@ from .const import ( CONF_UART_PORT, FAMILIES, FAMILY_BK7231N, + FAMILY_BK7238, FAMILY_COMPONENT, FAMILY_FRIENDLY, FAMILY_RTL8710B, @@ -56,19 +57,22 @@ CODEOWNERS = ["@kuba2k2"] AUTO_LOAD = ["preferences"] IS_TARGET_PLATFORM = True -# BK7231N SDK options to disable unused features. +# BLE 5.x BK SDK options to disable unused features. # Disabling BLE saves ~21KB RAM and ~200KB Flash because BLE init code is # called unconditionally by the SDK. ESPHome doesn't use BLE on LibreTiny. # -# This only works on BK7231N (BLE 5.x). Other BK72XX chips using BLE 4.2 -# (BK7231T, BK7231Q, BK7251; BK7252 boards use the BK7251 family) have a bug -# where the BLE library still links and references undefined symbols when -# CFG_SUPPORT_BLE=0. +# This only works on BLE 5.x BK chips (BK7231N, BK7238). Other BK72XX chips +# using BLE 4.2 (BK7231T, BK7231Q, BK7251; BK7252 boards use the BK7251 family) +# have a bug where the BLE library still links and references undefined symbols +# when CFG_SUPPORT_BLE=0. +# +# On BK7238 the SDK also hangs at WiFi STA enable when BLE init runs, so +# disabling it is required for reliable boot, not just an optimization. # # Other options like CFG_TX_EVM_TEST, CFG_RX_SENSITIVITY_TEST, CFG_SUPPORT_BKREG, # CFG_SUPPORT_OTA_HTTP, and CFG_USE_SPI_SLAVE were evaluated but provide no # NOLINT # measurable benefit - the linker already strips unreferenced code via -gc-sections. -_BK7231N_SYS_CONFIG_OPTIONS = [ +_BLE5_BK_SYS_CONFIG_OPTIONS = [ "CFG_SUPPORT_BLE=0", ] @@ -549,9 +553,9 @@ async def component_to_code(config): cg.add_platformio_option("custom_fw_version", __version__) # Apply chip-specific SDK options to save RAM/Flash - if config[CONF_FAMILY] == FAMILY_BK7231N: + if config[CONF_FAMILY] in (FAMILY_BK7231N, FAMILY_BK7238): cg.add_platformio_option( - "custom_options.sys_config#h", _BK7231N_SYS_CONFIG_OPTIONS + "custom_options.sys_config#h", _BLE5_BK_SYS_CONFIG_OPTIONS ) # Tune lwIP for ESPHome's actual needs. diff --git a/esphome/components/rp2040/core.cpp b/esphome/components/rp2040/core.cpp index d3dc1cf2bb..11f23ccfef 100644 --- a/esphome/components/rp2040/core.cpp +++ b/esphome/components/rp2040/core.cpp @@ -1,41 +1,6 @@ #ifdef USE_RP2040 -#include "core.h" -#include "esphome/core/defines.h" -#ifdef USE_RP2040_CRASH_HANDLER -#include "crash_handler.h" -#endif -#include "esphome/core/hal.h" -#include "esphome/core/helpers.h" - -#include "hardware/timer.h" -#include "hardware/watchdog.h" - -namespace esphome { - -// yield(), delay(), micros(), millis(), millis_64() inlined in hal.h. -void HOT delayMicroseconds(uint32_t us) { delay_microseconds_safe(us); } -void arch_restart() { - watchdog_reboot(0, 0, 10); - while (1) { - continue; - } -} - -void arch_init() { -#ifdef USE_RP2040_CRASH_HANDLER - rp2040::crash_handler_read_and_clear(); -#endif -#if USE_RP2040_WATCHDOG_TIMEOUT > 0 - watchdog_enable(USE_RP2040_WATCHDOG_TIMEOUT, false); -#endif -} - -void HOT arch_feed_wdt() { watchdog_update(); } - -uint32_t HOT arch_get_cpu_cycle_count() { return ulMainGetRunTimeCounterValue(); } -uint32_t arch_get_cpu_freq_hz() { return RP2040::f_cpu(); } - -} // namespace esphome +// HAL functions live in hal.cpp. core.cpp is intentionally empty for +// rp2040 — there is no extra component bootstrap to keep here. #endif // USE_RP2040 diff --git a/esphome/components/rp2040/hal.cpp b/esphome/components/rp2040/hal.cpp new file mode 100644 index 0000000000..7475205d60 --- /dev/null +++ b/esphome/components/rp2040/hal.cpp @@ -0,0 +1,41 @@ +#ifdef USE_RP2040 + +#include "core.h" +#include "esphome/core/defines.h" +#include "esphome/core/hal.h" +#ifdef USE_RP2040_CRASH_HANDLER +#include "crash_handler.h" +#endif + +#include "hardware/watchdog.h" + +// Empty rp2040 namespace block to satisfy ci-custom's lint_namespace check. +// HAL functions live in namespace esphome (root) — they are not part of the +// rp2040 component's API. +namespace esphome::rp2040 {} // namespace esphome::rp2040 + +namespace esphome { + +// yield(), delay(), micros(), millis(), millis_64(), delayMicroseconds(), +// arch_feed_wdt(), arch_get_cpu_cycle_count() inlined in core/hal/hal_rp2040.h. +void arch_restart() { + watchdog_reboot(0, 0, 10); + while (1) { + continue; + } +} + +void arch_init() { +#ifdef USE_RP2040_CRASH_HANDLER + rp2040::crash_handler_read_and_clear(); +#endif +#if USE_RP2040_WATCHDOG_TIMEOUT > 0 + watchdog_enable(USE_RP2040_WATCHDOG_TIMEOUT, false); +#endif +} + +uint32_t arch_get_cpu_freq_hz() { return RP2040::f_cpu(); } + +} // namespace esphome + +#endif // USE_RP2040 diff --git a/esphome/components/sendspin/__init__.py b/esphome/components/sendspin/__init__.py index 58687ae838..1348bf2bfc 100644 --- a/esphome/components/sendspin/__init__.py +++ b/esphome/components/sendspin/__init__.py @@ -24,6 +24,7 @@ CONF_SENDSPIN_ID = "sendspin_id" CONF_INITIAL_STATIC_DELAY = "initial_static_delay" CONF_FIXED_DELAY = "fixed_delay" +CONF_DECODE_MEMORY = "decode_memory" # sendspin-cpp library lives in the global `sendspin` namespace. sendspin_library_ns = cg.global_ns.namespace("sendspin") @@ -39,6 +40,18 @@ CODEC_FORMAT_UNSUPPORTED = SendspinCodecFormat.enum("UNSUPPORTED") AudioSupportedFormatObject = sendspin_library_ns.struct("AudioSupportedFormatObject") PlayerRoleConfig = sendspin_library_ns.struct("PlayerRoleConfig") +# MemoryLocation enum (from sendspin/types.h) controls SPIRAM-vs-internal-RAM placement +# preference for the player role's transfer buffers. +SendspinMemoryLocation = sendspin_library_ns.enum("MemoryLocation", is_class=True) + +MEMORY_PSRAM = "psram" +MEMORY_INTERNAL = "internal" +MEMORY_LOCATIONS = [MEMORY_PSRAM, MEMORY_INTERNAL] +MEMORY_LOCATION_ENUM = { + MEMORY_PSRAM: SendspinMemoryLocation.PREFER_EXTERNAL, + MEMORY_INTERNAL: SendspinMemoryLocation.PREFER_INTERNAL, +} + # Trailing underscore avoids clashing with sendspin-cpp's global `sendspin` namespace. # Analysis tools strip the trailing underscore (same pattern as `template_`). sendspin_ns = cg.esphome_ns.namespace("sendspin_") @@ -193,7 +206,7 @@ async def to_code(config: ConfigType) -> None: ) # sendspin-cpp library - esp32.add_idf_component(name="sendspin/sendspin-cpp", ref="0.3.1") + esp32.add_idf_component(name="sendspin/sendspin-cpp", ref="0.4.0") cg.add_define("USE_SENDSPIN", True) # for MDNS @@ -249,14 +262,23 @@ async def to_code(config: ConfigType) -> None: "CONFIG_SPIRAM_ALLOW_STACK_EXTERNAL_MEMORY", True ) - player_config_struct = cg.StructInitializer( - PlayerRoleConfig, + # Library defaults: priority 18 (one above httpd_priority 17 so the decoder is not + # starved by the HTTP server during the initial encoded-audio burst at stream start), + # interpolation/decode buffer locations PREFER_EXTERNAL. + player_struct_fields = [ ("audio_formats", audio_format_structs), ("audio_buffer_capacity", player_cfg[CONF_BUFFER_SIZE]), ("fixed_delay_us", player_cfg[CONF_FIXED_DELAY]), ("initial_static_delay_ms", player_cfg[CONF_INITIAL_STATIC_DELAY]), ("psram_stack", psram_stack), - ("priority", 2), + ] + if (decode_memory := player_cfg.get(CONF_DECODE_MEMORY)) is not None: + player_struct_fields.append( + ("decode_buffer_location", MEMORY_LOCATION_ENUM[decode_memory]) + ) + player_config_struct = cg.StructInitializer( + PlayerRoleConfig, + *player_struct_fields, ) cg.add(var.set_player_config(player_config_struct)) else: diff --git a/esphome/components/sendspin/media_source/__init__.py b/esphome/components/sendspin/media_source/__init__.py index 6d61a8a636..f689ab01cb 100644 --- a/esphome/components/sendspin/media_source/__init__.py +++ b/esphome/components/sendspin/media_source/__init__.py @@ -13,9 +13,11 @@ from esphome.cpp_generator import MockObj, TemplateArgsType from esphome.types import ConfigType from .. import ( + CONF_DECODE_MEMORY, CONF_FIXED_DELAY, CONF_INITIAL_STATIC_DELAY, CONF_SENDSPIN_ID, + MEMORY_LOCATIONS, SendspinHub, _validate_task_stack_in_psram, register_player_config, @@ -57,6 +59,7 @@ def _register(config: ConfigType) -> ConfigType: CONF_INITIAL_STATIC_DELAY: config[CONF_INITIAL_STATIC_DELAY], CONF_FIXED_DELAY: config[CONF_FIXED_DELAY], CONF_TASK_STACK_IN_PSRAM: config.get(CONF_TASK_STACK_IN_PSRAM, False), + CONF_DECODE_MEMORY: config.get(CONF_DECODE_MEMORY), } ) return config @@ -82,6 +85,7 @@ CONFIG_SCHEMA = cv.All( cv.Optional(CONF_SAMPLE_RATE, default=48000): cv.int_range( min=16000, max=96000 ), + cv.Optional(CONF_DECODE_MEMORY): cv.one_of(*MEMORY_LOCATIONS, lower=True), } ), cv.only_on_esp32, diff --git a/esphome/components/zephyr/core.cpp b/esphome/components/zephyr/core.cpp index 93a9a1ae8e..d1bdaee02d 100644 --- a/esphome/components/zephyr/core.cpp +++ b/esphome/components/zephyr/core.cpp @@ -1,8 +1,6 @@ #ifdef USE_ZEPHYR #include -#include -#include #include #include "esphome/core/hal.h" #include "esphome/core/helpers.h" @@ -10,55 +8,7 @@ namespace esphome { -#ifdef CONFIG_WATCHDOG -static int wdt_channel_id = -1; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) -static const device *const WDT = DEVICE_DT_GET(DT_ALIAS(watchdog0)); -#endif - -void yield() { ::k_yield(); } -uint32_t millis() { return static_cast(millis_64()); } -uint64_t millis_64() { return static_cast(k_uptime_get()); } -uint32_t micros() { return k_ticks_to_us_floor32(k_uptime_ticks()); } -void delayMicroseconds(uint32_t us) { ::k_usleep(us); } -void delay(uint32_t ms) { ::k_msleep(ms); } - -void arch_init() { -#ifdef CONFIG_WATCHDOG - if (device_is_ready(WDT)) { - static wdt_timeout_cfg wdt_config{}; - wdt_config.flags = WDT_FLAG_RESET_SOC; -#ifdef USE_ZIGBEE - // zboss thread use a lot of cpu cycles during start - wdt_config.window.max = 10000; -#else - wdt_config.window.max = 2000; -#endif - wdt_channel_id = wdt_install_timeout(WDT, &wdt_config); - if (wdt_channel_id >= 0) { - uint8_t options = 0; -#ifdef USE_DEBUG - options |= WDT_OPT_PAUSE_HALTED_BY_DBG; -#endif -#ifdef USE_DEEP_SLEEP - options |= WDT_OPT_PAUSE_IN_SLEEP; -#endif - wdt_setup(WDT, options); - } - } -#endif -} - -void arch_feed_wdt() { -#ifdef CONFIG_WATCHDOG - if (wdt_channel_id >= 0) { - wdt_feed(WDT, wdt_channel_id); - } -#endif -} - -void arch_restart() { sys_reboot(SYS_REBOOT_COLD); } -uint32_t arch_get_cpu_cycle_count() { return k_cycle_get_32(); } -uint32_t arch_get_cpu_freq_hz() { return sys_clock_hw_cycles_per_sec(); } +// HAL functions live in hal.cpp. Mutex::Mutex() { auto *mutex = new k_mutex(); diff --git a/esphome/components/zephyr/hal.cpp b/esphome/components/zephyr/hal.cpp new file mode 100644 index 0000000000..5c08ed2519 --- /dev/null +++ b/esphome/components/zephyr/hal.cpp @@ -0,0 +1,63 @@ +#ifdef USE_ZEPHYR + +#include "esphome/core/defines.h" +#include "esphome/core/hal.h" + +#include +#include + +// Empty zephyr namespace block to satisfy ci-custom's lint_namespace check. +// HAL functions live in namespace esphome (root) — they are not part of the +// zephyr component's API. +namespace esphome::zephyr {} // namespace esphome::zephyr + +namespace esphome { + +#ifdef CONFIG_WATCHDOG +static int wdt_channel_id = -1; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) +static const device *const WDT = DEVICE_DT_GET(DT_ALIAS(watchdog0)); +#endif + +// yield(), delay(), micros(), millis(), millis_64(), delayMicroseconds(), +// arch_get_cpu_cycle_count(), arch_get_cpu_freq_hz() inlined in +// core/hal/hal_zephyr.h. + +void arch_init() { +#ifdef CONFIG_WATCHDOG + if (device_is_ready(WDT)) { + static wdt_timeout_cfg wdt_config{}; + wdt_config.flags = WDT_FLAG_RESET_SOC; +#ifdef USE_ZIGBEE + // zboss thread uses a lot of CPU cycles during startup + wdt_config.window.max = 10000; +#else + wdt_config.window.max = 2000; +#endif + wdt_channel_id = wdt_install_timeout(WDT, &wdt_config); + if (wdt_channel_id >= 0) { + uint8_t options = 0; +#ifdef USE_DEBUG + options |= WDT_OPT_PAUSE_HALTED_BY_DBG; +#endif +#ifdef USE_DEEP_SLEEP + options |= WDT_OPT_PAUSE_IN_SLEEP; +#endif + wdt_setup(WDT, options); + } + } +#endif +} + +void arch_feed_wdt() { +#ifdef CONFIG_WATCHDOG + if (wdt_channel_id >= 0) { + wdt_feed(WDT, wdt_channel_id); + } +#endif +} + +void arch_restart() { sys_reboot(SYS_REBOOT_COLD); } + +} // namespace esphome + +#endif // USE_ZEPHYR diff --git a/esphome/components/zigbee/zigbee_esp32.py b/esphome/components/zigbee/zigbee_esp32.py index 1b98df6c0a..9081582c7b 100644 --- a/esphome/components/zigbee/zigbee_esp32.py +++ b/esphome/components/zigbee/zigbee_esp32.py @@ -9,6 +9,7 @@ from esphome.components.esp32 import ( add_idf_component, add_idf_sdkconfig_option, add_partition, + idf_version, require_vfs_select, ) import esphome.config_validation as cv @@ -186,6 +187,10 @@ async def _zigbee_add_sdkconfigs(config: ConfigType) -> None: # The pre-built Zigbee library uses esp_log_default_level which requires # dynamic log level control to be enabled add_idf_sdkconfig_option("CONFIG_LOG_DYNAMIC_LEVEL_CONTROL", True) + # The pre-built Zigbee library is compiled against newlib which requires newlib + # reentrancy to be enabled with picolibc compatibility. + if idf_version() >= cv.Version(6, 0, 0): + add_idf_sdkconfig_option("CONFIG_LIBC_PICOLIBC_NEWLIB_COMPATIBILITY", True) async def attributes_to_code( diff --git a/esphome/core/hal/hal_host.h b/esphome/core/hal/hal_host.h index a8896fdf63..d7f317176e 100644 --- a/esphome/core/hal/hal_host.h +++ b/esphome/core/hal/hal_host.h @@ -3,6 +3,7 @@ #ifdef USE_HOST #include +#include #define IRAM_ATTR #define PROGMEM @@ -13,17 +14,18 @@ namespace esphome { /// Host has no ISR concept. __attribute__((always_inline)) inline bool in_isr_context() { return false; } -void yield(); +__attribute__((always_inline)) inline void yield() { ::sched_yield(); } + void delay(uint32_t ms); uint32_t micros(); uint32_t millis(); uint64_t millis_64(); - void delayMicroseconds(uint32_t us); // NOLINT(readability-identifier-naming) -void arch_feed_wdt(); uint32_t arch_get_cpu_cycle_count(); -void arch_init(); -uint32_t arch_get_cpu_freq_hz(); + +__attribute__((always_inline)) inline void arch_init() {} +__attribute__((always_inline)) inline void arch_feed_wdt() {} +__attribute__((always_inline)) inline uint32_t arch_get_cpu_freq_hz() { return 1000000000U; } } // namespace esphome diff --git a/esphome/core/hal/hal_rp2040.h b/esphome/core/hal/hal_rp2040.h index 46f6e421cd..27a9b23c0b 100644 --- a/esphome/core/hal/hal_rp2040.h +++ b/esphome/core/hal/hal_rp2040.h @@ -20,8 +20,17 @@ extern "C" unsigned long millis(void); // Forward decl from . extern "C" uint64_t time_us_64(void); +// Forward decls from pico-sdk / FreeRTOS port for the inline arch_* +// wrappers below. +extern "C" void watchdog_update(void); +extern "C" unsigned long ulMainGetRunTimeCounterValue(void); + namespace esphome { +// Forward decl from helpers.h. +// NOLINTNEXTLINE(readability-redundant-declaration) +void delay_microseconds_safe(uint32_t us); + /// Returns true when executing inside an interrupt handler. __attribute__((always_inline)) inline bool in_isr_context() { uint32_t ipsr; @@ -35,9 +44,13 @@ __attribute__((always_inline)) inline uint32_t micros() { return static_cast(::time_us_64()); } -void delayMicroseconds(uint32_t us); // NOLINT(readability-identifier-naming) -void arch_feed_wdt(); -uint32_t arch_get_cpu_cycle_count(); +// NOLINTNEXTLINE(readability-identifier-naming) +__attribute__((always_inline)) inline void delayMicroseconds(uint32_t us) { delay_microseconds_safe(us); } +__attribute__((always_inline)) inline void arch_feed_wdt() { watchdog_update(); } +__attribute__((always_inline)) inline uint32_t arch_get_cpu_cycle_count() { + return static_cast(ulMainGetRunTimeCounterValue()); +} + void arch_init(); uint32_t arch_get_cpu_freq_hz(); diff --git a/esphome/core/hal/hal_zephyr.h b/esphome/core/hal/hal_zephyr.h index d4b37b5eb6..613b3911c1 100644 --- a/esphome/core/hal/hal_zephyr.h +++ b/esphome/core/hal/hal_zephyr.h @@ -4,6 +4,8 @@ #include +#include + #define IRAM_ATTR #define PROGMEM @@ -13,17 +15,19 @@ namespace esphome { /// Zephyr/nRF52: not currently consulted — wake path is platform-specific. __attribute__((always_inline)) inline bool in_isr_context() { return false; } -void yield(); -void delay(uint32_t ms); -uint32_t micros(); -uint32_t millis(); -uint64_t millis_64(); +__attribute__((always_inline)) inline void yield() { ::k_yield(); } +__attribute__((always_inline)) inline void delay(uint32_t ms) { ::k_msleep(ms); } +__attribute__((always_inline)) inline uint32_t micros() { return k_ticks_to_us_floor32(k_uptime_ticks()); } +__attribute__((always_inline)) inline uint64_t millis_64() { return static_cast(k_uptime_get()); } +__attribute__((always_inline)) inline uint32_t millis() { return static_cast(millis_64()); } + +// NOLINTNEXTLINE(readability-identifier-naming) +__attribute__((always_inline)) inline void delayMicroseconds(uint32_t us) { ::k_usleep(us); } +__attribute__((always_inline)) inline uint32_t arch_get_cpu_cycle_count() { return k_cycle_get_32(); } +__attribute__((always_inline)) inline uint32_t arch_get_cpu_freq_hz() { return sys_clock_hw_cycles_per_sec(); } -void delayMicroseconds(uint32_t us); // NOLINT(readability-identifier-naming) void arch_feed_wdt(); -uint32_t arch_get_cpu_cycle_count(); void arch_init(); -uint32_t arch_get_cpu_freq_hz(); } // namespace esphome diff --git a/esphome/core/helpers.h b/esphome/core/helpers.h index 355db6c7f4..07bcb7a74f 100644 --- a/esphome/core/helpers.h +++ b/esphome/core/helpers.h @@ -2045,7 +2045,8 @@ void delay_microseconds_safe(uint32_t us); * Returns `nullptr` in case no memory is available. * * By setting flags, it can be configured to: - * - perform external allocation falling back to main memory if SPI RAM is full or unavailable + * - perform external allocation falling back to internal memory if SPI RAM is full or unavailable (default) + * - perform internal allocation falling back to external memory (with PREFER_INTERNAL) * - perform external allocation only * - perform internal allocation only */ @@ -2054,16 +2055,26 @@ template class RAMAllocator { using value_type = T; enum Flags { - NONE = 0, // Perform external allocation and fall back to internal memory - ALLOC_EXTERNAL = 1 << 0, // Perform external allocation only. - ALLOC_INTERNAL = 1 << 1, // Perform internal allocation only. - ALLOW_FAILURE = 1 << 2, // Does nothing. Kept for compatibility. + NONE = 0, // Perform external allocation and fall back to internal memory + ALLOC_EXTERNAL = 1 << 0, // Perform external allocation only. + ALLOC_INTERNAL = 1 << 1, // Perform internal allocation only. + ALLOW_FAILURE = 1 << 2, // Does nothing. Kept for compatibility. + PREFER_INTERNAL = 1 << 3, // Perform internal allocation and fall back to external memory }; constexpr RAMAllocator() = default; - constexpr RAMAllocator(uint8_t flags) - : flags_((flags & (ALLOC_INTERNAL | ALLOC_EXTERNAL)) != 0 ? (flags & (ALLOC_INTERNAL | ALLOC_EXTERNAL)) - : (ALLOC_INTERNAL | ALLOC_EXTERNAL)) {} + constexpr RAMAllocator(uint8_t flags) { + if (flags & PREFER_INTERNAL) { + this->flags_ = ALLOC_INTERNAL | ALLOC_EXTERNAL | PREFER_INTERNAL; + return; + } + const uint8_t alloc_bits = flags & (ALLOC_INTERNAL | ALLOC_EXTERNAL); + if (alloc_bits != 0) { + this->flags_ = alloc_bits; + return; + } + this->flags_ = ALLOC_INTERNAL | ALLOC_EXTERNAL; + } template constexpr RAMAllocator(const RAMAllocator &other) : flags_{other.flags_} {} T *allocate(size_t n) { return this->allocate(n, sizeof(T)); } @@ -2072,12 +2083,8 @@ template class RAMAllocator { size_t size = n * manual_size; T *ptr = nullptr; #ifdef USE_ESP32 - if (this->flags_ & Flags::ALLOC_EXTERNAL) { - ptr = static_cast(heap_caps_malloc(size, MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT)); - } - if (ptr == nullptr && this->flags_ & Flags::ALLOC_INTERNAL) { - ptr = static_cast(heap_caps_malloc(size, MALLOC_CAP_INTERNAL | MALLOC_CAP_8BIT)); - } + const auto caps = this->get_caps_(); + ptr = static_cast(heap_caps_malloc_prefer(size, 2, caps[0], caps[1])); #else // Ignore ALLOC_EXTERNAL/ALLOC_INTERNAL flags if external allocation is not supported ptr = static_cast(malloc(size)); // NOLINT(cppcoreguidelines-owning-memory,cppcoreguidelines-no-malloc) @@ -2091,12 +2098,8 @@ template class RAMAllocator { size_t size = n * manual_size; T *ptr = nullptr; #ifdef USE_ESP32 - if (this->flags_ & Flags::ALLOC_EXTERNAL) { - ptr = static_cast(heap_caps_realloc(p, size, MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT)); - } - if (ptr == nullptr && this->flags_ & Flags::ALLOC_INTERNAL) { - ptr = static_cast(heap_caps_realloc(p, size, MALLOC_CAP_INTERNAL | MALLOC_CAP_8BIT)); - } + const auto caps = this->get_caps_(); + ptr = static_cast(heap_caps_realloc_prefer(p, size, 2, caps[0], caps[1])); #else // Ignore ALLOC_EXTERNAL/ALLOC_INTERNAL flags if external allocation is not supported ptr = static_cast(realloc(p, size)); // NOLINT(cppcoreguidelines-owning-memory,cppcoreguidelines-no-malloc) @@ -2147,6 +2150,24 @@ template class RAMAllocator { } private: +#ifdef USE_ESP32 + /// Returns {primary_caps, fallback_caps} for heap_caps_*_prefer based on the configured flags. + /// PREFER_INTERNAL implies both regions are enabled (enforced by the constructor), so when it is set + /// the primary is internal and the fallback is external. Otherwise the primary is whichever region + /// is enabled (external preferred when both are enabled), and the fallback is the other region (or + /// the same region when only one is enabled, making the second attempt a no-op). + std::array get_caps_() const { + constexpr uint32_t external_caps = MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT; + constexpr uint32_t internal_caps = MALLOC_CAP_INTERNAL | MALLOC_CAP_8BIT; + if (this->flags_ & PREFER_INTERNAL) { + return {internal_caps, external_caps}; + } + const uint32_t primary = (this->flags_ & ALLOC_EXTERNAL) ? external_caps : internal_caps; + const uint32_t fallback = (this->flags_ & ALLOC_INTERNAL) ? internal_caps : external_caps; + return {primary, fallback}; + } +#endif + uint8_t flags_{ALLOC_INTERNAL | ALLOC_EXTERNAL}; }; diff --git a/esphome/idf_component.yml b/esphome/idf_component.yml index 757fcf9dd7..f5a8dd8c60 100644 --- a/esphome/idf_component.yml +++ b/esphome/idf_component.yml @@ -20,15 +20,19 @@ dependencies: espressif/mdns: version: 1.11.0 espressif/esp_wifi_remote: - version: 1.4.0 + version: 1.5.1 + rules: + - if: "target in [esp32h2, esp32p4]" + espressif/wifi_remote_over_eppp: + version: 0.3.2 rules: - if: "target in [esp32h2, esp32p4]" espressif/eppp_link: - version: 1.1.4 + version: 1.1.5 rules: - if: "target in [esp32h2, esp32p4]" espressif/esp_hosted: - version: 2.12.1 + version: 2.12.6 rules: - if: "target in [esp32h2, esp32p4]" zorxx/multipart-parser: @@ -92,6 +96,6 @@ dependencies: esp32async/asynctcp: version: 3.4.91 sendspin/sendspin-cpp: - version: 0.3.1 + version: 0.4.0 lvgl/lvgl: version: 9.5.0 diff --git a/script/determine-jobs.py b/script/determine-jobs.py index 6fd7ab297c..c0cf8ecbdc 100755 --- a/script/determine-jobs.py +++ b/script/determine-jobs.py @@ -6,8 +6,7 @@ what files have changed. It outputs JSON with the following structure: { "integration_tests": true/false, - "integration_tests_run_all": true/false, - "integration_test_files": ["tests/integration/test_foo.py", ...], + "integration_test_buckets": [{"name": "1/3", "tests": ["tests/integration/test_foo.py", ...]}, ...], "clang_tidy": true/false, "clang_format": true/false, "python_linters": true/false, @@ -81,6 +80,62 @@ CLANG_TIDY_SPLIT_THRESHOLD = 65 # Isolated components count as 10x, groupable components count as 1x COMPONENT_TEST_BATCH_SIZE = 40 +# Integration test bucketing: when more than the threshold tests are scheduled, +# fan out across this many parallel jobs. Below the threshold, a single job runs. +INTEGRATION_TESTS_SPLIT_THRESHOLD = 10 +INTEGRATION_TESTS_SPLIT_BUCKETS = 3 + + +def _split_list(items: list[str], n: int) -> list[list[str]]: + """Split a list into n roughly-equal contiguous parts (matches script/clang-tidy).""" + k, m = divmod(len(items), n) + return [items[i * k + min(i, m) : (i + 1) * k + min(i + 1, m)] for i in range(n)] + + +def _all_integration_test_files() -> list[str]: + """Return all integration test file paths, sorted, relative to repo root.""" + return sorted( + str(p.relative_to(root_path)) + for p in (Path(root_path) / "tests" / "integration").glob("test_*.py") + ) + + +def _compute_integration_test_buckets( + integration_run_all: bool, + integration_test_files: list[str], +) -> tuple[bool, list[dict[str, Any]]]: + """Compute (run_integration, buckets) from the determine_integration_tests result. + + Pure function for unit testing — no I/O beyond `_all_integration_test_files` + when `integration_run_all` is set. + + `buckets` is a list of `{name, tests}` dicts where `tests` is a JSON-friendly + list of file paths so the workflow can build a bash array via jq, avoiding + shell word-splitting / glob hazards. + """ + if integration_run_all: + files = _all_integration_test_files() + else: + files = sorted(integration_test_files) + + # Empty list (e.g. run_all expansion with no files on disk) would otherwise + # cause the workflow to invoke pytest with no path argument and collect + # tests outside tests/integration/. Suppress the run instead. + if not files: + return False, [] + + if len(files) > INTEGRATION_TESTS_SPLIT_THRESHOLD: + parts = [ + part for part in _split_list(files, INTEGRATION_TESTS_SPLIT_BUCKETS) if part + ] + buckets = [ + {"name": f"{i + 1}/{len(parts)}", "tests": part} + for i, part in enumerate(parts) + ] + else: + buckets = [{"name": "1/1", "tests": files}] + return True, buckets + class Platform(StrEnum): """Platform identifiers for memory impact analysis.""" @@ -812,7 +867,9 @@ def main() -> None: integration_run_all, integration_test_files = determine_integration_tests( args.branch ) - run_integration = integration_run_all or bool(integration_test_files) + run_integration, integration_test_buckets = _compute_integration_test_buckets( + integration_run_all, integration_test_files + ) run_clang_tidy = should_run_clang_tidy(args.branch) run_clang_format = should_run_clang_format(args.branch) run_python_linters = should_run_python_linters(args.branch) @@ -944,8 +1001,7 @@ def main() -> None: output: dict[str, Any] = { "integration_tests": run_integration, - "integration_tests_run_all": integration_run_all, - "integration_test_files": integration_test_files, + "integration_test_buckets": integration_test_buckets, "clang_tidy": run_clang_tidy, "clang_tidy_mode": clang_tidy_mode, "clang_format": run_clang_format, diff --git a/tests/components/sendspin/common-media_source.yaml b/tests/components/sendspin/common-media_source.yaml index 4a7cd79c67..5b33a54647 100644 --- a/tests/components/sendspin/common-media_source.yaml +++ b/tests/components/sendspin/common-media_source.yaml @@ -7,3 +7,4 @@ media_source: initial_static_delay: 5ms static_delay_adjustable: true fixed_delay: 480us + decode_memory: internal diff --git a/tests/script/test_determine_jobs.py b/tests/script/test_determine_jobs.py index 44c110b689..e85f1757b0 100644 --- a/tests/script/test_determine_jobs.py +++ b/tests/script/test_determine_jobs.py @@ -122,10 +122,19 @@ def test_main_all_tests_should_run( "esphome/helpers.py", ] + # Stable, deterministic stand-in for the tests/integration/ glob so the + # bucket assertions don't drift with the real test count. + fake_test_files = [f"tests/integration/test_{i:03d}.py" for i in range(15)] + # Run main function with mocked argv with ( patch("sys.argv", ["determine-jobs.py"]), patch.object(determine_jobs, "_is_clang_tidy_full_scan", return_value=False), + patch.object( + determine_jobs, + "_all_integration_test_files", + return_value=fake_test_files, + ), patch.object( determine_jobs, "get_changed_components", @@ -161,8 +170,24 @@ def test_main_all_tests_should_run( output = json.loads(captured.out) assert output["integration_tests"] is True - assert output["integration_tests_run_all"] is True - assert output["integration_test_files"] == [] + # run_all=True expands to the full glob and pre-buckets into 3 parts. + # Each bucket's `tests` is a JSON list of file paths. + assert isinstance(output["integration_test_buckets"], list) + assert len(output["integration_test_buckets"]) == 3 + assert [b["name"] for b in output["integration_test_buckets"]] == [ + "1/3", + "2/3", + "3/3", + ] + for bucket in output["integration_test_buckets"]: + assert isinstance(bucket["tests"], list) + for path in bucket["tests"]: + assert isinstance(path, str) + bucket_files = [f for b in output["integration_test_buckets"] for f in b["tests"]] + assert bucket_files == fake_test_files + # Bucket sizes are balanced (max-min difference at most 1). + sizes = [len(b["tests"]) for b in output["integration_test_buckets"]] + assert max(sizes) - min(sizes) <= 1 assert output["clang_tidy"] is True assert output["clang_tidy_mode"] in ["nosplit", "split"] assert output["clang_format"] is True @@ -247,8 +272,7 @@ def test_main_no_tests_should_run( output = json.loads(captured.out) assert output["integration_tests"] is False - assert output["integration_tests_run_all"] is False - assert output["integration_test_files"] == [] + assert output["integration_test_buckets"] == [] assert output["clang_tidy"] is False assert output["clang_tidy_mode"] == "disabled" assert output["clang_format"] is False @@ -332,8 +356,7 @@ def test_main_with_branch_argument( output = json.loads(captured.out) assert output["integration_tests"] is False - assert output["integration_tests_run_all"] is False - assert output["integration_test_files"] == [] + assert output["integration_test_buckets"] == [] assert output["clang_tidy"] is True assert output["clang_tidy_mode"] in ["nosplit", "split"] assert output["clang_format"] is False @@ -357,6 +380,59 @@ def test_main_with_branch_argument( assert output["cpp_unit_tests_components"] == ["mqtt"] +def test_compute_integration_test_buckets_empty() -> None: + """No integration tests scheduled => (False, []).""" + run, buckets = determine_jobs._compute_integration_test_buckets(False, []) + assert run is False + assert buckets == [] + + +def test_compute_integration_test_buckets_below_threshold() -> None: + """A small explicit list (<= threshold) => single 1/1 bucket with that list.""" + files = [f"tests/integration/test_{name}.py" for name in ("c", "a", "b")] + run, buckets = determine_jobs._compute_integration_test_buckets(False, files) + assert run is True + assert buckets == [{"name": "1/1", "tests": sorted(files)}] + + +def test_compute_integration_test_buckets_at_threshold_stays_single() -> None: + """Exactly INTEGRATION_TESTS_SPLIT_THRESHOLD files => still one bucket + (the split kicks in only when count is strictly greater than threshold).""" + files = [ + f"tests/integration/test_{i:02d}.py" + for i in range(determine_jobs.INTEGRATION_TESTS_SPLIT_THRESHOLD) + ] + run, buckets = determine_jobs._compute_integration_test_buckets(False, files) + assert run is True + assert len(buckets) == 1 + assert buckets[0]["name"] == "1/1" + assert buckets[0]["tests"] == sorted(files) + + +def test_compute_integration_test_buckets_just_over_threshold_splits() -> None: + """One file over the threshold triggers the 3-bucket fan-out, balanced.""" + n = determine_jobs.INTEGRATION_TESTS_SPLIT_THRESHOLD + 1 + files = [f"tests/integration/test_{i:02d}.py" for i in range(n)] + run, buckets = determine_jobs._compute_integration_test_buckets(False, files) + assert run is True + assert [b["name"] for b in buckets] == ["1/3", "2/3", "3/3"] + union = [path for b in buckets for path in b["tests"]] + assert union == sorted(files) + sizes = [len(b["tests"]) for b in buckets] + assert max(sizes) - min(sizes) <= 1 + + +def test_compute_integration_test_buckets_run_all_with_empty_glob_disables_run() -> ( + None +): + """run_all=True but glob returns no files => run suppressed (otherwise + pytest would collect tests outside tests/integration/).""" + with patch.object(determine_jobs, "_all_integration_test_files", return_value=[]): + run, buckets = determine_jobs._compute_integration_test_buckets(True, []) + assert run is False + assert buckets == [] + + def test_determine_integration_tests( monkeypatch: pytest.MonkeyPatch, ) -> None: