diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 461e676c4e..f7710589c5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -106,6 +106,7 @@ jobs: script/build_codeowners.py --check script/build_language_schema.py --check script/generate-esp32-boards.py --check + script/generate-rp2040-boards.py --check pytest: name: Run pytest @@ -170,6 +171,8 @@ jobs: - common outputs: integration-tests: ${{ steps.determine.outputs.integration-tests }} + integration-tests-run-all: ${{ steps.determine.outputs.integration-tests-run-all }} + integration-test-files: ${{ steps.determine.outputs.integration-test-files }} clang-tidy: ${{ steps.determine.outputs.clang-tidy }} clang-tidy-mode: ${{ steps.determine.outputs.clang-tidy-mode }} python-linters: ${{ steps.determine.outputs.python-linters }} @@ -210,6 +213,8 @@ jobs: # Extract individual fields echo "integration-tests=$(echo "$output" | jq -r '.integration_tests')" >> $GITHUB_OUTPUT + echo "integration-tests-run-all=$(echo "$output" | jq -r '.integration_tests_run_all')" >> $GITHUB_OUTPUT + echo "integration-test-files=$(echo "$output" | jq -c '.integration_test_files')" >> $GITHUB_OUTPUT echo "clang-tidy=$(echo "$output" | jq -r '.clang_tidy')" >> $GITHUB_OUTPUT echo "clang-tidy-mode=$(echo "$output" | jq -r '.clang_tidy_mode')" >> $GITHUB_OUTPUT echo "python-linters=$(echo "$output" | jq -r '.python_linters')" >> $GITHUB_OUTPUT @@ -261,9 +266,20 @@ jobs: - name: Register matcher run: echo "::add-matcher::.github/workflows/matchers/pytest.json" - name: Run integration tests + env: + INTEGRATION_TEST_FILES: ${{ needs.determine-jobs.outputs.integration-test-files }} + INTEGRATION_TESTS_RUN_ALL: ${{ needs.determine-jobs.outputs.integration-tests-run-all }} run: | . venv/bin/activate - pytest -vv --no-cov --tb=native -n auto tests/integration/ + if [[ "$INTEGRATION_TESTS_RUN_ALL" == "true" ]]; then + echo "Running all integration tests" + pytest -vv --no-cov --tb=native -n auto tests/integration/ + else + # Parse JSON array into bash array to avoid shell expansion issues + mapfile -t test_files < <(echo "$INTEGRATION_TEST_FILES" | jq -r '.[]') + echo "Running ${#test_files[@]} specific integration tests" + pytest -vv --no-cov --tb=native -n auto "${test_files[@]}" + fi cpp-unit-tests: name: Run C++ unit tests diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 2d8f698395..5e2bfe09ce 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -11,7 +11,7 @@ ci: repos: - repo: https://github.com/astral-sh/ruff-pre-commit # Ruff version. - rev: v0.15.5 + rev: v0.15.6 hooks: # Run the linter. - id: ruff diff --git a/esphome/analyze_memory/__init__.py b/esphome/analyze_memory/__init__.py index bf1bcbfa05..7954c22822 100644 --- a/esphome/analyze_memory/__init__.py +++ b/esphome/analyze_memory/__init__.py @@ -1,6 +1,6 @@ """Memory usage analyzer for ESPHome compiled binaries.""" -from collections import defaultdict +from collections import Counter, defaultdict from dataclasses import dataclass, field import logging from pathlib import Path @@ -40,6 +40,15 @@ _READELF_SECTION_PATTERN = re.compile( r"\s*\[\s*\d+\]\s+([\.\w]+)\s+\w+\s+[\da-fA-F]+\s+[\da-fA-F]+\s+([\da-fA-F]+)" ) +# Regex for extracting call targets from objdump disassembly +# Matches direct call instructions across architectures: +# Xtensa: call0/call4/call8/call12/callx0/callx4/callx8/callx12 +# ARM: bl/blx +# Captures the mangled symbol name inside angle brackets. +_CALL_TARGET_PATTERN = re.compile( + r"\t(?:call(?:0|4|8|12)|callx(?:0|4|8|12)|blx?)\s+[\da-fA-F]+ <([^>]+)>" +) + # Component category prefixes _COMPONENT_PREFIX_ESPHOME = "[esphome]" _COMPONENT_PREFIX_EXTERNAL = "[external]" @@ -197,6 +206,8 @@ class MemoryAnalyzer: self._lib_hash_to_name: dict[str, str] = {} # Heuristic category to library redirect: "mdns_lib" -> "[lib]mdns" self._heuristic_to_lib: dict[str, str] = {} + # Function call counts: mangled_name -> call_count + self._function_call_counts: Counter[str] = Counter() def analyze(self) -> dict[str, ComponentMemory]: """Analyze the ELF file and return component memory usage.""" @@ -206,6 +217,7 @@ class MemoryAnalyzer: self._categorize_symbols() self._analyze_cswtch_symbols() self._analyze_sdk_libraries() + self._analyze_function_calls() return dict(self.components) def _parse_sections(self) -> None: @@ -384,8 +396,9 @@ class MemoryAnalyzer: return _LOGGER.info("Demangling %d symbols", len(symbols)) - self._demangle_cache = batch_demangle(symbols, objdump_path=self.objdump_path) - _LOGGER.info("Successfully demangled %d symbols", len(self._demangle_cache)) + demangled = batch_demangle(symbols, objdump_path=self.objdump_path) + self._demangle_cache.update(demangled) + _LOGGER.info("Successfully demangled %d symbols", len(demangled)) def _demangle_symbol(self, symbol: str) -> str: """Get demangled C++ symbol name from cache.""" @@ -1011,6 +1024,43 @@ class MemoryAnalyzer: total_size, ) + def _analyze_function_calls(self) -> None: + """Count function call sites by parsing disassembly output. + + Parses direct call instructions (call0/call8/bl/blx) from objdump -d + to count how many times each function is called. This helps identify + inlining candidates — frequently called small functions benefit most + from inlining. + """ + result = run_tool( + [self.objdump_path, "-d", str(self.elf_path)], + timeout=60, + ) + if result is None or result.returncode != 0: + _LOGGER.debug("Failed to disassemble ELF for function call analysis") + return + + self._function_call_counts = Counter( + match.group(1) + for line in result.stdout.splitlines() + if (match := _CALL_TARGET_PATTERN.search(line)) + ) + + # Demangle any call targets not already in the cache + missing = [ + name + for name in self._function_call_counts + if name not in self._demangle_cache + ] + if missing: + self._batch_demangle_symbols(missing) + + _LOGGER.debug( + "Function call analysis: %d unique targets, %d total calls", + len(self._function_call_counts), + sum(self._function_call_counts.values()), + ) + def get_unattributed_ram(self) -> tuple[int, int, int]: """Get unattributed RAM sizes (SDK/framework overhead). diff --git a/esphome/analyze_memory/cli.py b/esphome/analyze_memory/cli.py index dbc19c6b89..acaf5f4562 100644 --- a/esphome/analyze_memory/cli.py +++ b/esphome/analyze_memory/cli.py @@ -231,6 +231,110 @@ class MemoryAnalyzerCLI(MemoryAnalyzer): lines.append(f" {size:>6,} B {sym_name}") lines.append("") + # Number of top called functions to show + TOP_CALLS_LIMIT: int = 50 + # Number of inlining candidates to show + INLINE_CANDIDATES_LIMIT: int = 25 + # Maximum function size in bytes to consider for inlining + INLINE_SIZE_THRESHOLD: int = 16 + + def _build_symbol_sizes(self) -> dict[str, int]: + """Build a size lookup from all component symbols: mangled_name -> size.""" + return { + symbol: size + for symbols in self._component_symbols.values() + for symbol, _, size, _ in symbols + } + + def _format_call_row( + self, index: int, mangled: str, count: int, symbol_sizes: dict[str, int] + ) -> str: + """Format a single row for call frequency tables.""" + demangled = self._demangle_cache.get(mangled, mangled) + if len(demangled) > 80: + demangled = f"{demangled[:77]}..." + size = symbol_sizes.get(mangled) + size_str = f"{size:>5,} B" if size is not None else " ?" + return f"{index:>3} {count:>5} {size_str} {demangled}" + + def _add_call_table_header(self, lines: list[str]) -> None: + """Add the header row for call frequency tables.""" + lines.append(f"{'#':>3} {'Calls':>5} {'Size':>7} Function") + lines.append(f"{'---':>3} {'-----':>5} {'-------':>7} {'-' * 60}") + + def _add_function_call_analysis(self, lines: list[str]) -> None: + """Add function call frequency analysis section. + + Shows the most frequently called functions by call site count. + """ + self._add_section_header(lines, "Top Called Functions") + + symbol_sizes = self._build_symbol_sizes() + + # Sort by call count descending + sorted_calls = sorted( + self._function_call_counts.items(), key=lambda x: x[1], reverse=True + ) + + self._add_call_table_header(lines) + + for i, (mangled, count) in enumerate(sorted_calls[: self.TOP_CALLS_LIMIT]): + lines.append(self._format_call_row(i + 1, mangled, count, symbol_sizes)) + + total_calls = sum(self._function_call_counts.values()) + lines.append("") + lines.append( + f"Total: {len(self._function_call_counts)} unique targets, " + f"{total_calls:,} call sites" + ) + lines.append("") + + def _add_inline_candidates(self, lines: list[str]) -> None: + """Add inlining candidates section. + + Shows frequently called functions that are small enough to benefit + from inlining (< 16 bytes). These are the best candidates for + reducing call overhead. + """ + self._add_section_header( + lines, + f"Inlining Candidates (<{self.INLINE_SIZE_THRESHOLD} B, by call count)", + ) + + symbol_sizes = self._build_symbol_sizes() + + # Filter to small functions with known size, sort by call count + candidates = sorted( + ( + (mangled, count) + for mangled, count in self._function_call_counts.items() + if mangled in symbol_sizes + and symbol_sizes[mangled] < self.INLINE_SIZE_THRESHOLD + ), + key=lambda x: x[1], + reverse=True, + ) + + if not candidates: + lines.append("No candidates found.") + lines.append("") + return + + self._add_call_table_header(lines) + + for i, (mangled, count) in enumerate( + candidates[: self.INLINE_CANDIDATES_LIMIT] + ): + lines.append(self._format_call_row(i + 1, mangled, count, symbol_sizes)) + + lines.append("") + lines.append( + f"Showing top {min(len(candidates), self.INLINE_CANDIDATES_LIMIT)} " + f"of {len(candidates)} functions under " + f"{self.INLINE_SIZE_THRESHOLD} B" + ) + lines.append("") + def generate_report(self, detailed: bool = False) -> str: """Generate a formatted memory report.""" components = sorted( @@ -533,6 +637,11 @@ class MemoryAnalyzerCLI(MemoryAnalyzer): if self._cswtch_symbols: self._add_cswtch_analysis(lines) + # Function call frequency analysis + if self._function_call_counts: + self._add_function_call_analysis(lines) + self._add_inline_candidates(lines) + lines.append( "Note: This analysis covers symbols in the ELF file. Some runtime allocations may not be included." ) diff --git a/esphome/components/adc/adc_sensor.h b/esphome/components/adc/adc_sensor.h index 91cf4eaafc..cf48ccd9c3 100644 --- a/esphome/components/adc/adc_sensor.h +++ b/esphome/components/adc/adc_sensor.h @@ -22,7 +22,8 @@ namespace adc { #ifdef USE_ESP32 // clang-format off -#if (ESP_IDF_VERSION_MAJOR == 5 && \ +#if ESP_IDF_VERSION_MAJOR >= 6 || \ + (ESP_IDF_VERSION_MAJOR == 5 && \ ((ESP_IDF_VERSION_MINOR == 0 && ESP_IDF_VERSION_PATCH >= 5) || \ (ESP_IDF_VERSION_MINOR == 1 && ESP_IDF_VERSION_PATCH >= 3) || \ (ESP_IDF_VERSION_MINOR >= 2)) \ diff --git a/esphome/components/api/api_frame_helper.h b/esphome/components/api/api_frame_helper.h index 98de24501e..5e07ad43a9 100644 --- a/esphome/components/api/api_frame_helper.h +++ b/esphome/components/api/api_frame_helper.h @@ -134,12 +134,16 @@ class APIFrameHelper { // // For log messages: Use Nagle to coalesce multiple small log packets into // fewer larger packets, reducing WiFi overhead. However, we limit batching - // to 3 messages to avoid excessive LWIP buffer pressure on memory-constrained - // devices like ESP8266. LWIP's TCP_OVERSIZE option coalesces the data into - // shared pbufs, but holding data too long waiting for Nagle's timer causes - // buffer exhaustion and dropped messages. + // to avoid excessive LWIP buffer pressure on memory-constrained devices. + // LWIP's TCP_OVERSIZE option coalesces the data into shared pbufs, but + // holding data too long waiting for Nagle's timer causes buffer exhaustion + // and dropped messages. // - // Flow: Log 1 (Nagle on) -> Log 2 (Nagle on) -> Log 3 (NODELAY, flush all) + // ESP32 (TCP_SND_BUF=4×MSS+) / RP2040 (8×MSS) / LibreTiny (4×MSS): 4 logs per cycle + // ESP8266 (2×MSS): 3 logs per cycle (tightest buffers) + // + // Flow (ESP32/RP2040/LT): Log 1 (Nagle on) -> Log 2 -> Log 3 -> Log 4 (NODELAY, flush) + // Flow (ESP8266): Log 1 (Nagle on) -> Log 2 -> Log 3 (NODELAY, flush all) // void set_nodelay_for_message(bool is_log_message) { if (!is_log_message) { @@ -150,7 +154,7 @@ class APIFrameHelper { return; } - // Log messages 1-3: state transitions -1 -> 1 -> 2 -> -1 (flush on 3rd) + // Log messages: state transitions -1 -> 1 -> ... -> LOG_NAGLE_COUNT -> -1 (flush) if (this->nodelay_state_ == NODELAY_ON) { this->set_nodelay_raw_(false); this->nodelay_state_ = 1; @@ -255,10 +259,16 @@ class APIFrameHelper { uint8_t tx_buf_tail_{0}; uint8_t tx_buf_count_{0}; // Nagle batching state for log messages. NODELAY_ON (-1) means NODELAY is enabled - // (immediate send). Values 1-2 count log messages in the current Nagle batch. + // (immediate send). Values 1..LOG_NAGLE_COUNT count log messages in the current Nagle batch. // After LOG_NAGLE_COUNT logs, we switch to NODELAY to flush and reset. + // ESP8266 has the tightest TCP send buffer (2×MSS) and needs conservative batching. + // ESP32 (4×MSS+), RP2040 (8×MSS), and LibreTiny (4×MSS) can coalesce more. static constexpr int8_t NODELAY_ON = -1; +#ifdef USE_ESP8266 static constexpr int8_t LOG_NAGLE_COUNT = 2; +#else + static constexpr int8_t LOG_NAGLE_COUNT = 3; +#endif int8_t nodelay_state_{NODELAY_ON}; // Internal helper to set TCP_NODELAY socket option diff --git a/esphome/components/api/proto.h b/esphome/components/api/proto.h index d1c955b1fb..814a3f4456 100644 --- a/esphome/components/api/proto.h +++ b/esphome/components/api/proto.h @@ -602,7 +602,7 @@ class ProtoSize { static constexpr uint32_t calc_sint32(uint32_t field_id_size, int32_t value) { return value ? field_id_size + varint(encode_zigzag32(value)) : 0; } - static constexpr uint32_t calc_sint32_force(uint32_t field_id_size, int32_t value) { + static constexpr inline uint32_t ESPHOME_ALWAYS_INLINE calc_sint32_force(uint32_t field_id_size, int32_t value) { return field_id_size + varint(encode_zigzag32(value)); } static constexpr uint32_t calc_int64(uint32_t field_id_size, int64_t value) { @@ -614,13 +614,13 @@ class ProtoSize { static constexpr uint32_t calc_uint64(uint32_t field_id_size, uint64_t value) { return value ? field_id_size + varint(value) : 0; } - static constexpr uint32_t calc_uint64_force(uint32_t field_id_size, uint64_t value) { + static constexpr inline uint32_t ESPHOME_ALWAYS_INLINE calc_uint64_force(uint32_t field_id_size, uint64_t value) { return field_id_size + varint(value); } static constexpr uint32_t calc_length(uint32_t field_id_size, size_t len) { return len ? field_id_size + varint(static_cast(len)) + static_cast(len) : 0; } - static constexpr uint32_t calc_length_force(uint32_t field_id_size, size_t len) { + static constexpr inline uint32_t ESPHOME_ALWAYS_INLINE calc_length_force(uint32_t field_id_size, size_t len) { return field_id_size + varint(static_cast(len)) + static_cast(len); } static constexpr uint32_t calc_sint64(uint32_t field_id_size, int64_t value) { @@ -638,7 +638,8 @@ class ProtoSize { static constexpr uint32_t calc_message(uint32_t field_id_size, uint32_t nested_size) { return nested_size ? field_id_size + varint(nested_size) + nested_size : 0; } - static constexpr uint32_t calc_message_force(uint32_t field_id_size, uint32_t nested_size) { + static constexpr inline uint32_t ESPHOME_ALWAYS_INLINE calc_message_force(uint32_t field_id_size, + uint32_t nested_size) { return field_id_size + varint(nested_size) + nested_size; } }; diff --git a/esphome/components/const/__init__.py b/esphome/components/const/__init__.py index 059bf3f26a..f6da32569f 100644 --- a/esphome/components/const/__init__.py +++ b/esphome/components/const/__init__.py @@ -3,6 +3,7 @@ CODEOWNERS = ["@esphome/core"] CONF_BYTE_ORDER = "byte_order" +CONF_CLIMATE_ID = "climate_id" BYTE_ORDER_LITTLE = "little_endian" BYTE_ORDER_BIG = "big_endian" diff --git a/esphome/components/esp32_rmt_led_strip/led_strip.cpp b/esphome/components/esp32_rmt_led_strip/led_strip.cpp index 66b41931aa..ca97a181fd 100644 --- a/esphome/components/esp32_rmt_led_strip/led_strip.cpp +++ b/esphome/components/esp32_rmt_led_strip/led_strip.cpp @@ -99,8 +99,6 @@ void ESP32RMTLEDStripLightOutput::setup() { channel.gpio_num = gpio_num_t(this->pin_); channel.mem_block_symbols = this->rmt_symbols_; channel.trans_queue_depth = 1; - channel.flags.io_loop_back = 0; - channel.flags.io_od_mode = 0; channel.flags.invert_out = this->invert_out_; channel.flags.with_dma = this->use_dma_; channel.intr_priority = 0; diff --git a/esphome/components/mipi_dsi/mipi_dsi.cpp b/esphome/components/mipi_dsi/mipi_dsi.cpp index 815b9d75a1..7103e0868d 100644 --- a/esphome/components/mipi_dsi/mipi_dsi.cpp +++ b/esphome/components/mipi_dsi/mipi_dsi.cpp @@ -54,6 +54,17 @@ void MIPI_DSI::setup() { this->smark_failed(LOG_STR("new_panel_io_dbi failed"), err); return; } + // clang-format off +#if ESP_IDF_VERSION >= ESP_IDF_VERSION_VAL(6, 0, 0) + auto color_format = LCD_COLOR_FMT_RGB565; + if (this->color_depth_ == display::COLOR_BITNESS_888) { + color_format = LCD_COLOR_FMT_RGB888; + } + esp_lcd_dpi_panel_config_t dpi_config = {.virtual_channel = 0, + .dpi_clk_src = MIPI_DSI_DPI_CLK_SRC_DEFAULT, + .dpi_clock_freq_mhz = this->pclk_frequency_, + .in_color_format = color_format, +#else auto pixel_format = LCD_COLOR_PIXEL_FORMAT_RGB565; if (this->color_depth_ == display::COLOR_BITNESS_888) { pixel_format = LCD_COLOR_PIXEL_FORMAT_RGB888; @@ -62,6 +73,7 @@ void MIPI_DSI::setup() { .dpi_clk_src = MIPI_DSI_DPI_CLK_SRC_DEFAULT, .dpi_clock_freq_mhz = this->pclk_frequency_, .pixel_format = pixel_format, +#endif .num_fbs = 1, // number of frame buffers to allocate .video_timing = { @@ -77,6 +89,7 @@ void MIPI_DSI::setup() { .flags = { .use_dma2d = true, }}; + // clang-format on err = esp_lcd_new_panel_dpi(this->bus_handle_, &dpi_config, &this->handle_); if (err != ESP_OK) { this->smark_failed(LOG_STR("esp_lcd_new_panel_dpi failed"), err); diff --git a/esphome/components/pid/climate.py b/esphome/components/pid/climate.py index 0e66b67637..18e33b8039 100644 --- a/esphome/components/pid/climate.py +++ b/esphome/components/pid/climate.py @@ -57,7 +57,7 @@ CONFIG_SCHEMA = cv.All( cv.Optional(CONF_KD_MULTIPLIER, default=0.0): cv.float_, cv.Optional( CONF_DEADBAND_OUTPUT_AVERAGING_SAMPLES, default=1 - ): cv.int_, + ): cv.positive_not_null_int, } ), cv.Required(CONF_CONTROL_PARAMETERS): cv.Schema( @@ -68,8 +68,12 @@ CONFIG_SCHEMA = cv.All( cv.Optional(CONF_STARTING_INTEGRAL_TERM, default=0.0): cv.float_, cv.Optional(CONF_MIN_INTEGRAL, default=-1): cv.float_, cv.Optional(CONF_MAX_INTEGRAL, default=1): cv.float_, - cv.Optional(CONF_DERIVATIVE_AVERAGING_SAMPLES, default=1): cv.int_, - cv.Optional(CONF_OUTPUT_AVERAGING_SAMPLES, default=1): cv.int_, + cv.Optional( + CONF_DERIVATIVE_AVERAGING_SAMPLES, default=1 + ): cv.positive_not_null_int, + cv.Optional( + CONF_OUTPUT_AVERAGING_SAMPLES, default=1 + ): cv.positive_not_null_int, } ), } @@ -102,13 +106,15 @@ async def to_code(config): cg.add(var.set_starting_integral_term(params[CONF_STARTING_INTEGRAL_TERM])) cg.add(var.set_derivative_samples(params[CONF_DERIVATIVE_AVERAGING_SAMPLES])) - cg.add(var.set_output_samples(params[CONF_OUTPUT_AVERAGING_SAMPLES])) + output_samples = params[CONF_OUTPUT_AVERAGING_SAMPLES] + cg.add(var.set_output_samples(output_samples)) if CONF_MIN_INTEGRAL in params: cg.add(var.set_min_integral(params[CONF_MIN_INTEGRAL])) if CONF_MAX_INTEGRAL in params: cg.add(var.set_max_integral(params[CONF_MAX_INTEGRAL])) + deadband_output_samples = 1 if CONF_DEADBAND_PARAMETERS in config: params = config[CONF_DEADBAND_PARAMETERS] cg.add(var.set_threshold_low(params[CONF_THRESHOLD_LOW])) @@ -116,11 +122,11 @@ async def to_code(config): cg.add(var.set_kp_multiplier(params[CONF_KP_MULTIPLIER])) cg.add(var.set_ki_multiplier(params[CONF_KI_MULTIPLIER])) cg.add(var.set_kd_multiplier(params[CONF_KD_MULTIPLIER])) - cg.add( - var.set_deadband_output_samples( - params[CONF_DEADBAND_OUTPUT_AVERAGING_SAMPLES] - ) - ) + deadband_output_samples = params[CONF_DEADBAND_OUTPUT_AVERAGING_SAMPLES] + cg.add(var.set_deadband_output_samples(deadband_output_samples)) + + # Single shared output buffer sized to max of both modes + cg.add(var.init_output_buffer(max(output_samples, deadband_output_samples))) cg.add(var.set_default_target_temperature(config[CONF_DEFAULT_TARGET_TEMPERATURE])) diff --git a/esphome/components/pid/pid_climate.h b/esphome/components/pid/pid_climate.h index dc0a92efed..3708c29ff1 100644 --- a/esphome/components/pid/pid_climate.h +++ b/esphome/components/pid/pid_climate.h @@ -28,7 +28,11 @@ class PIDClimate : public climate::Climate, public Component { void set_min_integral(float min_integral) { controller_.min_integral_ = min_integral; } void set_max_integral(float max_integral) { controller_.max_integral_ = max_integral; } void set_output_samples(int in) { controller_.output_samples_ = in; } - void set_derivative_samples(int in) { controller_.derivative_samples_ = in; } + void set_derivative_samples(int in) { + controller_.derivative_samples_ = in; + if (in > 1) // No allocation needed when samples=1 (ring_buffer_average_ short-circuits) + controller_.derivative_window_.init(in); + } void set_threshold_low(float in) { controller_.threshold_low_ = in; } void set_threshold_high(float in) { controller_.threshold_high_ = in; } @@ -38,6 +42,10 @@ class PIDClimate : public climate::Climate, public Component { void set_starting_integral_term(float in) { controller_.set_starting_integral_term(in); } void set_deadband_output_samples(int in) { controller_.deadband_output_samples_ = in; } + void init_output_buffer(int size) { + if (size > 1) // No allocation needed when samples=1 (ring_buffer_average_ short-circuits) + controller_.output_window_.init(size); + } float get_output_value() const { return output_value_; } float get_error_value() const { return controller_.error_; } diff --git a/esphome/components/pid/pid_controller.cpp b/esphome/components/pid/pid_controller.cpp index 5d7aecdb05..cab15331cd 100644 --- a/esphome/components/pid/pid_controller.cpp +++ b/esphome/components/pid/pid_controller.cpp @@ -21,9 +21,9 @@ float PIDController::update(float setpoint, float process_value) { // u(t) := p(t) + i(t) + d(t) float output = proportional_term_ + integral_term_ + derivative_term_; - // smooth/sample the output + // smooth/sample the output using shared buffer with mode-appropriate sample count int samples = in_deadband() ? deadband_output_samples_ : output_samples_; - return weighted_average_(output_list_, output, samples); + return ring_buffer_average_(output_window_, output, samples); } bool PIDController::in_deadband() { @@ -83,7 +83,7 @@ void PIDController::calculate_derivative_term_(float setpoint) { previous_setpoint_ = setpoint; // smooth the derivative samples - derivative = weighted_average_(derivative_list_, derivative, derivative_samples_); + derivative = ring_buffer_average_(derivative_window_, derivative, derivative_samples_); derivative_term_ = kd_ * derivative; @@ -93,25 +93,23 @@ void PIDController::calculate_derivative_term_(float setpoint) { } } -float PIDController::weighted_average_(std::deque &list, float new_value, int samples) { - // if only 1 sample needed, clear the list and return - if (samples == 1) { - list.clear(); +float PIDController::ring_buffer_average_(FixedRingBuffer &buf, float new_value, int max_samples) { + // if only 1 sample needed (or invalid), clear the buffer and return + if (max_samples <= 1) { + buf.clear(); return new_value; } - // add the new item to the list - list.push_front(new_value); + // Trim oldest entries to make room (handles mode-switching where buffer + // may have more entries than the current mode needs) + while (buf.size() >= static_cast(max_samples)) + buf.pop(); + buf.push(new_value); - // keep only 'samples' readings, by popping off the back of the list - while (samples > 0 && list.size() > static_cast(samples)) - list.pop_back(); - - // calculate and return the average of all values in the list float sum = 0; - for (auto &elem : list) - sum += elem; - return sum / list.size(); + for (auto val : buf) + sum += val; + return sum / buf.size(); } float PIDController::calculate_relative_time_() { diff --git a/esphome/components/pid/pid_controller.h b/esphome/components/pid/pid_controller.h index e2a7030b57..6848a23965 100644 --- a/esphome/components/pid/pid_controller.h +++ b/esphome/components/pid/pid_controller.h @@ -1,6 +1,7 @@ #pragma once + #include "esphome/core/hal.h" -#include +#include "esphome/core/helpers.h" #include namespace esphome { @@ -24,10 +25,10 @@ struct PIDController { /// Differential gain K_d. float kd_ = 0; - // smooth the derivative value using a weighted average over X samples - int derivative_samples_ = 8; + // smooth the derivative value using an average over X samples + int derivative_samples_ = 1; - /// smooth the output value using a weighted average over X values + /// smooth the output value using an average over X values int output_samples_ = 1; float threshold_low_ = 0.0f; @@ -50,7 +51,10 @@ struct PIDController { void calculate_proportional_term_(); void calculate_integral_term_(); void calculate_derivative_term_(float setpoint); - float weighted_average_(std::deque &list, float new_value, int samples); + + /// Ring buffer smoothing using FixedRingBuffer (single allocation at setup) + float ring_buffer_average_(FixedRingBuffer &buf, float new_value, int max_samples); + float calculate_relative_time_(); /// Error from previous update used for derivative term @@ -60,12 +64,12 @@ struct PIDController { float accumulated_integral_ = 0; uint32_t last_time_ = 0; - // this is a list of derivative values for smoothing. - std::deque derivative_list_; + // Ring buffer for derivative smoothing + FixedRingBuffer derivative_window_; - // this is a list of output values for smoothing. - std::deque output_list_; + // Ring buffer for output smoothing (shared between normal and deadband modes) + FixedRingBuffer output_window_; -}; // Struct PID Controller +}; // Struct PIDController } // namespace pid } // namespace esphome diff --git a/esphome/components/pid/sensor/__init__.py b/esphome/components/pid/sensor/__init__.py index 4547f4d708..d26e88e38a 100644 --- a/esphome/components/pid/sensor/__init__.py +++ b/esphome/components/pid/sensor/__init__.py @@ -1,5 +1,6 @@ import esphome.codegen as cg from esphome.components import sensor +from esphome.components.const import CONF_CLIMATE_ID import esphome.config_validation as cv from esphome.const import CONF_TYPE, ICON_GAUGE, STATE_CLASS_MEASUREMENT, UNIT_PERCENT @@ -21,7 +22,6 @@ PID_CLIMATE_SENSOR_TYPES = { "KD": PIDClimateSensorType.PID_SENSOR_TYPE_KD, } -CONF_CLIMATE_ID = "climate_id" CONFIG_SCHEMA = ( sensor.sensor_schema( PIDClimateSensor, diff --git a/esphome/components/remote_receiver/remote_receiver_rmt.cpp b/esphome/components/remote_receiver/remote_receiver_rmt.cpp index 96b23bd0f5..596608a4d0 100644 --- a/esphome/components/remote_receiver/remote_receiver_rmt.cpp +++ b/esphome/components/remote_receiver/remote_receiver_rmt.cpp @@ -44,7 +44,6 @@ void RemoteReceiverComponent::setup() { channel.intr_priority = 0; channel.flags.invert_in = 0; channel.flags.with_dma = this->with_dma_; - channel.flags.io_loop_back = 0; esp_err_t error = rmt_new_rx_channel(&channel, &this->channel_); if (error != ESP_OK) { this->error_code_ = error; diff --git a/esphome/components/remote_transmitter/remote_transmitter_rmt.cpp b/esphome/components/remote_transmitter/remote_transmitter_rmt.cpp index 71773e3ddf..3c9a12d472 100644 --- a/esphome/components/remote_transmitter/remote_transmitter_rmt.cpp +++ b/esphome/components/remote_transmitter/remote_transmitter_rmt.cpp @@ -120,11 +120,13 @@ void RemoteTransmitterComponent::configure_rmt_() { channel.gpio_num = gpio_num_t(this->pin_->get_pin()); channel.mem_block_symbols = this->rmt_symbols_; channel.trans_queue_depth = 1; - channel.flags.io_loop_back = open_drain; - channel.flags.io_od_mode = open_drain; channel.flags.invert_out = 0; channel.flags.with_dma = this->with_dma_; channel.intr_priority = 0; +#if ESP_IDF_VERSION < ESP_IDF_VERSION_VAL(6, 0, 0) + channel.flags.io_loop_back = open_drain; + channel.flags.io_od_mode = open_drain; +#endif error = rmt_new_tx_channel(&channel, &this->channel_); if (error != ESP_OK) { this->error_code_ = error; @@ -136,6 +138,13 @@ void RemoteTransmitterComponent::configure_rmt_() { this->mark_failed(); return; } +#if ESP_IDF_VERSION >= ESP_IDF_VERSION_VAL(6, 0, 0) + if (open_drain) { + gpio_num_t gpio = gpio_num_t(this->pin_->get_pin()); + gpio_od_enable(gpio); + gpio_input_enable(gpio); + } +#endif if (this->pin_->get_flags() & gpio::FLAG_PULLUP) { gpio_pullup_en(gpio_num_t(this->pin_->get_pin())); } else { diff --git a/esphome/components/rp2040/generate_boards.py b/esphome/components/rp2040/generate_boards.py index a0e3699f37..7ea02d185e 100644 --- a/esphome/components/rp2040/generate_boards.py +++ b/esphome/components/rp2040/generate_boards.py @@ -6,6 +6,7 @@ Usage: python esphome/components/rp2040/generate_boards.py import json from pathlib import Path import re +import subprocess import sys from jinja2 import Environment, FileSystemLoader @@ -157,7 +158,7 @@ def generate(arduino_pico_path: Path) -> str: board_pins, boards = load_boards(arduino_pico_path) template = _jinja_env.get_template("boards.jinja2") - return template.render( + content = template.render( cyw43_gpio_offset=CYW43_GPIO_OFFSET, cyw43_max_gpio=CYW43_GPIO_OFFSET + CYW43_GPIO_COUNT - 1, default_max_pin=DEFAULT_MAX_PIN, @@ -165,6 +166,15 @@ def generate(arduino_pico_path: Path) -> str: boards=sorted(boards.items()), ) + # Format output to match pre-commit ruff formatting + result = subprocess.run( + [sys.executable, "-m", "ruff", "format", "--stdin-filename", "boards.py"], + input=content.encode(), + capture_output=True, + check=True, + ) + return result.stdout.decode() + def main(): if len(sys.argv) < 2: diff --git a/esphome/components/runtime_image/bmp_decoder.h b/esphome/components/runtime_image/bmp_decoder.h index 73e54f5430..a52a561584 100644 --- a/esphome/components/runtime_image/bmp_decoder.h +++ b/esphome/components/runtime_image/bmp_decoder.h @@ -26,6 +26,10 @@ class BmpDecoder : public ImageDecoder { int HOT decode(uint8_t *buffer, size_t size) override; bool is_finished() const override { + if (this->bits_per_pixel_ == 0) { + // header not yet received, so dimensions not yet determined + return false; + } // BMP is finished when we've decoded all pixel data return this->paint_index_ >= static_cast(this->width_ * this->height_); } diff --git a/esphome/components/select/select_call.cpp b/esphome/components/select/select_call.cpp index 45fb42c116..83f5052fc8 100644 --- a/esphome/components/select/select_call.cpp +++ b/esphome/components/select/select_call.cpp @@ -41,7 +41,7 @@ SelectCall &SelectCall::with_index(size_t index) { this->operation_ = SELECT_OP_SET; if (index >= this->parent_->size()) { ESP_LOGW(TAG, "'%s' - Index value %zu out of bounds", this->parent_->get_name().c_str(), index); - this->index_ = {}; // Store nullopt for invalid index + this->index_ = nullopt; // Store nullopt for invalid index } else { this->index_ = index; } @@ -52,7 +52,7 @@ optional SelectCall::calculate_target_index_(const char *name) { const auto &options = this->parent_->traits.get_options(); if (options.empty()) { ESP_LOGW(TAG, "'%s' - Select has no options", name); - return {}; + return nullopt; } if (this->operation_ == SELECT_OP_FIRST) { @@ -67,7 +67,7 @@ optional SelectCall::calculate_target_index_(const char *name) { ESP_LOGD(TAG, "'%s' - Setting", name); if (!this->index_.has_value()) { ESP_LOGW(TAG, "'%s' - No option set", name); - return {}; + return nullopt; } return this->index_; } @@ -96,7 +96,7 @@ optional SelectCall::calculate_target_index_(const char *name) { return active_index + 1; } - return {}; // Can't navigate further without cycling + return nullopt; // Can't navigate further without cycling } void SelectCall::perform() { diff --git a/esphome/components/sensor/__init__.py b/esphome/components/sensor/__init__.py index 4be6ed1b84..64d4dc4177 100644 --- a/esphome/components/sensor/__init__.py +++ b/esphome/components/sensor/__init__.py @@ -403,9 +403,9 @@ async def filter_out_filter_to_code(config, filter_id): QUANTILE_SCHEMA = cv.All( cv.Schema( { - cv.Optional(CONF_WINDOW_SIZE, default=5): cv.positive_not_null_int, - cv.Optional(CONF_SEND_EVERY, default=5): cv.positive_not_null_int, - cv.Optional(CONF_SEND_FIRST_AT, default=1): cv.positive_not_null_int, + cv.Optional(CONF_WINDOW_SIZE, default=5): cv.int_range(min=1, max=65535), + cv.Optional(CONF_SEND_EVERY, default=5): cv.int_range(min=1, max=65535), + cv.Optional(CONF_SEND_FIRST_AT, default=1): cv.int_range(min=1, max=65535), cv.Optional(CONF_QUANTILE, default=0.9): cv.zero_to_one_float, } ), @@ -427,9 +427,9 @@ async def quantile_filter_to_code(config, filter_id): MEDIAN_SCHEMA = cv.All( cv.Schema( { - cv.Optional(CONF_WINDOW_SIZE, default=5): cv.positive_not_null_int, - cv.Optional(CONF_SEND_EVERY, default=5): cv.positive_not_null_int, - cv.Optional(CONF_SEND_FIRST_AT, default=1): cv.positive_not_null_int, + cv.Optional(CONF_WINDOW_SIZE, default=5): cv.int_range(min=1, max=65535), + cv.Optional(CONF_SEND_EVERY, default=5): cv.int_range(min=1, max=65535), + cv.Optional(CONF_SEND_FIRST_AT, default=1): cv.int_range(min=1, max=65535), } ), validate_send_first_at, @@ -449,9 +449,9 @@ async def median_filter_to_code(config, filter_id): MIN_SCHEMA = cv.All( cv.Schema( { - cv.Optional(CONF_WINDOW_SIZE, default=5): cv.positive_not_null_int, - cv.Optional(CONF_SEND_EVERY, default=5): cv.positive_not_null_int, - cv.Optional(CONF_SEND_FIRST_AT, default=1): cv.positive_not_null_int, + cv.Optional(CONF_WINDOW_SIZE, default=5): cv.int_range(min=1, max=65535), + cv.Optional(CONF_SEND_EVERY, default=5): cv.int_range(min=1, max=65535), + cv.Optional(CONF_SEND_FIRST_AT, default=1): cv.int_range(min=1, max=65535), } ), validate_send_first_at, @@ -483,9 +483,9 @@ async def min_filter_to_code(config, filter_id): MAX_SCHEMA = cv.All( cv.Schema( { - cv.Optional(CONF_WINDOW_SIZE, default=5): cv.positive_not_null_int, - cv.Optional(CONF_SEND_EVERY, default=5): cv.positive_not_null_int, - cv.Optional(CONF_SEND_FIRST_AT, default=1): cv.positive_not_null_int, + cv.Optional(CONF_WINDOW_SIZE, default=5): cv.int_range(min=1, max=65535), + cv.Optional(CONF_SEND_EVERY, default=5): cv.int_range(min=1, max=65535), + cv.Optional(CONF_SEND_FIRST_AT, default=1): cv.int_range(min=1, max=65535), } ), validate_send_first_at, @@ -509,9 +509,9 @@ async def max_filter_to_code(config, filter_id): SLIDING_AVERAGE_SCHEMA = cv.All( cv.Schema( { - cv.Optional(CONF_WINDOW_SIZE, default=15): cv.positive_not_null_int, - cv.Optional(CONF_SEND_EVERY, default=15): cv.positive_not_null_int, - cv.Optional(CONF_SEND_FIRST_AT, default=1): cv.positive_not_null_int, + cv.Optional(CONF_WINDOW_SIZE, default=15): cv.int_range(min=1, max=65535), + cv.Optional(CONF_SEND_EVERY, default=15): cv.int_range(min=1, max=65535), + cv.Optional(CONF_SEND_FIRST_AT, default=1): cv.int_range(min=1, max=65535), } ), validate_send_first_at, @@ -540,8 +540,8 @@ EXPONENTIAL_AVERAGE_SCHEMA = cv.All( cv.Schema( { cv.Optional(CONF_ALPHA, default=0.1): cv.positive_float, - cv.Optional(CONF_SEND_EVERY, default=15): cv.positive_not_null_int, - cv.Optional(CONF_SEND_FIRST_AT, default=1): cv.positive_not_null_int, + cv.Optional(CONF_SEND_EVERY, default=15): cv.int_range(min=1, max=65535), + cv.Optional(CONF_SEND_FIRST_AT, default=1): cv.int_range(min=1, max=65535), } ), validate_send_first_at, diff --git a/esphome/components/sensor/filter.cpp b/esphome/components/sensor/filter.cpp index 0fe1effe17..d995ee4111 100644 --- a/esphome/components/sensor/filter.cpp +++ b/esphome/components/sensor/filter.cpp @@ -41,26 +41,14 @@ void Filter::initialize(Sensor *parent, Filter *next) { } // SlidingWindowFilter -SlidingWindowFilter::SlidingWindowFilter(size_t window_size, size_t send_every, size_t send_first_at) - : window_size_(window_size), send_every_(send_every), send_at_(send_every - send_first_at) { - // Allocate ring buffer once at initialization +SlidingWindowFilter::SlidingWindowFilter(uint16_t window_size, uint16_t send_every, uint16_t send_first_at) + : send_every_(send_every), send_at_(send_every - send_first_at) { this->window_.init(window_size); } optional SlidingWindowFilter::new_value(float value) { - // Add value to ring buffer - if (this->window_count_ < this->window_size_) { - // Buffer not yet full - just append - this->window_.push_back(value); - this->window_count_++; - } else { - // Buffer full - overwrite oldest value (ring buffer) - this->window_[this->window_head_] = value; - this->window_head_++; - if (this->window_head_ >= this->window_size_) { - this->window_head_ = 0; - } - } + // Add value to ring buffer (overwrites oldest when full) + this->window_.push_overwrite(value); // Check if we should send a result if (++this->send_at_ >= this->send_every_) { @@ -77,9 +65,8 @@ FixedVector SortedWindowFilter::get_window_values_() { // Copy window without NaN values using FixedVector (no heap allocation) // Returns unsorted values - caller will use std::nth_element for partial sorting as needed FixedVector values; - values.init(this->window_count_); - for (size_t i = 0; i < this->window_count_; i++) { - float v = this->window_[i]; + values.init(this->window_.size()); + for (float v : this->window_) { if (!std::isnan(v)) { values.push_back(v); } @@ -150,8 +137,7 @@ float MaxFilter::compute_result() { return this->find_extremum_window_count_; i++) { - float v = this->window_[i]; + for (float v : this->window_) { if (!std::isnan(v)) { sum += v; valid_count++; @@ -161,7 +147,7 @@ float SlidingWindowMovingAverageFilter::compute_result() { } // ExponentialMovingAverageFilter -ExponentialMovingAverageFilter::ExponentialMovingAverageFilter(float alpha, size_t send_every, size_t send_first_at) +ExponentialMovingAverageFilter::ExponentialMovingAverageFilter(float alpha, uint16_t send_every, uint16_t send_first_at) : alpha_(alpha), send_every_(send_every), send_at_(send_every - send_first_at) {} optional ExponentialMovingAverageFilter::new_value(float value) { if (!std::isnan(value)) { @@ -183,7 +169,7 @@ optional ExponentialMovingAverageFilter::new_value(float value) { } return {}; } -void ExponentialMovingAverageFilter::set_send_every(size_t send_every) { this->send_every_ = send_every; } +void ExponentialMovingAverageFilter::set_send_every(uint16_t send_every) { this->send_every_ = send_every; } void ExponentialMovingAverageFilter::set_alpha(float alpha) { this->alpha_ = alpha; } // ThrottleAverageFilter @@ -511,7 +497,7 @@ optional ToNTCTemperatureFilter::new_value(float value) { } // StreamingFilter (base class) -StreamingFilter::StreamingFilter(size_t window_size, size_t send_first_at) +StreamingFilter::StreamingFilter(uint16_t window_size, uint16_t send_first_at) : window_size_(window_size), send_first_at_(send_first_at) {} optional StreamingFilter::new_value(float value) { diff --git a/esphome/components/sensor/filter.h b/esphome/components/sensor/filter.h index 8bfcdb37cf..6a76bd373e 100644 --- a/esphome/components/sensor/filter.h +++ b/esphome/components/sensor/filter.h @@ -52,7 +52,7 @@ class Filter { */ class SlidingWindowFilter : public Filter { public: - SlidingWindowFilter(size_t window_size, size_t send_every, size_t send_first_at); + SlidingWindowFilter(uint16_t window_size, uint16_t send_every, uint16_t send_first_at); optional new_value(float value) final; @@ -60,14 +60,10 @@ class SlidingWindowFilter : public Filter { /// Called by new_value() to compute the filtered result from the current window virtual float compute_result() = 0; - /// Access the sliding window values (ring buffer implementation) - /// Use: for (size_t i = 0; i < window_count_; i++) { float val = window_[i]; } - FixedVector window_; - size_t window_head_{0}; ///< Index where next value will be written - size_t window_count_{0}; ///< Number of valid values in window (0 to window_size_) - size_t window_size_; ///< Maximum window size - size_t send_every_; ///< Send result every N values - size_t send_at_; ///< Counter for send_every + /// Sliding window ring buffer - automatically overwrites oldest values when full + FixedRingBuffer window_; + uint16_t send_every_; ///< Send result every N values + uint16_t send_at_; ///< Counter for send_every }; /** Base class for Min/Max filters. @@ -84,8 +80,7 @@ class MinMaxFilter : public SlidingWindowFilter { template float find_extremum_() { float result = NAN; Compare comp; - for (size_t i = 0; i < this->window_count_; i++) { - float v = this->window_[i]; + for (float v : this->window_) { if (!std::isnan(v)) { result = std::isnan(result) ? v : (comp(v, result) ? v : result); } @@ -239,18 +234,18 @@ class SlidingWindowMovingAverageFilter : public SlidingWindowFilter { */ class ExponentialMovingAverageFilter : public Filter { public: - ExponentialMovingAverageFilter(float alpha, size_t send_every, size_t send_first_at); + ExponentialMovingAverageFilter(float alpha, uint16_t send_every, uint16_t send_first_at); optional new_value(float value) override; - void set_send_every(size_t send_every); + void set_send_every(uint16_t send_every); void set_alpha(float alpha); protected: float accumulator_{NAN}; float alpha_; - size_t send_every_; - size_t send_at_; + uint16_t send_every_; + uint16_t send_at_; bool first_value_{true}; }; @@ -570,7 +565,7 @@ class ToNTCTemperatureFilter : public Filter { */ class StreamingFilter : public Filter { public: - StreamingFilter(size_t window_size, size_t send_first_at); + StreamingFilter(uint16_t window_size, uint16_t send_first_at); optional new_value(float value) final; @@ -584,9 +579,9 @@ class StreamingFilter : public Filter { /// Called by new_value() to reset internal state after sending a result virtual void reset_batch() = 0; - size_t window_size_; - size_t count_{0}; - size_t send_first_at_; + uint16_t window_size_; + uint16_t count_{0}; + uint16_t send_first_at_; bool first_send_{true}; }; diff --git a/esphome/components/template/text/template_text.h b/esphome/components/template/text/template_text.h index 7f176db09e..229a61d9b8 100644 --- a/esphome/components/template/text/template_text.h +++ b/esphome/components/template/text/template_text.h @@ -24,23 +24,23 @@ class TemplateTextSaverBase { template class TextSaver : public TemplateTextSaverBase { public: bool save(const std::string &value) override { - int diff = value.compare(this->prev_); - if (diff != 0) { - // If string is bigger than the allocation, do not save it. - // We don't need to waste ram setting prev_value either. - int size = value.size(); - if (size <= SZ) { - // Make it into a length prefixed thing - unsigned char temp[SZ + 1]; - memcpy(temp + 1, value.c_str(), size); - // SZ should be pre checked at the schema level, it can't go past the char range. - temp[0] = ((unsigned char) size); - this->pref_.save(&temp); - this->prev_.assign(value); - return true; - } + if (value == this->prev_) { + return true; // No change, nothing to save } - return false; + // If string is bigger than the allocation, do not save it. + // We don't need to waste ram setting prev_value either. + int size = value.size(); + if (size > SZ) { + return false; + } + // Make it into a length prefixed thing + unsigned char temp[SZ + 1]; + memcpy(temp + 1, value.c_str(), size); + // SZ should be pre checked at the schema level, it can't go past the char range. + temp[0] = ((unsigned char) size); + this->pref_.save(&temp); + this->prev_.assign(value); + return true; } // Make the preference object. Fill the provided location with the saved data diff --git a/esphome/components/wifi/wifi_component.cpp b/esphome/components/wifi/wifi_component.cpp index 09f883ed61..346276692a 100644 --- a/esphome/components/wifi/wifi_component.cpp +++ b/esphome/components/wifi/wifi_component.cpp @@ -6,7 +6,7 @@ #include #ifdef USE_ESP32 -#if (ESP_IDF_VERSION_MAJOR >= 5 && ESP_IDF_VERSION_MINOR >= 1) +#if ESP_IDF_VERSION >= ESP_IDF_VERSION_VAL(5, 1, 0) #include #else #include diff --git a/esphome/components/wifi/wifi_component.h b/esphome/components/wifi/wifi_component.h index 883cc1344b..aeb32352a9 100644 --- a/esphome/components/wifi/wifi_component.h +++ b/esphome/components/wifi/wifi_component.h @@ -18,7 +18,7 @@ #endif #if defined(USE_ESP32) && defined(USE_WIFI_WPA2_EAP) -#if (ESP_IDF_VERSION_MAJOR >= 5) && (ESP_IDF_VERSION_MINOR >= 1) +#if ESP_IDF_VERSION >= ESP_IDF_VERSION_VAL(5, 1, 0) #include #else #include diff --git a/esphome/components/wifi/wifi_component_esp_idf.cpp b/esphome/components/wifi/wifi_component_esp_idf.cpp index eca3f19249..2866ec1513 100644 --- a/esphome/components/wifi/wifi_component_esp_idf.cpp +++ b/esphome/components/wifi/wifi_component_esp_idf.cpp @@ -17,7 +17,7 @@ #include #include #ifdef USE_WIFI_WPA2_EAP -#if (ESP_IDF_VERSION_MAJOR >= 5) && (ESP_IDF_VERSION_MINOR >= 1) +#if ESP_IDF_VERSION >= ESP_IDF_VERSION_VAL(5, 1, 0) #include #else #include @@ -75,7 +75,11 @@ struct IDFWiFiEvent { #if USE_NETWORK_IPV6 ip_event_got_ip6_t ip_got_ip6; #endif /* USE_NETWORK_IPV6 */ +#if ESP_IDF_VERSION >= ESP_IDF_VERSION_VAL(6, 0, 0) + ip_event_assigned_ip_to_client_t ip_assigned_ip_to_client; +#else ip_event_ap_staipassigned_t ip_ap_staipassigned; +#endif } data; }; @@ -116,8 +120,13 @@ void event_handler(void *arg, esp_event_base_t event_base, int32_t event_id, voi memcpy(&event.data.ap_staconnected, event_data, sizeof(wifi_event_ap_staconnected_t)); } else if (event_base == WIFI_EVENT && event_id == WIFI_EVENT_AP_STADISCONNECTED) { memcpy(&event.data.ap_stadisconnected, event_data, sizeof(wifi_event_ap_stadisconnected_t)); +#if ESP_IDF_VERSION >= ESP_IDF_VERSION_VAL(6, 0, 0) + } else if (event_base == IP_EVENT && event_id == IP_EVENT_ASSIGNED_IP_TO_CLIENT) { + memcpy(&event.data.ip_assigned_ip_to_client, event_data, sizeof(ip_event_assigned_ip_to_client_t)); +#else } else if (event_base == IP_EVENT && event_id == IP_EVENT_AP_STAIPASSIGNED) { memcpy(&event.data.ip_ap_staipassigned, event_data, sizeof(ip_event_ap_staipassigned_t)); +#endif } else { // did not match any event, don't send anything return; @@ -407,7 +416,7 @@ bool WiFiComponent::wifi_sta_connect_(const WiFiAP &ap) { if (eap_opt.has_value()) { // note: all certificates and keys have to be null terminated. Lengths are appended by +1 to include \0. EAPAuth eap = *eap_opt; -#if (ESP_IDF_VERSION_MAJOR >= 5) && (ESP_IDF_VERSION_MINOR >= 1) +#if ESP_IDF_VERSION >= ESP_IDF_VERSION_VAL(5, 1, 0) err = esp_eap_client_set_identity((uint8_t *) eap.identity.c_str(), eap.identity.length()); #else err = esp_wifi_sta_wpa2_ent_set_identity((uint8_t *) eap.identity.c_str(), eap.identity.length()); @@ -419,7 +428,7 @@ bool WiFiComponent::wifi_sta_connect_(const WiFiAP &ap) { int client_cert_len = strlen(eap.client_cert); int client_key_len = strlen(eap.client_key); if (ca_cert_len) { -#if (ESP_IDF_VERSION_MAJOR >= 5) && (ESP_IDF_VERSION_MINOR >= 1) +#if ESP_IDF_VERSION >= ESP_IDF_VERSION_VAL(5, 1, 0) err = esp_eap_client_set_ca_cert((uint8_t *) eap.ca_cert, ca_cert_len + 1); #else err = esp_wifi_sta_wpa2_ent_set_ca_cert((uint8_t *) eap.ca_cert, ca_cert_len + 1); @@ -432,7 +441,7 @@ bool WiFiComponent::wifi_sta_connect_(const WiFiAP &ap) { // validation is not required as the config tool has already validated it if (client_cert_len && client_key_len) { // if we have certs, this must be EAP-TLS -#if (ESP_IDF_VERSION_MAJOR >= 5) && (ESP_IDF_VERSION_MINOR >= 1) +#if ESP_IDF_VERSION >= ESP_IDF_VERSION_VAL(5, 1, 0) err = esp_eap_client_set_certificate_and_key((uint8_t *) eap.client_cert, client_cert_len + 1, (uint8_t *) eap.client_key, client_key_len + 1, (uint8_t *) eap.password.c_str(), eap.password.length()); @@ -446,7 +455,7 @@ bool WiFiComponent::wifi_sta_connect_(const WiFiAP &ap) { } } else { // in the absence of certs, assume this is username/password based -#if (ESP_IDF_VERSION_MAJOR >= 5) && (ESP_IDF_VERSION_MINOR >= 1) +#if ESP_IDF_VERSION >= ESP_IDF_VERSION_VAL(5, 1, 0) err = esp_eap_client_set_username((uint8_t *) eap.username.c_str(), eap.username.length()); #else err = esp_wifi_sta_wpa2_ent_set_username((uint8_t *) eap.username.c_str(), eap.username.length()); @@ -454,7 +463,7 @@ bool WiFiComponent::wifi_sta_connect_(const WiFiAP &ap) { if (err != ESP_OK) { ESP_LOGV(TAG, "set_username failed %d", err); } -#if (ESP_IDF_VERSION_MAJOR >= 5) && (ESP_IDF_VERSION_MINOR >= 1) +#if ESP_IDF_VERSION >= ESP_IDF_VERSION_VAL(5, 1, 0) err = esp_eap_client_set_password((uint8_t *) eap.password.c_str(), eap.password.length()); #else err = esp_wifi_sta_wpa2_ent_set_password((uint8_t *) eap.password.c_str(), eap.password.length()); @@ -463,7 +472,7 @@ bool WiFiComponent::wifi_sta_connect_(const WiFiAP &ap) { ESP_LOGV(TAG, "set_password failed %d", err); } // set TTLS Phase 2, defaults to MSCHAPV2 -#if (ESP_IDF_VERSION_MAJOR >= 5) && (ESP_IDF_VERSION_MINOR >= 1) +#if ESP_IDF_VERSION >= ESP_IDF_VERSION_VAL(5, 1, 0) err = esp_eap_client_set_ttls_phase2_method(eap.ttls_phase_2); #else err = esp_wifi_sta_wpa2_ent_set_ttls_phase2_method(eap.ttls_phase_2); @@ -472,7 +481,7 @@ bool WiFiComponent::wifi_sta_connect_(const WiFiAP &ap) { ESP_LOGV(TAG, "set_ttls_phase2_method failed %d", err); } } -#if (ESP_IDF_VERSION_MAJOR >= 5) && (ESP_IDF_VERSION_MINOR >= 1) +#if ESP_IDF_VERSION >= ESP_IDF_VERSION_VAL(5, 1, 0) err = esp_wifi_sta_enterprise_enable(); #else err = esp_wifi_sta_wpa2_ent_enable(); @@ -628,14 +637,26 @@ const char *get_disconnect_reason_str(uint8_t reason) { return "Auth Expired"; case WIFI_REASON_AUTH_LEAVE: return "Auth Leave"; +#if ESP_IDF_VERSION >= ESP_IDF_VERSION_VAL(6, 0, 0) + case WIFI_REASON_DISASSOC_DUE_TO_INACTIVITY: + return "Disassociated Due to Inactivity"; +#else case WIFI_REASON_ASSOC_EXPIRE: return "Association Expired"; +#endif case WIFI_REASON_ASSOC_TOOMANY: return "Too Many Associations"; +#if ESP_IDF_VERSION >= ESP_IDF_VERSION_VAL(6, 0, 0) + case WIFI_REASON_CLASS2_FRAME_FROM_NONAUTH_STA: + return "Class 2 Frame from Non-Authenticated STA"; + case WIFI_REASON_CLASS3_FRAME_FROM_NONASSOC_STA: + return "Class 3 Frame from Non-Associated STA"; +#else case WIFI_REASON_NOT_AUTHED: return "Not Authenticated"; case WIFI_REASON_NOT_ASSOCED: return "Not Associated"; +#endif case WIFI_REASON_ASSOC_LEAVE: return "Association Leave"; case WIFI_REASON_ASSOC_NOT_AUTHED: @@ -688,7 +709,7 @@ const char *get_disconnect_reason_str(uint8_t reason) { return "Association comeback time too long"; case WIFI_REASON_SA_QUERY_TIMEOUT: return "SA query timeout"; -#if (ESP_IDF_VERSION_MAJOR >= 5) && (ESP_IDF_VERSION_MINOR >= 2) +#if ESP_IDF_VERSION >= ESP_IDF_VERSION_VAL(5, 2, 0) case WIFI_REASON_NO_AP_FOUND_W_COMPATIBLE_SECURITY: return "No AP found with compatible security"; case WIFI_REASON_NO_AP_FOUND_IN_AUTHMODE_THRESHOLD: @@ -917,8 +938,13 @@ void WiFiComponent::wifi_process_event_(IDFWiFiEvent *data) { ESP_LOGV(TAG, "AP client disconnected MAC=%s", mac_buf); #endif +#if ESP_IDF_VERSION >= ESP_IDF_VERSION_VAL(6, 0, 0) + } else if (data->event_base == IP_EVENT && data->event_id == IP_EVENT_ASSIGNED_IP_TO_CLIENT) { + const auto &it = data->data.ip_assigned_ip_to_client; +#else } else if (data->event_base == IP_EVENT && data->event_id == IP_EVENT_AP_STAIPASSIGNED) { const auto &it = data->data.ip_ap_staipassigned; +#endif ESP_LOGV(TAG, "AP client assigned IP " IPSTR, IP2STR(&it.ip)); } } diff --git a/esphome/core/config.py b/esphome/core/config.py index d4a839cb79..e112720f2b 100644 --- a/esphome/core/config.py +++ b/esphome/core/config.py @@ -589,7 +589,10 @@ async def _add_looping_components() -> None: async def to_code(config: ConfigType) -> None: cg.add_global(cg.global_ns.namespace("esphome").using) # These can be used by user lambdas, put them to default scope + # picolibc (IDF 6.0+) declares isnan in global scope, conflicting with using std::isnan + cg.add_global(cg.RawStatement("#ifndef __PICOLIBC__")) cg.add_global(cg.RawExpression("using std::isnan")) + cg.add_global(cg.RawStatement("#endif")) cg.add_global(cg.RawExpression("using std::min")) cg.add_global(cg.RawExpression("using std::max")) diff --git a/esphome/core/helpers.h b/esphome/core/helpers.h index 20421350bc..a87ac92dbd 100644 --- a/esphome/core/helpers.h +++ b/esphome/core/helpers.h @@ -301,7 +301,7 @@ template class StaticVector { /// Not thread-safe. All access (push/pop/iteration) must occur from a single /// context, or the caller must provide external synchronization. template class StaticRingBuffer { - using index_type = std::conditional_t<(N <= 255), uint8_t, uint16_t>; + using index_type = std::conditional_t<(N <= std::numeric_limits::max()), uint8_t, uint16_t>; public: class Iterator { @@ -356,6 +356,13 @@ template class StaticRingBuffer { index_type size() const { return this->count_; } bool empty() const { return this->count_ == 0; } + /// Clear all elements (reset to empty) + void clear() { + this->head_ = 0; + this->tail_ = 0; + this->count_ = 0; + } + Iterator begin() { return Iterator(this, 0); } Iterator end() { return Iterator(this, this->count_); } ConstIterator begin() const { return ConstIterator(this, 0); } @@ -368,6 +375,128 @@ template class StaticRingBuffer { index_type count_{0}; }; +/// Fixed-capacity circular buffer - allocates once at runtime, never reallocates. +/// Runtime-sized equivalent of StaticRingBuffer - use when capacity is only known at initialization. +/// Supports FIFO push/pop and iteration over queued elements. +/// Not thread-safe. +template::max()> class FixedRingBuffer { + using index_type = std::conditional_t< + (MAX_CAPACITY <= std::numeric_limits::max()), uint8_t, + std::conditional_t<(MAX_CAPACITY <= std::numeric_limits::max()), uint16_t, uint32_t>>; + + public: + class Iterator { + public: + Iterator(FixedRingBuffer *buf, index_type pos) : buf_(buf), pos_(pos) {} + T &operator*() { return buf_->data_[(buf_->head_ + pos_) % buf_->capacity_]; } + Iterator &operator++() { + ++pos_; + return *this; + } + bool operator!=(const Iterator &other) const { return pos_ != other.pos_; } + + private: + FixedRingBuffer *buf_; + index_type pos_; + }; + + class ConstIterator { + public: + ConstIterator(const FixedRingBuffer *buf, index_type pos) : buf_(buf), pos_(pos) {} + const T &operator*() const { return buf_->data_[(buf_->head_ + pos_) % buf_->capacity_]; } + ConstIterator &operator++() { + ++pos_; + return *this; + } + bool operator!=(const ConstIterator &other) const { return pos_ != other.pos_; } + + private: + const FixedRingBuffer *buf_; + index_type pos_; + }; + + FixedRingBuffer() = default; + ~FixedRingBuffer() { + if constexpr (std::is_trivial::value) { + ::operator delete(this->data_); + } else { + delete[] this->data_; + } + } + + // Disable copy + FixedRingBuffer(const FixedRingBuffer &) = delete; + FixedRingBuffer &operator=(const FixedRingBuffer &) = delete; + + /// Allocate capacity - can only be called once + void init(index_type capacity) { + if constexpr (std::is_trivial::value) { + // Raw allocation without initialization (elements are written before read) + // NOLINTNEXTLINE(bugprone-sizeof-expression) + this->data_ = static_cast(::operator new(capacity * sizeof(T))); + } else { + this->data_ = new T[capacity]; + } + this->capacity_ = capacity; + } + + /// Push a value. Returns false if full. + bool push(const T &value) { + if (this->count_ >= this->capacity_) + return false; + this->data_[this->tail_] = value; + this->tail_ = (this->tail_ + 1) % this->capacity_; + ++this->count_; + return true; + } + + /// Push a value, overwriting the oldest if full. + void push_overwrite(const T &value) { + this->data_[this->tail_] = value; + this->tail_ = (this->tail_ + 1) % this->capacity_; + if (this->count_ >= this->capacity_) { + // Buffer full - advance head to drop oldest, count stays at capacity + this->head_ = this->tail_; + } else { + ++this->count_; + } + } + + /// Remove the oldest element. + void pop() { + if (this->count_ > 0) { + this->head_ = (this->head_ + 1) % this->capacity_; + --this->count_; + } + } + + T &front() { return this->data_[this->head_]; } + const T &front() const { return this->data_[this->head_]; } + index_type size() const { return this->count_; } + bool empty() const { return this->count_ == 0; } + index_type capacity() const { return this->capacity_; } + bool full() const { return this->count_ == this->capacity_; } + + /// Clear all elements (reset to empty, keep capacity) + void clear() { + this->head_ = 0; + this->tail_ = 0; + this->count_ = 0; + } + + Iterator begin() { return Iterator(this, 0); } + Iterator end() { return Iterator(this, this->count_); } + ConstIterator begin() const { return ConstIterator(this, 0); } + ConstIterator end() const { return ConstIterator(this, this->count_); } + + protected: + T *data_{nullptr}; + index_type head_{0}; + index_type tail_{0}; + index_type count_{0}; + index_type capacity_{0}; +}; + /// Fixed-capacity vector - allocates once at runtime, never reallocates /// This avoids std::vector template overhead (_M_realloc_insert, _M_default_append) /// when size is known at initialization but not at compile time diff --git a/esphome/core/scheduler.cpp b/esphome/core/scheduler.cpp index 63e1006b03..72b183384e 100644 --- a/esphome/core/scheduler.cpp +++ b/esphome/core/scheduler.cpp @@ -105,10 +105,11 @@ static void validate_static_string(const char *name) { // avoid the main thread modifying the list while it is being accessed. // Calculate random offset for interval timers -// Extracted from set_timer_common_ to reduce code size - float math + random_float() -// only needed for intervals, not timeouts +// Extracted from set_timer_common_ to reduce code size - only needed for intervals, not timeouts uint32_t Scheduler::calculate_interval_offset_(uint32_t delay) { - return static_cast(std::min(delay / 2, MAX_INTERVAL_DELAY) * random_float()); + uint32_t max_offset = std::min(delay / 2, MAX_INTERVAL_DELAY); + // Multiply-and-shift: uniform random in [0, max_offset) without floating point + return static_cast((static_cast(random_uint32()) * max_offset) >> 32); } // Check if a retry was already cancelled in items_ or to_add_ diff --git a/requirements_test.txt b/requirements_test.txt index 93a20896aa..acd8383a2f 100644 --- a/requirements_test.txt +++ b/requirements_test.txt @@ -1,6 +1,6 @@ pylint==4.0.5 flake8==7.3.0 # also change in .pre-commit-config.yaml when updating -ruff==0.15.5 # also change in .pre-commit-config.yaml when updating +ruff==0.15.6 # also change in .pre-commit-config.yaml when updating pyupgrade==3.21.2 # also change in .pre-commit-config.yaml when updating pre-commit diff --git a/script/determine-jobs.py b/script/determine-jobs.py index 318ac04a7d..6808a3cf6c 100755 --- a/script/determine-jobs.py +++ b/script/determine-jobs.py @@ -6,6 +6,8 @@ what files have changed. It outputs JSON with the following structure: { "integration_tests": true/false, + "integration_tests_run_all": true/false, + "integration_test_files": ["tests/integration/test_foo.py", ...], "clang_tidy": true/false, "clang_format": true/false, "python_linters": true/false, @@ -56,13 +58,13 @@ from helpers import ( core_changed, filter_component_and_test_cpp_files, filter_component_and_test_files, - get_all_dependencies, get_changed_components, get_component_from_path, get_component_test_files, - get_components_from_integration_fixtures, get_components_with_dependencies, get_cpp_changed_components, + get_fixture_to_test_files, + get_integration_test_files_for_components, get_target_branch, git_ls_files, parse_test_filename, @@ -143,65 +145,88 @@ MEMORY_IMPACT_PLATFORM_PREFERENCE = [ ] -def should_run_integration_tests(branch: str | None = None) -> bool: - """Determine if integration tests should run based on changed files. +def determine_integration_tests(branch: str | None = None) -> tuple[bool, list[str]]: + """Determine which integration tests should run based on changed files. - This function is used by the CI workflow to intelligently skip integration tests when they're - not needed, saving significant CI time and resources. + This function is used by the CI workflow to intelligently skip or filter + integration tests, saving significant CI time and resources. - Integration tests will run when ANY of the following conditions are met: + Returns (run_all=True, []) when ANY of the following conditions are met: 1. Core C++ files changed (esphome/core/*) - Any .cpp, .h, .tcc files in the core directory - These files contain fundamental functionality used throughout ESPHome - - Examples: esphome/core/component.cpp, esphome/core/application.h 2. Core Python files changed (esphome/core/*.py) - Only .py files in the esphome/core/ directory - These are core Python files that affect the entire system - - Examples: esphome/core/config.py, esphome/core/__init__.py - - NOT included: esphome/*.py, esphome/dashboard/*.py, esphome/components/*/*.py - 3. Integration test files changed - - Any file in tests/integration/ directory - - This includes test files themselves and fixture YAML files - - Examples: tests/integration/test_api.py, tests/integration/fixtures/api.yaml + 3. Integration test infrastructure files changed + - conftest.py, types.py, const.py, entity_utils.py, state_utils.py, etc. - 4. Components used by integration tests (or their dependencies) changed - - The function parses all YAML files in tests/integration/fixtures/ - - Extracts which components are used in integration tests - - Recursively finds all dependencies of those components - - If any of these components have changes, tests must run - - Example: If api.yaml uses 'sensor' and 'api' components, and 'api' depends on 'socket', - then changes to sensor/, api/, or socket/ components trigger tests + Returns (run_all=False, [test_files...]) when: + + 4. Specific integration test files changed + - Only those specific test files are returned + + 5. Components used by integration tests (or their dependencies) changed + - Only test files whose fixtures use the changed components are returned Args: branch: Branch to compare against. If None, uses default. Returns: - True if integration tests should run, False otherwise. + Tuple of (run_all, test_files) where: + - run_all: True if all integration tests should run + - test_files: List of specific test file paths to run (empty if run_all + is True, or if no tests need to run) """ files = changed_files(branch) if core_changed(files): - # If any core files changed, run integration tests - return True + # If any core files changed, run all integration tests + return (True, []) - # Check if any integration test files changed - if any("tests/integration" in file for file in files): - return True + # If infrastructure Python files changed (conftest, utils, etc.), run all tests + # Excludes test files (test_*.py), fixtures, and non-Python files (README.md) + if any( + f.startswith("tests/integration/") + and f.endswith(".py") + and not f.startswith("tests/integration/test_") + and "/fixtures/" not in f + for f in files + ): + return (True, []) - # Get all components used in integration tests and their dependencies - fixture_components = get_components_from_integration_fixtures() - all_required_components = get_all_dependencies(fixture_components) + # Collect specific test files that need to run + test_files: set[str] = set() + fixture_to_test_files = get_fixture_to_test_files() - # Check if any required components changed - for file in files: - component = get_component_from_path(file) - if component and component in all_required_components: - return True + for f in files: + if f.startswith("tests/integration/test_") and f.endswith(".py"): + test_files.add(f) + elif f.startswith("tests/integration/fixtures/"): + if f.endswith(".yaml"): + # Fixture YAML changed - add corresponding test file(s) + test_files.update(fixture_to_test_files.get(Path(f).stem, ())) + else: + # Non-YAML fixture file changed (e.g., external_components/) + # Run all tests since we can't determine which tests are affected + return (True, []) - return False + # Find test files whose fixtures use any of the changed components + changed_component_set = { + component for file in files if (component := get_component_from_path(file)) + } + if changed_component_set: + test_files.update( + get_integration_test_files_for_components(changed_component_set) + ) + + if test_files: + return (False, sorted(test_files)) + + return (False, []) @cache @@ -682,7 +707,10 @@ def main() -> None: args = parser.parse_args() # Determine what should run - run_integration = should_run_integration_tests(args.branch) + integration_run_all, integration_test_files = determine_integration_tests( + args.branch + ) + run_integration = integration_run_all or bool(integration_test_files) run_clang_tidy = should_run_clang_tidy(args.branch) run_clang_format = should_run_clang_format(args.branch) run_python_linters = should_run_python_linters(args.branch) @@ -810,6 +838,8 @@ def main() -> None: output: dict[str, Any] = { "integration_tests": run_integration, + "integration_tests_run_all": integration_run_all, + "integration_test_files": integration_test_files, "clang_tidy": run_clang_tidy, "clang_tidy_mode": clang_tidy_mode, "clang_format": run_clang_format, diff --git a/script/generate-rp2040-boards.py b/script/generate-rp2040-boards.py new file mode 100755 index 0000000000..1b4846fd2b --- /dev/null +++ b/script/generate-rp2040-boards.py @@ -0,0 +1,61 @@ +#!/usr/bin/env python3 + +from __future__ import annotations + +import argparse +from pathlib import Path +import subprocess +import sys +import tempfile + +from esphome.components.rp2040 import RECOMMENDED_ARDUINO_FRAMEWORK_VERSION +from esphome.components.rp2040.generate_boards import generate +from esphome.helpers import write_file_if_changed + +ver = RECOMMENDED_ARDUINO_FRAMEWORK_VERSION +version_tag: str = f"{ver.major}.{ver.minor}.{ver.patch}" +root: Path = Path(__file__).parent.parent +boards_file_path: Path = root / "esphome" / "components" / "rp2040" / "boards.py" + + +def main(check: bool) -> None: + with tempfile.TemporaryDirectory() as tempdir: + subprocess.run( + [ + "git", + "clone", + "-q", + "-c", + "advice.detachedHead=false", + "--depth", + "1", + "--branch", + version_tag, + "https://github.com/earlephilhower/arduino-pico", + tempdir, + ], + check=True, + ) + + content: str = generate(Path(tempdir)) + + if check: + existing_content: str = boards_file_path.read_text(encoding="utf-8") + if existing_content != content: + print("esphome/components/rp2040/boards.py is not up to date.") + print("Please run `script/generate-rp2040-boards.py`") + sys.exit(1) + print("esphome/components/rp2040/boards.py is up to date") + elif write_file_if_changed(boards_file_path, content): + print("RP2040 boards updated successfully.") + + +if __name__ == "__main__": + parser: argparse.ArgumentParser = argparse.ArgumentParser() + parser.add_argument( + "--check", + help="Check if the boards.py file is up to date.", + action="store_true", + ) + args: argparse.Namespace = parser.parse_args() + main(args.check) diff --git a/script/helpers.py b/script/helpers.py index 6ee286a657..9665af70ec 100644 --- a/script/helpers.py +++ b/script/helpers.py @@ -700,37 +700,141 @@ def get_all_dependencies( return all_components +def _extract_components_from_yaml(config: dict) -> set[str]: + """Extract component names from a parsed YAML config. + + Args: + config: Parsed YAML configuration dictionary + + Returns: + Set of component names found in the config + """ + components: set[str] = set() + + # Add all top-level component keys (skip YAML anchor keys starting with '.') + components.update(k for k in config if isinstance(k, str) and not k.startswith(".")) + + # Add platform values from list entries (e.g., sensor -> platform: template adds "template") + for value in config.values(): + if isinstance(value, list): + components.update( + item["platform"] + for item in value + if isinstance(item, dict) and "platform" in item + ) + + return components + + def get_components_from_integration_fixtures() -> set[str]: """Extract all components used in integration test fixtures. Returns: Set of component names used in integration test fixtures """ + return { + comp + for components in get_components_per_integration_fixture().values() + for comp in components + } + + +@cache +def get_components_per_integration_fixture() -> dict[str, set[str]]: + """Extract components used in each integration test fixture. + + Returns: + Dictionary mapping fixture name (stem) to set of component names + """ from esphome import yaml_util - components: set[str] = set() + result: dict[str, set[str]] = {} fixtures_dir = Path(__file__).parent.parent / "tests" / "integration" / "fixtures" for yaml_file in fixtures_dir.glob("*.yaml"): - config: dict[str, any] | None = yaml_util.load_yaml(yaml_file) + config: dict[str, Any] | None = yaml_util.load_yaml(yaml_file) if not config: continue - # Add all top-level component keys (skip YAML anchor keys starting with '.') - components.update( - k for k in config if isinstance(k, str) and not k.startswith(".") - ) + result[yaml_file.stem] = _extract_components_from_yaml(config) - # Add platform components (e.g., output.template) - for value in config.values(): - if not isinstance(value, list): - continue + return result - for item in value: - if isinstance(item, dict) and "platform" in item: - components.add(item["platform"]) - return components +_TEST_FUNC_RE = re.compile(r"async def (test_\w+)") + + +@cache +def get_fixture_to_test_files() -> dict[str, frozenset[str]]: + """Map integration test fixture names to the test files that use them. + + Returns: + Dictionary mapping fixture name to frozenset of test file paths + (relative to repo root) + """ + integration_dir = Path(__file__).parent.parent / "tests" / "integration" + result: dict[str, set[str]] = {} + + for test_file in integration_dir.glob("test_*.py"): + content = test_file.read_text(encoding="utf-8") + rel_path = test_file.relative_to(Path(__file__).parent.parent).as_posix() + for func in _TEST_FUNC_RE.findall(content): + base_name = func.replace("test_", "").partition("[")[0] + result.setdefault(base_name, set()).add(rel_path) + + return {k: frozenset(v) for k, v in result.items()} + + +@cache +def _get_component_to_integration_test_files() -> dict[str, frozenset[str]]: + """Build index mapping each component to the test files that depend on it. + + Resolves full dependency trees once per fixture, then inverts the mapping + so lookups are O(1) per component. + + Returns: + Dictionary mapping component name to frozenset of test file paths + """ + fixture_components = get_components_per_integration_fixture() + fixture_to_test_files = get_fixture_to_test_files() + + result: dict[str, set[str]] = {} + for fixture_name, components in fixture_components.items(): + test_files = fixture_to_test_files.get(fixture_name) + if not test_files: + continue + # Get full dependency tree for this fixture's components + all_deps = get_all_dependencies(components) + for dep in all_deps: + result.setdefault(dep, set()).update(test_files) + + return {k: frozenset(v) for k, v in result.items()} + + +def get_integration_test_files_for_components( + changed_components: set[str], +) -> list[str]: + """Get integration test file paths that use any of the given components. + + Uses a precomputed component → test files index for O(C) lookup + where C is the number of changed components. + + Args: + changed_components: Set of component names that have changed + + Returns: + Sorted list of test file paths relative to repo root + (e.g., ["tests/integration/test_api.py", ...]) + """ + component_to_tests = _get_component_to_integration_test_files() + + return sorted( + { + test_file + for component in changed_components + for test_file in component_to_tests.get(component, ()) + } + ) def filter_component_and_test_files(file_path: str) -> bool: diff --git a/tests/integration/fixtures/online_image_bmp.yaml b/tests/integration/fixtures/online_image_bmp.yaml new file mode 100644 index 0000000000..e36514e9ae --- /dev/null +++ b/tests/integration/fixtures/online_image_bmp.yaml @@ -0,0 +1,27 @@ +esphome: + name: online-image-bmp + +host: + +http_request: + +display: + +online_image: + - url: http://127.0.0.1:HTTP_PORT/foo.bmp + id: myimg + format: BMP + type: RGB + on_download_finished: + logger.log: + format: "download finished. cache hit: %u" + args: [cached] + +api: + actions: + - action: fetch_image + then: + - component.update: myimg + +logger: + level: DEBUG diff --git a/tests/integration/fixtures/template_text_save.yaml b/tests/integration/fixtures/template_text_save.yaml new file mode 100644 index 0000000000..526561732d --- /dev/null +++ b/tests/integration/fixtures/template_text_save.yaml @@ -0,0 +1,23 @@ +esphome: + name: host-template-text-save-test + +host: + +api: + batch_delay: 0ms + +logger: + +preferences: + flash_write_interval: 0s + +text: + - platform: template + name: "Test Text Restore" + id: test_text_restore + optimistic: true + min_length: 0 + max_length: 10 + mode: text + initial_value: "hello" + restore_value: true diff --git a/tests/integration/test_online_image_bmp.py b/tests/integration/test_online_image_bmp.py new file mode 100644 index 0000000000..7c32154fdd --- /dev/null +++ b/tests/integration/test_online_image_bmp.py @@ -0,0 +1,119 @@ +from __future__ import annotations + +import asyncio +import re + +import pytest + +from .types import APIClientConnectedFactory, RunCompiledFunction + +# black 8x8 RGB BMP, generated with +# from PIL import Image +# from io import BytesIO +# b = BytesIO() +# img = Image.new("RGB", (8, 8)) +# img.save(b, format="BMP") +# b.getvalue() +BMP_IMAGE = b"BM\xf6\x00\x00\x00\x00\x00\x00\x006\x00\x00\x00(\x00\x00\x00\x08\x00\x00\x00\x08\x00\x00\x00\x01\x00\x18\x00\x00\x00\x00\x00\xc0\x00\x00\x00\xc4\x0e\x00\x00\xc4\x0e\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" +LEN_BMP_IMAGE = len(BMP_IMAGE) + + +def handle_http(http_request_future): + async def handler(reader, writer): + try: + async with asyncio.timeout(1.0): + data = await reader.readuntil(b"\r\n") + + # ensure our request matches the expectation + expected_request = b"GET /foo.bmp HTTP/1.1\r\n" + assert data[: len(expected_request)] == expected_request + + # consume rest of request + async with asyncio.timeout(1.0): + data = await reader.readuntil(b"\r\n\r\n") + + http_request_future.set_result(True) + + http_response = [ + b"HTTP/1.1 200 OK", + b"Content-Length: %d" % LEN_BMP_IMAGE, + b"Content-Type: text/plain", + b"Connection: close", + b"", + b"", + ] + writer.write(b"\r\n".join(http_response)) + await writer.drain() + + writer.write(BMP_IMAGE) + + await writer.drain() + except Exception as exc: + if not http_request_future.done(): + http_request_future.set_exception(exc) + raise + finally: + writer.close() + + return handler + + +@pytest.mark.asyncio +async def test_online_image_bmp( + yaml_config: str, + run_compiled: RunCompiledFunction, + api_client_connected: APIClientConnectedFactory, +) -> None: + """Esphome shouldn't block the main loop when a http response is slow""" + loop = asyncio.get_running_loop() + + # Track http request + http_request_future = loop.create_future() + download_finished_future = loop.create_future() + downloaded_bytes_future = loop.create_future() + + def check_output(line: str) -> None: + """Check log output for expected messages.""" + + if match := re.search(r"Image fully downloaded, (\d+) bytes", line): + downloaded_bytes_future.set_result(int(match.group(1))) + + if "download finished" in line: + download_finished_future.set_result(True) + + server = await asyncio.start_server( + handle_http(http_request_future), "127.0.0.1", 0 + ) + http_server_port = server.sockets[0].getsockname()[1] + + config = yaml_config.replace("HTTP_PORT", str(http_server_port)) + + # Run with log monitoring + async with ( + server, + run_compiled(config, line_callback=check_output), + api_client_connected() as client, + ): + # Verify device info + + device_info = await client.device_info() + assert device_info is not None + assert device_info.name == "online-image-bmp" + + # List services to find our test service + _, services = await client.list_entities_services() + + # Find test service + request_service = next((s for s in services if s.name == "fetch_image"), None) + + assert request_service is not None, "fetch_image service not found" + + await client.execute_service(request_service, {}) + + async with asyncio.timeout(0.1): + await http_request_future + + async with asyncio.timeout(0.5): + numbytes = await downloaded_bytes_future + assert numbytes == LEN_BMP_IMAGE + await download_finished_future diff --git a/tests/integration/test_template_text_save.py b/tests/integration/test_template_text_save.py new file mode 100644 index 0000000000..47c8e3188a --- /dev/null +++ b/tests/integration/test_template_text_save.py @@ -0,0 +1,131 @@ +"""Integration test for template text restore_value persistence. + +Tests that: +1. A template text with restore_value saves its value to preferences +2. The saved value persists across restarts (binary re-run) +3. Setting the same value again does not produce a spurious "too long" warning +""" + +from __future__ import annotations + +import asyncio +from pathlib import Path +import socket +from typing import Any + +from aioesphomeapi import TextInfo, TextState +import pytest + +from .conftest import run_binary_and_wait_for_port, wait_and_connect_api_client +from .state_utils import InitialStateHelper, require_entity +from .types import CompileFunction, ConfigWriter + + +@pytest.mark.asyncio +async def test_template_text_save( + yaml_config: str, + write_yaml_config: ConfigWriter, + compile_esphome: CompileFunction, + reserved_tcp_port: tuple[int, socket.socket], +) -> None: + """Test template text save/restore persistence and duplicate-save behavior.""" + port, port_socket = reserved_tcp_port + + # Clean up any stale preference file from previous runs + prefs_file = ( + Path.home() / ".esphome" / "prefs" / "host-template-text-save-test.prefs" + ) + if prefs_file.exists(): + prefs_file.unlink() + + # Write and compile once + config_path = await write_yaml_config(yaml_config) + binary_path = await compile_esphome(config_path) + + # Release the reserved port so the binary can bind to it + port_socket.close() + + # --- First run: set a value and verify no spurious warnings --- + warning_lines: list[str] = [] + + def capture_warnings(line: str) -> None: + if "too long to save" in line.lower(): + warning_lines.append(line) + + async with ( + run_binary_and_wait_for_port( + binary_path, "127.0.0.1", port, line_callback=capture_warnings + ), + wait_and_connect_api_client(port=port) as client, + ): + device_info = await client.device_info() + assert device_info.name == "host-template-text-save-test" + + entities, _ = await client.list_entities_services() + text_entity = require_entity( + entities, "test_text_restore", TextInfo, "Test Text Restore" + ) + + # Set up state tracking + loop = asyncio.get_running_loop() + state_futures: dict[int, asyncio.Future[Any]] = {} + + def on_state(state: Any) -> None: + if state.key in state_futures and not state_futures[state.key].done(): + state_futures[state.key].set_result(state) + + initial_state_helper = InitialStateHelper(entities) + client.subscribe_states(initial_state_helper.on_state_wrapper(on_state)) + await initial_state_helper.wait_for_initial_states() + + # Verify initial value from config + initial = initial_state_helper.initial_states[text_entity.key] + assert isinstance(initial, TextState) + assert initial.state == "hello" + + async def wait_for_state(key: int, timeout: float = 2.0) -> Any: + state_futures[key] = loop.create_future() + try: + return await asyncio.wait_for(state_futures[key], timeout) + finally: + state_futures.pop(key, None) + + # Set a new value that fits within max_length + client.text_command(key=text_entity.key, state="world") + state = await wait_for_state(text_entity.key) + assert state.state == "world" + + # Set the same value again - should NOT produce "too long" warning + client.text_command(key=text_entity.key, state="world") + # Give time for the warning to appear (if any) + await asyncio.sleep(0.5) + + # No warnings should have appeared + assert warning_lines == [], ( + f"Unexpected 'too long to save' warning(s): {warning_lines}" + ) + + # --- Second run: verify the value was restored from preferences --- + async with ( + run_binary_and_wait_for_port(binary_path, "127.0.0.1", port), + wait_and_connect_api_client(port=port) as client, + ): + entities, _ = await client.list_entities_services() + text_entity = require_entity( + entities, "test_text_restore", TextInfo, "Test Text Restore" + ) + + initial_state_helper = InitialStateHelper(entities) + client.subscribe_states(initial_state_helper.on_state_wrapper(lambda s: None)) + await initial_state_helper.wait_for_initial_states() + + # The value should be "world" - restored from preferences + restored = initial_state_helper.initial_states[text_entity.key] + assert isinstance(restored, TextState) + assert restored.state == "world", ( + f"Expected restored value 'world', got '{restored.state}'" + ) + + # Clean up preference file + if prefs_file.exists(): + prefs_file.unlink() diff --git a/tests/script/test_determine_jobs.py b/tests/script/test_determine_jobs.py index 61ef8985df..5c81ad374b 100644 --- a/tests/script/test_determine_jobs.py +++ b/tests/script/test_determine_jobs.py @@ -29,9 +29,9 @@ spec.loader.exec_module(determine_jobs) @pytest.fixture -def mock_should_run_integration_tests() -> Generator[Mock, None, None]: - """Mock should_run_integration_tests from helpers.""" - with patch.object(determine_jobs, "should_run_integration_tests") as mock: +def mock_determine_integration_tests() -> Generator[Mock, None, None]: + """Mock determine_integration_tests.""" + with patch.object(determine_jobs, "determine_integration_tests") as mock: yield mock @@ -87,7 +87,7 @@ def clear_determine_jobs_caches() -> None: def test_main_all_tests_should_run( - mock_should_run_integration_tests: Mock, + mock_determine_integration_tests: Mock, mock_should_run_clang_tidy: Mock, mock_should_run_clang_format: Mock, mock_should_run_python_linters: Mock, @@ -100,7 +100,7 @@ def test_main_all_tests_should_run( # Ensure we're not in GITHUB_ACTIONS mode for this test monkeypatch.delenv("GITHUB_ACTIONS", raising=False) - mock_should_run_integration_tests.return_value = True + mock_determine_integration_tests.return_value = (True, []) mock_should_run_clang_tidy.return_value = True mock_should_run_clang_format.return_value = True mock_should_run_python_linters.return_value = True @@ -152,6 +152,8 @@ def test_main_all_tests_should_run( output = json.loads(captured.out) assert output["integration_tests"] is True + assert output["integration_tests_run_all"] is True + assert output["integration_test_files"] == [] assert output["clang_tidy"] is True assert output["clang_tidy_mode"] in ["nosplit", "split"] assert output["clang_format"] is True @@ -183,7 +185,7 @@ def test_main_all_tests_should_run( def test_main_no_tests_should_run( - mock_should_run_integration_tests: Mock, + mock_determine_integration_tests: Mock, mock_should_run_clang_tidy: Mock, mock_should_run_clang_format: Mock, mock_should_run_python_linters: Mock, @@ -196,7 +198,7 @@ def test_main_no_tests_should_run( # Ensure we're not in GITHUB_ACTIONS mode for this test monkeypatch.delenv("GITHUB_ACTIONS", raising=False) - mock_should_run_integration_tests.return_value = False + mock_determine_integration_tests.return_value = (False, []) mock_should_run_clang_tidy.return_value = False mock_should_run_clang_format.return_value = False mock_should_run_python_linters.return_value = False @@ -233,6 +235,8 @@ def test_main_no_tests_should_run( output = json.loads(captured.out) assert output["integration_tests"] is False + assert output["integration_tests_run_all"] is False + assert output["integration_test_files"] == [] assert output["clang_tidy"] is False assert output["clang_tidy_mode"] == "disabled" assert output["clang_format"] is False @@ -253,7 +257,7 @@ def test_main_no_tests_should_run( def test_main_with_branch_argument( - mock_should_run_integration_tests: Mock, + mock_determine_integration_tests: Mock, mock_should_run_clang_tidy: Mock, mock_should_run_clang_format: Mock, mock_should_run_python_linters: Mock, @@ -266,7 +270,7 @@ def test_main_with_branch_argument( # Ensure we're not in GITHUB_ACTIONS mode for this test monkeypatch.delenv("GITHUB_ACTIONS", raising=False) - mock_should_run_integration_tests.return_value = False + mock_determine_integration_tests.return_value = (False, []) mock_should_run_clang_tidy.return_value = True mock_should_run_clang_format.return_value = False mock_should_run_python_linters.return_value = True @@ -302,7 +306,7 @@ def test_main_with_branch_argument( determine_jobs.main() # Check that functions were called with branch - mock_should_run_integration_tests.assert_called_once_with("main") + mock_determine_integration_tests.assert_called_once_with("main") mock_should_run_clang_tidy.assert_called_once_with("main") mock_should_run_clang_format.assert_called_once_with("main") mock_should_run_python_linters.assert_called_once_with("main") @@ -312,6 +316,8 @@ def test_main_with_branch_argument( output = json.loads(captured.out) assert output["integration_tests"] is False + assert output["integration_tests_run_all"] is False + assert output["integration_test_files"] == [] assert output["clang_tidy"] is True assert output["clang_tidy_mode"] in ["nosplit", "split"] assert output["clang_format"] is False @@ -334,30 +340,33 @@ def test_main_with_branch_argument( assert output["cpp_unit_tests_components"] == ["mqtt"] -def test_should_run_integration_tests( +def test_determine_integration_tests( monkeypatch: pytest.MonkeyPatch, ) -> None: - """Test should_run_integration_tests function.""" - # Core C++ files trigger tests + """Test determine_integration_tests function.""" + # Core C++ files trigger run_all with patch.object( determine_jobs, "changed_files", return_value=["esphome/core/component.cpp"] ): - result = determine_jobs.should_run_integration_tests() - assert result is True + run_all, test_files = determine_jobs.determine_integration_tests() + assert run_all is True + assert test_files == [] - # Core Python files trigger tests + # Core Python files trigger run_all with patch.object( determine_jobs, "changed_files", return_value=["esphome/core/config.py"] ): - result = determine_jobs.should_run_integration_tests() - assert result is True + run_all, test_files = determine_jobs.determine_integration_tests() + assert run_all is True + assert test_files == [] # Python files directly in esphome/ do NOT trigger tests with patch.object( determine_jobs, "changed_files", return_value=["esphome/config.py"] ): - result = determine_jobs.should_run_integration_tests() - assert result is False + run_all, test_files = determine_jobs.determine_integration_tests() + assert run_all is False + assert test_files == [] # Python files in subdirectories (not core) do NOT trigger tests with patch.object( @@ -365,35 +374,151 @@ def test_should_run_integration_tests( "changed_files", return_value=["esphome/dashboard/web_server.py"], ): - result = determine_jobs.should_run_integration_tests() - assert result is False + run_all, test_files = determine_jobs.determine_integration_tests() + assert run_all is False + assert test_files == [] -def test_should_run_integration_tests_with_branch() -> None: - """Test should_run_integration_tests with branch argument.""" +def test_determine_integration_tests_with_branch() -> None: + """Test determine_integration_tests with branch argument.""" with patch.object(determine_jobs, "changed_files") as mock_changed: mock_changed.return_value = [] - determine_jobs.should_run_integration_tests("release") + run_all, test_files = determine_jobs.determine_integration_tests("release") mock_changed.assert_called_once_with("release") + assert run_all is False + assert test_files == [] -def test_should_run_integration_tests_component_dependency() -> None: - """Test that integration tests run when components used in fixtures change.""" +def test_determine_integration_tests_component_dependency() -> None: + """Test that integration tests return specific test files when components used in fixtures change.""" with ( patch.object( determine_jobs, "changed_files", return_value=["esphome/components/api/api.cpp"], ), + patch.object(determine_jobs, "get_fixture_to_test_files") as mock_fixture_map, patch.object( - determine_jobs, "get_components_from_integration_fixtures" - ) as mock_fixtures, + determine_jobs, "get_integration_test_files_for_components" + ) as mock_test_files, ): - mock_fixtures.return_value = {"api", "sensor"} - with patch.object(determine_jobs, "get_all_dependencies") as mock_deps: - mock_deps.return_value = {"api", "sensor", "network"} - result = determine_jobs.should_run_integration_tests() - assert result is True + mock_fixture_map.return_value = {} + mock_test_files.return_value = [ + "tests/integration/test_api.py", + "tests/integration/test_sensor.py", + ] + run_all, test_files = determine_jobs.determine_integration_tests() + assert run_all is False + assert test_files == [ + "tests/integration/test_api.py", + "tests/integration/test_sensor.py", + ] + + +def test_determine_integration_tests_component_only_affected_tests() -> None: + """Test that only tests using the changed component are returned.""" + with ( + patch.object( + determine_jobs, + "changed_files", + return_value=["esphome/components/modbus/modbus.cpp"], + ), + patch.object(determine_jobs, "get_fixture_to_test_files", return_value={}), + patch.object( + determine_jobs, "get_integration_test_files_for_components" + ) as mock_test_files, + ): + mock_test_files.return_value = [ + "tests/integration/test_uart_mock_modbus.py", + ] + run_all, test_files = determine_jobs.determine_integration_tests() + assert run_all is False + assert test_files == ["tests/integration/test_uart_mock_modbus.py"] + # Verify it was called with the right component + mock_test_files.assert_called_once_with({"modbus"}) + + +def test_determine_integration_tests_infra_file_runs_all() -> None: + """Test that changing infrastructure files (conftest.py, etc.) runs all tests.""" + with patch.object( + determine_jobs, + "changed_files", + return_value=["tests/integration/conftest.py"], + ): + run_all, test_files = determine_jobs.determine_integration_tests() + assert run_all is True + assert test_files == [] + + +def test_determine_integration_tests_readme_does_not_run_all() -> None: + """Test that changing README.md does not trigger integration tests.""" + with patch.object( + determine_jobs, + "changed_files", + return_value=["tests/integration/README.md"], + ): + run_all, test_files = determine_jobs.determine_integration_tests() + assert run_all is False + assert test_files == [] + + +def test_determine_integration_tests_changed_test_file() -> None: + """Test that changing a specific test file only runs that test.""" + with ( + patch.object( + determine_jobs, + "changed_files", + return_value=["tests/integration/test_syslog.py"], + ), + patch.object(determine_jobs, "get_fixture_to_test_files", return_value={}), + patch.object( + determine_jobs, + "get_integration_test_files_for_components", + return_value=[], + ), + ): + run_all, test_files = determine_jobs.determine_integration_tests() + assert run_all is False + assert test_files == ["tests/integration/test_syslog.py"] + + +def test_determine_integration_tests_changed_fixture_yaml() -> None: + """Test that changing a fixture YAML runs the corresponding test file.""" + with ( + patch.object( + determine_jobs, + "changed_files", + return_value=["tests/integration/fixtures/uart_mock_modbus.yaml"], + ), + patch.object(determine_jobs, "get_fixture_to_test_files") as mock_fixture_map, + patch.object( + determine_jobs, + "get_integration_test_files_for_components", + return_value=[], + ), + ): + mock_fixture_map.return_value = { + "uart_mock_modbus": frozenset( + {"tests/integration/test_uart_mock_modbus.py"} + ), + } + run_all, test_files = determine_jobs.determine_integration_tests() + assert run_all is False + assert test_files == ["tests/integration/test_uart_mock_modbus.py"] + + +def test_determine_integration_tests_non_yaml_fixture_runs_all() -> None: + """Test that non-YAML changes under fixtures/ (e.g., external_components) run all tests.""" + with patch.object( + determine_jobs, + "changed_files", + return_value=[ + "tests/integration/fixtures/external_components/test_component/__init__.py" + ], + ): + run_all, test_files = determine_jobs.determine_integration_tests() + assert run_all is True + assert test_files == [] @pytest.mark.parametrize( @@ -538,7 +663,7 @@ def test_count_changed_cpp_files_with_branch() -> None: def test_main_filters_components_without_tests( - mock_should_run_integration_tests: Mock, + mock_determine_integration_tests: Mock, mock_should_run_clang_tidy: Mock, mock_should_run_clang_format: Mock, mock_should_run_python_linters: Mock, @@ -551,7 +676,7 @@ def test_main_filters_components_without_tests( # Ensure we're not in GITHUB_ACTIONS mode for this test monkeypatch.delenv("GITHUB_ACTIONS", raising=False) - mock_should_run_integration_tests.return_value = False + mock_determine_integration_tests.return_value = (False, []) mock_should_run_clang_tidy.return_value = False mock_should_run_clang_format.return_value = False mock_should_run_python_linters.return_value = False @@ -631,7 +756,7 @@ def test_main_filters_components_without_tests( def test_main_detects_components_with_variant_tests( - mock_should_run_integration_tests: Mock, + mock_determine_integration_tests: Mock, mock_should_run_clang_tidy: Mock, mock_should_run_clang_format: Mock, mock_should_run_python_linters: Mock, @@ -649,7 +774,7 @@ def test_main_detects_components_with_variant_tests( # Ensure we're not in GITHUB_ACTIONS mode for this test monkeypatch.delenv("GITHUB_ACTIONS", raising=False) - mock_should_run_integration_tests.return_value = False + mock_determine_integration_tests.return_value = (False, []) mock_should_run_clang_tidy.return_value = False mock_should_run_clang_format.return_value = False mock_should_run_python_linters.return_value = False @@ -999,7 +1124,7 @@ def test_detect_memory_impact_config_with_variant_tests(tmp_path: Path) -> None: def test_clang_tidy_mode_full_scan( - mock_should_run_integration_tests: Mock, + mock_determine_integration_tests: Mock, mock_should_run_clang_tidy: Mock, mock_should_run_clang_format: Mock, mock_should_run_python_linters: Mock, @@ -1010,7 +1135,7 @@ def test_clang_tidy_mode_full_scan( """Test that full scan (hash changed) always uses split mode.""" monkeypatch.delenv("GITHUB_ACTIONS", raising=False) - mock_should_run_integration_tests.return_value = False + mock_determine_integration_tests.return_value = (False, []) mock_should_run_clang_tidy.return_value = True mock_should_run_clang_format.return_value = False mock_should_run_python_linters.return_value = False @@ -1065,7 +1190,7 @@ def test_clang_tidy_mode_targeted_scan( component_count: int, files_per_component: int, expected_mode: str, - mock_should_run_integration_tests: Mock, + mock_determine_integration_tests: Mock, mock_should_run_clang_tidy: Mock, mock_should_run_clang_format: Mock, mock_should_run_python_linters: Mock, @@ -1076,7 +1201,7 @@ def test_clang_tidy_mode_targeted_scan( """Test clang-tidy mode selection based on files_to_check count.""" monkeypatch.delenv("GITHUB_ACTIONS", raising=False) - mock_should_run_integration_tests.return_value = False + mock_determine_integration_tests.return_value = (False, []) mock_should_run_clang_tidy.return_value = True mock_should_run_clang_format.return_value = False mock_should_run_python_linters.return_value = False @@ -1123,7 +1248,7 @@ def test_clang_tidy_mode_targeted_scan( def test_main_core_files_changed_still_detects_components( - mock_should_run_integration_tests: Mock, + mock_determine_integration_tests: Mock, mock_should_run_clang_tidy: Mock, mock_should_run_clang_format: Mock, mock_should_run_python_linters: Mock, @@ -1135,7 +1260,7 @@ def test_main_core_files_changed_still_detects_components( """Test that component changes are detected even when core files change.""" monkeypatch.delenv("GITHUB_ACTIONS", raising=False) - mock_should_run_integration_tests.return_value = True + mock_determine_integration_tests.return_value = (True, []) mock_should_run_clang_tidy.return_value = True mock_should_run_clang_format.return_value = True mock_should_run_python_linters.return_value = True @@ -1604,7 +1729,7 @@ def test_detect_platform_hint_from_filename_case_insensitive( def test_component_batching_beta_branch_40_per_batch( tmp_path: Path, - mock_should_run_integration_tests: Mock, + mock_determine_integration_tests: Mock, mock_should_run_clang_tidy: Mock, mock_should_run_clang_format: Mock, mock_should_run_python_linters: Mock, @@ -1628,7 +1753,7 @@ def test_component_batching_beta_branch_40_per_batch( (comp_dir / "test.esp32-idf.yaml").write_text(f"# Test for {comp}") # Setup mocks - mock_should_run_integration_tests.return_value = False + mock_determine_integration_tests.return_value = (False, []) mock_should_run_clang_tidy.return_value = False mock_should_run_clang_format.return_value = False mock_should_run_python_linters.return_value = False diff --git a/tests/script/test_helpers.py b/tests/script/test_helpers.py index 781054eb3b..e3802d2d51 100644 --- a/tests/script/test_helpers.py +++ b/tests/script/test_helpers.py @@ -36,6 +36,7 @@ def clear_helpers_cache() -> None: """Clear cached functions before each test.""" helpers._get_github_event_data.cache_clear() helpers._get_changed_files_github_actions.cache_clear() + helpers.get_components_per_integration_fixture.cache_clear() @pytest.mark.parametrize( @@ -1111,7 +1112,7 @@ def test_get_components_from_integration_fixtures() -> None: "gpio", } - mock_yaml_file = Mock() + mock_yaml_file = Mock(stem="test_fixture") with ( patch("pathlib.Path.glob") as mock_glob, @@ -1133,7 +1134,7 @@ def test_get_components_from_integration_fixtures_skips_yaml_anchors() -> None: ".binary_filters": {"filters": [{"settle": "50ms"}]}, } - mock_yaml_file = Mock() + mock_yaml_file = Mock(stem="test_fixture") with ( patch("pathlib.Path.glob") as mock_glob, @@ -1148,6 +1149,31 @@ def test_get_components_from_integration_fixtures_skips_yaml_anchors() -> None: assert components == {"sensor", "esphome", "template"} +def test_get_integration_test_files_for_components_real_fixtures() -> None: + """Test that component changes map to the correct real integration test files. + + This test uses real fixtures to verify the mapping stays correct + as new tests are added. + """ + # modbus should include at least the modbus test + modbus_tests = helpers.get_integration_test_files_for_components({"modbus"}) + assert "tests/integration/test_uart_mock_modbus.py" in modbus_tests + + # ld2410 should include at least the ld2410 test + ld2410_tests = helpers.get_integration_test_files_for_components({"ld2410"}) + assert "tests/integration/test_uart_mock_ld2410.py" in ld2410_tests + + # syslog should include at least the syslog test + syslog_tests = helpers.get_integration_test_files_for_components({"syslog"}) + assert "tests/integration/test_syslog.py" in syslog_tests + + # A component not used by any fixture should return nothing + fake_tests = helpers.get_integration_test_files_for_components( + {"nonexistent_component_xyz"} + ) + assert fake_tests == [] + + @pytest.mark.parametrize( "output,expected", [