diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 528e69c478..e87939f824 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -5,6 +5,7 @@ updates: directory: "/" schedule: interval: daily + open-pull-requests-limit: 10 ignore: # Hypotehsis is only used for testing and is updated quite often - dependency-name: hypothesis diff --git a/CODEOWNERS b/CODEOWNERS index f8cdfdc6c6..3c3e502058 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -417,6 +417,7 @@ esphome/components/restart/* @esphome/core esphome/components/rf_bridge/* @jesserockz esphome/components/rgbct/* @jesserockz esphome/components/ring_buffer/* @kahrendt +esphome/components/router/speaker/* @kahrendt esphome/components/rp2040/* @jesserockz esphome/components/rp2040_ble/* @bdraco esphome/components/rp2040_pio_led_strip/* @Papa-DMan diff --git a/esphome/__main__.py b/esphome/__main__.py index 17eeca2900..c29162fd23 100644 --- a/esphome/__main__.py +++ b/esphome/__main__.py @@ -608,7 +608,7 @@ def run_miniterm(config: ConfigType, port: str, args) -> int: try: module = importlib.import_module("esphome.components." + CORE.target_platform) - process_stacktrace = getattr(module, "process_stacktrace") + process_stacktrace = module.process_stacktrace except (AttributeError, ImportError): _LOGGER.info( 'Stacktrace analysis is unavailable: no compatible analyzer found for target platform "%s".', @@ -639,7 +639,7 @@ def run_miniterm(config: ConfigType, port: str, args) -> int: chunk = ser.read(ser.in_waiting or 1) if not chunk: continue - time_ = datetime.now() + time_ = datetime.now().astimezone() milliseconds = time_.microsecond // 1000 time_str = f"[{time_.hour:02}:{time_.minute:02}:{time_.second:02}.{milliseconds:03}]" @@ -795,7 +795,7 @@ def _check_and_emit_build_info() -> None: # Read build_info from JSON try: - with open(build_info_json_path, encoding="utf-8") as f: + with build_info_json_path.open(encoding="utf-8") as f: build_info = json.load(f) except (OSError, json.JSONDecodeError) as e: _LOGGER.debug("Failed to read build_info: %s", e) @@ -1057,7 +1057,7 @@ def _wait_for_serial_port( def _port_found() -> bool: if port is not None: if os.name == "posix": - return os.path.exists(port) + return Path(port).exists() return any(p.path == port for p in get_serial_ports()) ports = get_serial_ports() if known_ports is not None: @@ -1102,7 +1102,7 @@ def upload_program( host = devices[0] try: module = importlib.import_module("esphome.components." + CORE.target_platform) - if getattr(module, "upload_program")(config, args, host): + if module.upload_program(config, args, host): return 0, host except AttributeError: pass @@ -1354,7 +1354,7 @@ def _validate_bootloader_binary(binary: Path) -> None: def show_logs(config: ConfigType, args: ArgsProtocol, devices: list[str]) -> int | None: try: module = importlib.import_module("esphome.components." + CORE.target_platform) - if getattr(module, "show_logs")(config, args, devices): + if module.show_logs(config, args, devices): return 0 except AttributeError: pass @@ -1413,17 +1413,47 @@ def command_config(args: ArgsProtocol, config: ConfigType) -> int | None: if not CORE.verbose: config = strip_default_ids(config) output = yaml_util.dump(config, args.show_secrets) - # add the console decoration so the front-end can hide the secrets if not args.show_secrets: - output = re.sub( - r"(password|key|psk|ssid)\: (.+)", r"\1: \\033[8m\2\\033[28m", output - ) + output = _redact_with_legacy_fallback(output) if not CORE.quiet: safe_print(output) _LOGGER.info("Configuration is valid!") return 0 +# Legacy substring redaction fallback for unmigrated schemas; removed in +# 2026.12.0 once canonical sensitive fields are tagged. The lookahead skips +# values that already render themselves: ``\033[8m`` (SensitiveStr wrap), +# ``!secret`` (preserves the user-friendly tag), ``!lambda`` (multi-line +# block; first line is structural). The fragment must either start the +# field name or follow ``_`` so the warning names a real field; this avoids +# false positives like ``monkey:`` matching the ``key`` fragment. +_LEGACY_REDACTION_RE = re.compile( + r"(?P\b(?:\w+_)?(?:password|key|psk|ssid))\: " + r"(?!\\033\[8m|!secret\b|!lambda\b)(?P.+)" +) +_LEGACY_REDACTION_REMOVAL = "2026.12.0" + + +def _redact_with_legacy_fallback(output: str) -> str: + unmarked: set[str] = set() + + def _replace(m: re.Match[str]) -> str: + unmarked.add(m.group("key")) + return f"{m.group('key')}: \\033[8m{m.group('val')}\\033[28m" + + output = _LEGACY_REDACTION_RE.sub(_replace, output) + for key in sorted(unmarked): + _LOGGER.warning( + "Field '%s' is being redacted by a legacy substring heuristic. " + "Mark this field's schema validator with cv.sensitive(...) for " + "deterministic redaction; the heuristic will be removed in %s.", + key, + _LEGACY_REDACTION_REMOVAL, + ) + return output + + def command_config_hash(args: ArgsProtocol, config: ConfigType) -> int | None: # generating code might modify config, so it must be done in order to generate # a hash that will match what was generated when compiling and then running diff --git a/esphome/analyze_memory/cli.py b/esphome/analyze_memory/cli.py index 13f98c64a9..e063bf1b6d 100644 --- a/esphome/analyze_memory/cli.py +++ b/esphome/analyze_memory/cli.py @@ -7,6 +7,7 @@ from collections.abc import Callable import heapq import json from operator import itemgetter +from pathlib import Path import sys from typing import TYPE_CHECKING @@ -510,7 +511,7 @@ class MemoryAnalyzerCLI(MemoryAnalyzer): lines.append( f"{_COMPONENT_CORE} Symbols > {self.SYMBOL_SIZE_THRESHOLD} B ({len(large_core_symbols)} symbols):" ) - for i, (symbol, demangled, size) in enumerate(large_core_symbols): + for i, (_symbol, demangled, size) in enumerate(large_core_symbols): # Core symbols only track (symbol, demangled, size) without section info, # so we don't show section labels here lines.append( @@ -602,7 +603,7 @@ class MemoryAnalyzerCLI(MemoryAnalyzer): lines.append( f"{comp_name} Symbols > {self.SYMBOL_SIZE_THRESHOLD} B & storage ({len(large_symbols)} symbols):" ) - for i, (symbol, demangled, size, section) in enumerate(large_symbols): + for i, (_symbol, demangled, size, section) in enumerate(large_symbols): lines.append( f"{i + 1}. {self._format_symbol_with_section(demangled, size, section)}" ) @@ -641,7 +642,7 @@ class MemoryAnalyzerCLI(MemoryAnalyzer): lines.append( f" Symbols > {self.RAM_SYMBOL_SIZE_THRESHOLD} B ({len(large_ram_syms)}):" ) - for symbol, demangled, size, section in large_ram_syms[:10]: + for _symbol, demangled, size, section in large_ram_syms[:10]: # Format section label consistently by stripping leading dot section_label = section.lstrip(".") if section else "" display_name = _format_pstorage_name(demangled) @@ -700,7 +701,7 @@ class MemoryAnalyzerCLI(MemoryAnalyzer): content = "\n".join(lines) if output_file: - with open(output_file, "w", encoding="utf-8") as f: + with Path(output_file).open("w", encoding="utf-8") as f: f.write(content) else: print(content) @@ -737,7 +738,7 @@ def main(): build_dir = sys.argv[1] # Load build directory - from pathlib import Path + import json from esphome.platformio.toolchain import IDEData @@ -785,7 +786,7 @@ def main(): if not idedata_path.exists(): continue try: - with open(idedata_path, encoding="utf-8") as f: + with idedata_path.open(encoding="utf-8") as f: raw_data = json.load(f) idedata = IDEData(raw_data) print(f"Loaded idedata from: {idedata_path}", file=sys.stderr) diff --git a/esphome/analyze_memory/demangle.py b/esphome/analyze_memory/demangle.py index 8999108b51..7dbd6d4f63 100644 --- a/esphome/analyze_memory/demangle.py +++ b/esphome/analyze_memory/demangle.py @@ -154,7 +154,7 @@ def batch_demangle( failed_count = 0 for original, stripped, prefix, demangled in zip( - symbols, symbols_stripped, symbols_prefixes, demangled_lines + symbols, symbols_stripped, symbols_prefixes, demangled_lines, strict=True ): # Add back any prefix that was removed demangled = _restore_symbol_prefix(prefix, stripped, demangled) diff --git a/esphome/analyze_memory/toolchain.py b/esphome/analyze_memory/toolchain.py index 3a8a5f7be4..a724d52f25 100644 --- a/esphome/analyze_memory/toolchain.py +++ b/esphome/analyze_memory/toolchain.py @@ -3,7 +3,6 @@ from __future__ import annotations import logging -import os from pathlib import Path import subprocess from typing import TYPE_CHECKING @@ -37,7 +36,7 @@ def _find_in_platformio_packages(tool_name: str) -> str | None: Full path to the tool or None if not found """ # Get PlatformIO packages directory - platformio_home = Path(os.path.expanduser("~/.platformio/packages")) + platformio_home = Path("~/.platformio/packages").expanduser() if not platformio_home.exists(): return None diff --git a/esphome/build_gen/espidf.py b/esphome/build_gen/espidf.py index 96f84ebbd1..0b50f72382 100644 --- a/esphome/build_gen/espidf.py +++ b/esphome/build_gen/espidf.py @@ -24,7 +24,7 @@ def get_available_components() -> list[str] | None: return None try: - with open(project_desc, encoding="utf-8") as f: + with project_desc.open(encoding="utf-8") as f: data = json.load(f) component_info = data.get("build_component_info", {}) diff --git a/esphome/bundle.py b/esphome/bundle.py index 4537cbce9d..d38f68ebfd 100644 --- a/esphome/bundle.py +++ b/esphome/bundle.py @@ -412,7 +412,7 @@ class ConfigBundleCreator: @staticmethod def _add_to_tar(tar: tarfile.TarFile, bf: BundleFile) -> None: """Add a BundleFile to the tar archive with deterministic metadata.""" - with open(bf.source, "rb") as f: + with bf.source.open("rb") as f: _add_bytes_to_tar(tar, bf.path, f.read()) diff --git a/esphome/components/api/__init__.py b/esphome/components/api/__init__.py index ca74483a2b..932702d47a 100644 --- a/esphome/components/api/__init__.py +++ b/esphome/components/api/__init__.py @@ -234,7 +234,7 @@ ACTIONS_SCHEMA = automation.validate_automation( ENCRYPTION_SCHEMA = cv.Schema( { - cv.Optional(CONF_KEY): validate_encryption_key, + cv.Optional(CONF_KEY): cv.sensitive(validate_encryption_key), } ) diff --git a/esphome/components/api/api_connection.cpp b/esphome/components/api/api_connection.cpp index 8e32441163..798deb5197 100644 --- a/esphome/components/api/api_connection.cpp +++ b/esphome/components/api/api_connection.cpp @@ -1169,13 +1169,11 @@ void APIConnection::on_camera_image_request(const CameraImageRequest &msg) { void APIConnection::on_get_time_response(const GetTimeResponse &value) { if (homeassistant::global_homeassistant_time != nullptr) { homeassistant::global_homeassistant_time->set_epoch_time(value.epoch_seconds); -#ifdef USE_TIME_TIMEZONE - // Only apply if the sender provided pre-parsed timezone data. - // Old clients (before 2026.3.0) only send the timezone string without the parsed struct, - // so all parsed_timezone fields default to zero — skip to keep the codegen-configured timezone. - // For actual UTC (all zeros), this also skips, which is harmless since UTC is the default. - // Eventually the timezone string will be removed and only the struct will be sent. - { +#if defined(USE_HOMEASSISTANT_TIMEZONE) && defined(USE_TIME_TIMEZONE) + if (!value.timezone.empty()) { + // Check if the sender provided pre-parsed timezone data. + // If std_offset is non-zero or DST rules are present, the parsed data was populated. + // For UTC (all zeros), string parsing produces the same result, so the fallback is equivalent. const auto &pt = value.parsed_timezone; if (pt.std_offset_seconds != 0 || pt.dst_start.type != enums::DST_RULE_TYPE_NONE) { time::ParsedTimezone tz{}; diff --git a/esphome/components/api/api_server.cpp b/esphome/components/api/api_server.cpp index c30bd2e612..031fa342c1 100644 --- a/esphome/components/api/api_server.cpp +++ b/esphome/components/api/api_server.cpp @@ -1,6 +1,7 @@ #include "api_server.h" #ifdef USE_API #include +#include #include "api_connection.h" #include "esphome/components/network/util.h" #include "esphome/core/application.h" @@ -677,7 +678,7 @@ uint32_t APIServer::register_active_action_call(uint32_t client_call_id, APIConn // Schedule automatic cleanup after timeout (client will have given up by then) // Uses numeric ID overload to avoid heap allocation from str_sprintf this->set_timeout(action_call_id, USE_API_ACTION_CALL_TIMEOUT_MS, [this, action_call_id]() { - ESP_LOGD(TAG, "Action call %u timed out", action_call_id); + ESP_LOGD(TAG, "Action call %" PRIu32 " timed out", action_call_id); this->unregister_active_action_call(action_call_id); }); @@ -721,7 +722,7 @@ void APIServer::send_action_response(uint32_t action_call_id, bool success, Stri return; } } - ESP_LOGW(TAG, "Cannot send response: no active call found for action_call_id %u", action_call_id); + ESP_LOGW(TAG, "Cannot send response: no active call found for action_call_id %" PRIu32, action_call_id); } #ifdef USE_API_USER_DEFINED_ACTION_RESPONSES_JSON void APIServer::send_action_response(uint32_t action_call_id, bool success, StringRef error_message, @@ -733,7 +734,7 @@ void APIServer::send_action_response(uint32_t action_call_id, bool success, Stri return; } } - ESP_LOGW(TAG, "Cannot send response: no active call found for action_call_id %u", action_call_id); + ESP_LOGW(TAG, "Cannot send response: no active call found for action_call_id %" PRIu32, action_call_id); } #endif // USE_API_USER_DEFINED_ACTION_RESPONSES_JSON #endif // USE_API_USER_DEFINED_ACTION_RESPONSES diff --git a/esphome/components/api/client.py b/esphome/components/api/client.py index d6150fbd29..44edc035f9 100644 --- a/esphome/components/api/client.py +++ b/esphome/components/api/client.py @@ -101,13 +101,14 @@ async def async_run_logs( client_info=f"ESPHome Logs {__version__}", noise_psk=noise_psk, addresses=addresses, # Pass all addresses for automatic retry + provide_time=False, ) # Try platform-specific stacktrace handler first, fall back to generic platform_process_stacktrace = None try: module = importlib.import_module("esphome.components." + CORE.target_platform) - platform_process_stacktrace = getattr(module, "process_stacktrace") + platform_process_stacktrace = module.process_stacktrace except (AttributeError, ImportError): _LOGGER.info( 'Stacktrace analysis is unavailable: no compatible analyzer found for target platform "%s".', @@ -118,7 +119,7 @@ async def async_run_logs( def on_log(msg: SubscribeLogsResponse) -> None: """Handle a new log message.""" - time_ = datetime.now() + time_ = datetime.now().astimezone() message: bytes = msg.message text = message.decode("utf8", "backslashreplace") nanoseconds = time_.microsecond // 1000 diff --git a/esphome/components/as5600/__init__.py b/esphome/components/as5600/__init__.py index 444306cec3..c05e556376 100644 --- a/esphome/components/as5600/__init__.py +++ b/esphome/components/as5600/__init__.py @@ -100,7 +100,7 @@ def position(min=-MAX_POSITION, max=MAX_POSITION): if isinstance(value, str) and value.endswith("%"): value = percent_to_position(value) - if isinstance(value, str) and (value.endswith("°") or value.endswith("deg")): + if isinstance(value, str) and value.endswith(("°", "deg")): return angle_to_position( value, min=round(min * POSITION_TO_ANGLE), diff --git a/esphome/components/audio/audio_decoder.cpp b/esphome/components/audio/audio_decoder.cpp index d4ff59fc36..f709c23fb6 100644 --- a/esphome/components/audio/audio_decoder.cpp +++ b/esphome/components/audio/audio_decoder.cpp @@ -9,9 +9,12 @@ namespace esphome::audio { static const char *const TAG = "audio.decoder"; -static const uint32_t DECODING_TIMEOUT_MS = 50; // The decode function will yield after this duration static const uint32_t READ_WRITE_TIMEOUT_MS = 20; // Timeout for transferring audio data +// Max consecutive decode iterations that consume input but produce no output; e.g., skipping a large metadata block, +// before yielding and returning. +static const uint8_t MAX_NO_OUTPUT_ITERATIONS = 32; + static const uint32_t MAX_POTENTIALLY_FAILED_COUNT = 10; AudioDecoder::AudioDecoder(size_t input_buffer_size, size_t output_buffer_size) @@ -20,11 +23,13 @@ AudioDecoder::AudioDecoder(size_t input_buffer_size, size_t output_buffer_size) } esp_err_t AudioDecoder::add_source(std::weak_ptr &input_ring_buffer) { - auto source = AudioSourceTransferBuffer::create(this->input_buffer_size_); + // Zero-copy source reading directly from the ring buffer's internal storage. Raw file data is byte + // aligned, so no frame alignment is required. + auto source = RingBufferAudioSource::create(input_ring_buffer.lock(), this->input_buffer_size_); if (source == nullptr) { - return ESP_ERR_NO_MEM; + // create() only returns nullptr for invalid arguments (expired ring buffer or zero buffer size) + return ESP_ERR_INVALID_ARG; } - source->set_source(input_ring_buffer); this->input_buffer_ = std::move(source); return ESP_OK; } @@ -141,13 +146,7 @@ AudioDecoderState AudioDecoder::decode(bool stop_gracefully) { } FileDecoderState state = FileDecoderState::MORE_TO_PROCESS; - - uint32_t decoding_start = millis(); - - bool first_loop_iteration = true; - - size_t bytes_processed = 0; - size_t bytes_available_before_processing = 0; + uint8_t no_output_iterations = 0; while (state == FileDecoderState::MORE_TO_PROCESS) { // Transfer decoded out @@ -161,45 +160,39 @@ AudioDecoderState AudioDecoder::decode(bool stop_gracefully) { this->playback_ms_ += this->audio_stream_info_.value().frames_to_milliseconds_with_remainder(&this->accumulated_frames_written_); } + + if ((bytes_written > 0) && (this->output_transfer_buffer_->available() == 0)) { + // All decoded audio has been flushed to the sink; return so the caller can react to stop/pause before + // decoding the next batch + return AudioDecoderState::DECODING; + } } else { // If paused, block to avoid wasting CPU resources delay(READ_WRITE_TIMEOUT_MS); } - // Verify there is enough space to store more decoded audio and that the function hasn't been running too long - if ((this->output_transfer_buffer_->free() < this->free_buffer_required_) || - (millis() - decoding_start > DECODING_TIMEOUT_MS)) { + if (this->output_transfer_buffer_->available() > 0) { + // Output transfer buffer indicates backpressure, return so caller can handle other events; + // e.g., stop/pause, before trying again return AudioDecoderState::DECODING; } - // Decode more audio - - // Never shift the input buffer; every decoder buffers internally and consumes only what it processed. - size_t bytes_read = this->input_buffer_->fill(pdMS_TO_TICKS(READ_WRITE_TIMEOUT_MS), false); - - if (!first_loop_iteration && (this->input_buffer_->available() < bytes_processed)) { - // Less data is available than what was processed in last iteration, so don't attempt to decode. - // This attempts to avoid the decoder from consistently trying to decode an incomplete frame. The transfer buffer - // will shift the remaining data to the start and copy more from the source the next time the decode function is - // called - break; + // Reaching here means no decoded output is pending (any would have returned above). Bounds long no-output + // stretches; e.g., skipping a large metadata block, so a source that keeps the ring buffer full can't spin this + // loop without yielding and trip the watchdog. The delay yields allowing other tasks to feed the watchdog and + // the return keeps stop/pause responsive. + if (++no_output_iterations >= MAX_NO_OUTPUT_ITERATIONS) { + delay(1); + return AudioDecoderState::DECODING; } - bytes_available_before_processing = this->input_buffer_->available(); + // Expose the next chunk of file data. Every decoder buffers internally and consumes only what it + // processed, so the source does not need to accumulate or stitch chunks across fill() calls. + this->input_buffer_->fill(pdMS_TO_TICKS(READ_WRITE_TIMEOUT_MS), false); - if ((this->potentially_failed_count_ > 0) && (bytes_read == 0)) { - // Failed to decode in last attempt and there is no new data + const size_t available_before_decode = this->input_buffer_->available(); - if ((this->input_buffer_->free() == 0) && first_loop_iteration) { - // The input buffer is full (or read-only, e.g. const flash source). Since it previously failed on the exact - // same data, we can never recover. For const sources this is correct: the entire file is already available, so - // a decode failure is genuine, not a transient out-of-data condition. - state = FileDecoderState::FAILED; - } else { - // Attempt to get more data next time - state = FileDecoderState::IDLE; - } - } else if (this->input_buffer_->available() == 0) { + if (available_before_decode == 0) { // No data to decode, attempt to get more data next time state = FileDecoderState::IDLE; } else { @@ -231,9 +224,6 @@ AudioDecoderState AudioDecoder::decode(bool stop_gracefully) { } } - first_loop_iteration = false; - bytes_processed = bytes_available_before_processing - this->input_buffer_->available(); - if (state == FileDecoderState::POTENTIALLY_FAILED) { ++this->potentially_failed_count_; } else if (state == FileDecoderState::END_OF_FILE) { @@ -241,7 +231,16 @@ AudioDecoderState AudioDecoder::decode(bool stop_gracefully) { } else if (state == FileDecoderState::FAILED) { return AudioDecoderState::FAILED; } else if (state == FileDecoderState::MORE_TO_PROCESS) { - this->potentially_failed_count_ = 0; + // Reset the failsafe only when the iteration made forward progress: input was consumed or output was + // produced (output_transfer_buffer_ is drained empty above, so any available bytes are new). A + // MORE_TO_PROCESS that neither consumes input nor produces output means the decoder is stalled; count it + // toward the failsafe so a stuck stream eventually surfaces as FAILED instead of looping forever. + if ((this->input_buffer_->available() < available_before_decode) || + (this->output_transfer_buffer_->available() > 0)) { + this->potentially_failed_count_ = 0; + } else { + ++this->potentially_failed_count_; + } } } return AudioDecoderState::DECODING; diff --git a/esphome/components/audio/audio_decoder.h b/esphome/components/audio/audio_decoder.h index c34ebbc613..e772b7eb5f 100644 --- a/esphome/components/audio/audio_decoder.h +++ b/esphome/components/audio/audio_decoder.h @@ -61,15 +61,16 @@ class AudioDecoder { */ public: /// @brief Allocates the output transfer buffer and stores the input buffer size for later use by add_source() - /// @param input_buffer_size Size of the input transfer buffer in bytes. + /// @param input_buffer_size Soft cap on the bytes a ring buffer source exposes per fill, in bytes. /// @param output_buffer_size Size of the output transfer buffer in bytes. AudioDecoder(size_t input_buffer_size, size_t output_buffer_size); ~AudioDecoder() = default; - /// @brief Adds a source ring buffer for raw file data. Takes ownership of the ring buffer in a shared_ptr. - /// @param input_ring_buffer weak_ptr of a shared_ptr of the sink ring buffer to transfer ownership - /// @return ESP_OK if successsful, ESP_ERR_NO_MEM if the transfer buffer wasn't allocated + /// @brief Adds a source ring buffer for raw file data. Shares ownership of the ring buffer via a shared_ptr. + /// The decoder reads directly from the ring buffer's internal storage with a zero-copy RingBufferAudioSource. + /// @param input_ring_buffer weak_ptr of the source ring buffer to read from + /// @return ESP_OK if successful, ESP_ERR_INVALID_ARG if the ring buffer is expired or the buffer size is zero esp_err_t add_source(std::weak_ptr &input_ring_buffer); /// @brief Adds a sink ring buffer for decoded audio. Takes ownership of the ring buffer in a shared_ptr. diff --git a/esphome/components/audio/audio_resampler.cpp b/esphome/components/audio/audio_resampler.cpp index c04cc881f5..bef62ce190 100644 --- a/esphome/components/audio/audio_resampler.cpp +++ b/esphome/components/audio/audio_resampler.cpp @@ -12,16 +12,17 @@ static const uint32_t READ_WRITE_TIMEOUT_MS = 20; AudioResampler::AudioResampler(size_t input_buffer_size, size_t output_buffer_size) : input_buffer_size_(input_buffer_size), output_buffer_size_(output_buffer_size) { - this->input_transfer_buffer_ = AudioSourceTransferBuffer::create(input_buffer_size); this->output_transfer_buffer_ = AudioSinkTransferBuffer::create(output_buffer_size); } esp_err_t AudioResampler::add_source(std::weak_ptr &input_ring_buffer) { - if (this->input_transfer_buffer_ != nullptr) { - this->input_transfer_buffer_->set_source(input_ring_buffer); - return ESP_OK; + // The zero-copy RingBufferAudioSource is created lazily on the first resample() call, once both the ring + // buffer (stored here) and the input stream info (set by start()) are available, in either order. + this->source_ring_buffer_ = input_ring_buffer.lock(); + if (this->source_ring_buffer_ == nullptr) { + return ESP_ERR_INVALID_STATE; } - return ESP_ERR_NO_MEM; + return ESP_OK; } esp_err_t AudioResampler::add_sink(std::weak_ptr &output_ring_buffer) { @@ -47,7 +48,7 @@ esp_err_t AudioResampler::start(AudioStreamInfo &input_stream_info, AudioStreamI this->input_stream_info_ = input_stream_info; this->output_stream_info_ = output_stream_info; - if ((this->input_transfer_buffer_ == nullptr) || (this->output_transfer_buffer_ == nullptr)) { + if (this->output_transfer_buffer_ == nullptr) { return ESP_ERR_NO_MEM; } @@ -56,6 +57,13 @@ esp_err_t AudioResampler::start(AudioStreamInfo &input_stream_info, AudioStreamI return ESP_ERR_NOT_SUPPORTED; } + // Reject frame sizes that can't be used as the zero-copy source's alignment up front, where the caller checks + // the return code. The lazy create() in resample() keeps its own guard since it runs before the uint8_t cast. + const size_t bytes_per_frame = this->input_stream_info_.frames_to_bytes(1); + if ((bytes_per_frame == 0) || (bytes_per_frame > RingBufferAudioSource::MAX_ALIGNMENT_BYTES)) { + return ESP_ERR_NOT_SUPPORTED; + } + if ((input_stream_info.get_sample_rate() != output_stream_info.get_sample_rate()) || (input_stream_info.get_bits_per_sample() != output_stream_info.get_bits_per_sample())) { this->resampler_ = make_unique( @@ -87,8 +95,27 @@ esp_err_t AudioResampler::start(AudioStreamInfo &input_stream_info, AudioStreamI } AudioResamplerState AudioResampler::resample(bool stop_gracefully, int32_t *ms_differential) { + if (this->audio_source_ == nullptr) { + // Lazily create the zero-copy source on first use. Frame-aligned reads ensure multi-channel frames are + // never split across the ring buffer's wrap boundary. + const size_t bytes_per_frame = this->input_stream_info_.frames_to_bytes(1); + if ((bytes_per_frame == 0) || (bytes_per_frame > RingBufferAudioSource::MAX_ALIGNMENT_BYTES)) { + // Stream info is unset or the frame is too large to use as an alignment; the uint8_t cast below would + // truncate it and could yield a source that tears frames. + return AudioResamplerState::FAILED; + } + // Pass the shared_ptr by copy so a failed create() leaves source_ring_buffer_ intact; release our + // reference only after the source has taken ownership. + this->audio_source_ = RingBufferAudioSource::create(this->source_ring_buffer_, this->input_buffer_size_, + static_cast(bytes_per_frame)); + if (this->audio_source_ == nullptr) { + return AudioResamplerState::FAILED; + } + this->source_ring_buffer_.reset(); + } + if (stop_gracefully) { - if (!this->input_transfer_buffer_->has_buffered_data() && (this->output_transfer_buffer_->available() == 0)) { + if (!this->audio_source_->has_buffered_data() && (this->output_transfer_buffer_->available() == 0)) { return AudioResamplerState::FINISHED; } } @@ -102,9 +129,11 @@ AudioResamplerState AudioResampler::resample(bool stop_gracefully, int32_t *ms_d delay(READ_WRITE_TIMEOUT_MS); } - this->input_transfer_buffer_->transfer_data_from_source(pdMS_TO_TICKS(READ_WRITE_TIMEOUT_MS)); + // Expose a chunk of the ring buffer's internal storage. pre_shift is ignored by RingBufferAudioSource + // (there is no intermediate transfer buffer to compact). + this->audio_source_->fill(pdMS_TO_TICKS(READ_WRITE_TIMEOUT_MS), false); - if (this->input_transfer_buffer_->available() == 0) { + if (this->audio_source_->available() == 0) { // No samples available to process return AudioResamplerState::RESAMPLING; } @@ -112,17 +141,17 @@ AudioResamplerState AudioResampler::resample(bool stop_gracefully, int32_t *ms_d const size_t bytes_free = this->output_transfer_buffer_->free(); const uint32_t frames_free = this->output_stream_info_.bytes_to_frames(bytes_free); - const size_t bytes_available = this->input_transfer_buffer_->available(); + const size_t bytes_available = this->audio_source_->available(); const uint32_t frames_available = this->input_stream_info_.bytes_to_frames(bytes_available); if ((this->input_stream_info_.get_sample_rate() != this->output_stream_info_.get_sample_rate()) || (this->input_stream_info_.get_bits_per_sample() != this->output_stream_info_.get_bits_per_sample())) { // Adjust gain by -3 dB to avoid clipping due to the resampling process esp_audio_libs::resampler::ResamplerResults results = - this->resampler_->resample(this->input_transfer_buffer_->get_buffer_start(), - this->output_transfer_buffer_->get_buffer_end(), frames_available, frames_free, -3); + this->resampler_->resample(this->audio_source_->data(), this->output_transfer_buffer_->get_buffer_end(), + frames_available, frames_free, -3); - this->input_transfer_buffer_->decrease_buffer_length(this->input_stream_info_.frames_to_bytes(results.frames_used)); + this->audio_source_->consume(this->input_stream_info_.frames_to_bytes(results.frames_used)); this->output_transfer_buffer_->increase_buffer_length( this->output_stream_info_.frames_to_bytes(results.frames_generated)); @@ -146,10 +175,10 @@ AudioResamplerState AudioResampler::resample(bool stop_gracefully, int32_t *ms_d const size_t bytes_to_transfer = std::min(this->output_stream_info_.frames_to_bytes(frames_free), this->input_stream_info_.frames_to_bytes(frames_available)); - std::memcpy((void *) this->output_transfer_buffer_->get_buffer_end(), - (void *) this->input_transfer_buffer_->get_buffer_start(), bytes_to_transfer); + std::memcpy((void *) this->output_transfer_buffer_->get_buffer_end(), (const void *) this->audio_source_->data(), + bytes_to_transfer); - this->input_transfer_buffer_->decrease_buffer_length(bytes_to_transfer); + this->audio_source_->consume(bytes_to_transfer); this->output_transfer_buffer_->increase_buffer_length(bytes_to_transfer); } diff --git a/esphome/components/audio/audio_resampler.h b/esphome/components/audio/audio_resampler.h index 575ad13692..c09070c0ce 100644 --- a/esphome/components/audio/audio_resampler.h +++ b/esphome/components/audio/audio_resampler.h @@ -22,7 +22,7 @@ namespace esphome::audio { enum class AudioResamplerState : uint8_t { RESAMPLING, // More data is available to resample FINISHED, // All file data has been resampled and transferred - FAILED, // Unused state included for consistency among Audio classes + FAILED, // Failed to allocate the audio source }; class AudioResampler { @@ -32,14 +32,16 @@ class AudioResampler { * component). Also supports converting bits per sample. */ public: - /// @brief Allocates the input and output transfer buffers - /// @param input_buffer_size Size of the input transfer buffer in bytes. + /// @brief Allocates the output transfer buffer. The input source is created later in resample(). + /// @param input_buffer_size Max bytes exposed per fill() call on the zero-copy input source. /// @param output_buffer_size Size of the output transfer buffer in bytes. AudioResampler(size_t input_buffer_size, size_t output_buffer_size); - /// @brief Adds a source ring buffer for audio data. Takes ownership of the ring buffer in a shared_ptr. - /// @param input_ring_buffer weak_ptr of a shared_ptr of the sink ring buffer to transfer ownership - /// @return ESP_OK if successsful, ESP_ERR_NO_MEM if the transfer buffer wasn't allocated + /// @brief Sets the ring buffer the audio is read from and takes shared ownership of it. The zero-copy + /// RingBufferAudioSource that reads directly from its internal storage is created lazily on the first + /// resample() call, so add_source() and start() may be called in any order. + /// @param input_ring_buffer weak_ptr of a shared_ptr of the source ring buffer to transfer ownership + /// @return ESP_OK if successful, ESP_ERR_INVALID_STATE if the ring buffer is no longer alive esp_err_t add_source(std::weak_ptr &input_ring_buffer); /// @brief Adds a sink ring buffer for resampled audio. Takes ownership of the ring buffer in a shared_ptr. @@ -78,7 +80,8 @@ class AudioResampler { void set_pause_output_state(bool pause_state) { this->pause_output_ = pause_state; } protected: - std::unique_ptr input_transfer_buffer_; + std::shared_ptr source_ring_buffer_; + std::unique_ptr audio_source_; std::unique_ptr output_transfer_buffer_; size_t input_buffer_size_; diff --git a/esphome/components/audio/audio_transfer_buffer.cpp b/esphome/components/audio/audio_transfer_buffer.cpp index d9ce8060e2..a611549e58 100644 --- a/esphome/components/audio/audio_transfer_buffer.cpp +++ b/esphome/components/audio/audio_transfer_buffer.cpp @@ -252,6 +252,22 @@ void RingBufferAudioSource::consume(size_t bytes) { } } +void RingBufferAudioSource::clear_buffered_data() { + // Release the held item before reset() so the source no longer references memory the reset will reclaim. + if (this->acquired_item_ != nullptr) { + this->ring_buffer_->receive_release(this->acquired_item_); + this->acquired_item_ = nullptr; + } + this->current_data_ = nullptr; + this->current_available_ = 0; + this->queued_data_ = nullptr; + this->queued_length_ = 0; + this->item_trailing_ptr_ = nullptr; + this->item_trailing_length_ = 0; + this->splice_length_ = 0; + this->ring_buffer_->reset(); +} + bool RingBufferAudioSource::has_buffered_data() const { // splice_length_ is deliberately not considered here. It holds an incomplete frame whose completion // bytes must still arrive through the ring buffer, which ring_buffer_->available() already reports. diff --git a/esphome/components/audio/audio_transfer_buffer.h b/esphome/components/audio/audio_transfer_buffer.h index b713326141..074684f068 100644 --- a/esphome/components/audio/audio_transfer_buffer.h +++ b/esphome/components/audio/audio_transfer_buffer.h @@ -250,6 +250,10 @@ class RingBufferAudioSource : public AudioReadableBuffer { /// exposure stays in place and fill() returns 0 until it is fully consumed. size_t fill(TickType_t ticks_to_wait, bool pre_shift) override; + /// @brief Discards all buffered audio: releases any held ring buffer item, clears the source's in-flight + /// state, and resets the underlying ring buffer. Must be invoked from the ring buffer's consumer thread. + void clear_buffered_data(); + /// @brief Returns a mutable pointer to the currently exposed audio data. /// The pointer may reference the ring buffer's internal storage or, when exposing a stitched frame /// across a wrap boundary, an internal splice buffer. In either case mutations are safe but data diff --git a/esphome/components/audio_file/__init__.py b/esphome/components/audio_file/__init__.py index 23c90e9b76..53193c8008 100644 --- a/esphome/components/audio_file/__init__.py +++ b/esphome/components/audio_file/__init__.py @@ -72,7 +72,7 @@ def _file_schema(value: ConfigType | str) -> ConfigType: def _validate_file_shorthand(value: str) -> ConfigType: value = cv.string_strict(value) - if value.startswith("http://") or value.startswith("https://"): + if value.startswith(("http://", "https://")): return _file_schema( { CONF_TYPE: TYPE_WEB, @@ -98,7 +98,7 @@ def read_audio_file_and_type(file_config: ConfigType) -> tuple[bytes, MockObj]: else: raise cv.Invalid("Unsupported file source") - with open(path, "rb") as f: + with path.open("rb") as f: data = f.read() try: diff --git a/esphome/components/audio_file/media_source/__init__.py b/esphome/components/audio_file/media_source/__init__.py index 635a51b610..0710582813 100644 --- a/esphome/components/audio_file/media_source/__init__.py +++ b/esphome/components/audio_file/media_source/__init__.py @@ -1,7 +1,5 @@ -from typing import Any - import esphome.codegen as cg -from esphome.components import audio, esp32, media_source, psram +from esphome.components import audio, media_source, psram import esphome.config_validation as cv from esphome.const import CONF_ID, CONF_TASK_STACK_IN_PSRAM from esphome.types import ConfigType @@ -21,19 +19,13 @@ def _request_micro_decoder(config: ConfigType) -> ConfigType: return config -def _validate_task_stack_in_psram(value: Any) -> bool: - if value := cv.boolean(value): - return cv.requires_component(psram.DOMAIN)(value) - return value - - CONFIG_SCHEMA = cv.All( media_source.media_source_schema( AudioFileMediaSource, ) .extend( { - cv.Optional(CONF_TASK_STACK_IN_PSRAM): _validate_task_stack_in_psram, + cv.Optional(CONF_TASK_STACK_IN_PSRAM): psram.validate_task_stack_in_psram, } ) .extend(cv.COMPONENT_SCHEMA), @@ -49,6 +41,4 @@ async def to_code(config: ConfigType) -> None: if config.get(CONF_TASK_STACK_IN_PSRAM): cg.add(var.set_task_stack_in_psram(True)) - esp32.add_idf_sdkconfig_option( - "CONFIG_SPIRAM_ALLOW_STACK_EXTERNAL_MEMORY", True - ) + psram.request_external_task_stack() diff --git a/esphome/components/audio_http/media_source.py b/esphome/components/audio_http/media_source.py index 519d8df698..e8acbc81af 100644 --- a/esphome/components/audio_http/media_source.py +++ b/esphome/components/audio_http/media_source.py @@ -1,7 +1,5 @@ -from typing import Any - import esphome.codegen as cg -from esphome.components import audio, esp32, media_source, psram +from esphome.components import audio, media_source, psram import esphome.config_validation as cv from esphome.const import CONF_BUFFER_SIZE, CONF_ID, CONF_TASK_STACK_IN_PSRAM from esphome.types import ConfigType @@ -20,14 +18,6 @@ def _request_micro_decoder(config: ConfigType) -> ConfigType: return config -def _validate_task_stack_in_psram(value: Any) -> bool: - # Only require the psram component when actually enabling PSRAM stacks; validating - # the boolean first means `false` doesn't trigger the requires_component check. - if value := cv.boolean(value): - return cv.requires_component(psram.DOMAIN)(value) - return value - - CONFIG_SCHEMA = cv.All( media_source.media_source_schema( AudioHTTPMediaSource, @@ -37,7 +27,7 @@ CONFIG_SCHEMA = cv.All( cv.Optional(CONF_BUFFER_SIZE, default=50000): cv.int_range( min=5000, max=1000000 ), - cv.Optional(CONF_TASK_STACK_IN_PSRAM): _validate_task_stack_in_psram, + cv.Optional(CONF_TASK_STACK_IN_PSRAM): psram.validate_task_stack_in_psram, } ) .extend(cv.COMPONENT_SCHEMA), @@ -53,7 +43,5 @@ async def to_code(config: ConfigType) -> None: if config.get(CONF_TASK_STACK_IN_PSRAM): cg.add(var.set_task_stack_in_psram(True)) - esp32.add_idf_sdkconfig_option( - "CONFIG_SPIRAM_ALLOW_STACK_EXTERNAL_MEMORY", True - ) + psram.request_external_task_stack() cg.add(var.set_buffer_size(config[CONF_BUFFER_SIZE])) diff --git a/esphome/components/bluetooth_proxy/bluetooth_connection.cpp b/esphome/components/bluetooth_proxy/bluetooth_connection.cpp index a42fb3544c..7ba9e61e19 100644 --- a/esphome/components/bluetooth_proxy/bluetooth_connection.cpp +++ b/esphome/components/bluetooth_proxy/bluetooth_connection.cpp @@ -144,7 +144,7 @@ void BluetoothConnection::loop() { } } -void BluetoothConnection::on_disconnect_complete_(esp_err_t reason) { +void BluetoothConnection::on_disconnect_complete(esp_err_t reason) { // Called from both the CLOSE_EVT handler and the DISCONNECTING safety timeout in the // base class. Free the proxy slot, notify the API client, and reset send_service_. // address_ may already be 0 if reset_connection_ ran earlier on this teardown. diff --git a/esphome/components/bluetooth_proxy/bluetooth_connection.h b/esphome/components/bluetooth_proxy/bluetooth_connection.h index 54e7c83fb3..e5600f6af4 100644 --- a/esphome/components/bluetooth_proxy/bluetooth_connection.h +++ b/esphome/components/bluetooth_proxy/bluetooth_connection.h @@ -33,7 +33,7 @@ class BluetoothConnection final : public esp32_ble_client::BLEClientBase { protected: friend class BluetoothProxy; - void on_disconnect_complete_(esp_err_t reason) override; + void on_disconnect_complete(esp_err_t reason) override; bool supports_efficient_uuids_() const; void send_service_for_discovery_(); diff --git a/esphome/components/bme68x_bsec2/__init__.py b/esphome/components/bme68x_bsec2/__init__.py index 5083d283ef..62cd9e2e36 100644 --- a/esphome/components/bme68x_bsec2/__init__.py +++ b/esphome/components/bme68x_bsec2/__init__.py @@ -169,7 +169,7 @@ async def to_code_base(config): path = _compute_local_file_path(_compute_url(config)) try: - with open(path, encoding="utf-8") as f: + with path.open(encoding="utf-8") as f: bsec2_iaq_config = f.read() except Exception as e: raise core.EsphomeError( diff --git a/esphome/components/dsmr/dsmr.h b/esphome/components/dsmr/dsmr.h index 626a389c1f..e55db9f976 100644 --- a/esphome/components/dsmr/dsmr.h +++ b/esphome/components/dsmr/dsmr.h @@ -16,9 +16,14 @@ #include #include +// On ESP8266 Arduino, BearSSL is the native crypto. The mbedtls headers can +// still be in scope when a sibling component (e.g. wireguard) pulls in +// esp_mbedtls_esp8266, but that build leaves MBEDTLS_GCM_C disabled so the +// gcm.h symbols are unresolved at link time. Force BearSSL on ESP8266 to +// avoid that linker error. #if __has_include() #include -#elif __has_include() +#elif !defined(USE_ESP8266) && __has_include() #if __has_include() #include #endif @@ -33,7 +38,7 @@ namespace esphome::dsmr { #if __has_include() using Aes128GcmDecryptorImpl = dsmr_parser::Aes128GcmTfPsa; -#elif __has_include() +#elif !defined(USE_ESP8266) && __has_include() using Aes128GcmDecryptorImpl = dsmr_parser::Aes128GcmMbedTls; #else using Aes128GcmDecryptorImpl = dsmr_parser::Aes128GcmBearSsl; diff --git a/esphome/components/esp32/__init__.py b/esphome/components/esp32/__init__.py index 24312d64ad..7b94a26f54 100644 --- a/esphome/components/esp32/__init__.py +++ b/esphome/components/esp32/__init__.py @@ -46,7 +46,7 @@ from esphome.const import ( Toolchain, __version__, ) -from esphome.core import CORE, HexInt, Library +from esphome.core import CORE, EsphomeError, HexInt, Library from esphome.core.config import BOARD_MAX_LENGTH from esphome.coroutine import CoroPriority, coroutine_with_priority from esphome.espidf.component import generate_idf_component @@ -56,7 +56,7 @@ from esphome.types import ConfigType from esphome.writer import clean_build, clean_cmake_cache from .boards import BOARDS, STANDARD_BOARDS -from .const import ( # noqa +from .const import ( KEY_ARDUINO_LIBRARIES, KEY_BOARD, KEY_COMPONENTS, @@ -86,7 +86,7 @@ from .const import ( # noqa ) # force import gpio to register pin schema -from .gpio import esp32_pin_to_code # noqa +from .gpio import esp32_pin_to_code # noqa: F401 _LOGGER = logging.getLogger(__name__) AUTO_LOAD = ["preferences"] @@ -1816,12 +1816,12 @@ async def to_code(config): Path(__file__).parent / "iram_fix.py.script", ) else: - cg.add_build_flag("-Wno-error=format") - cg.add_build_flag("-Wno-error=maybe-uninitialized") - cg.add_build_flag("-Wno-error=overloaded-virtual") - cg.add_build_flag("-Wno-error=reorder") - cg.add_build_flag("-Wno-error=volatile") - cg.add_build_flag("-Wno-error=cpp") + # Demote IDF's blanket -Werror to warnings so third-party libs + # and user lambdas don't need a -Wno-error= per warning. + # The sdkconfig knob disables IDF's rewrite to -Werror=all (which + # can't be globally undone); -Wno-error then handles the demotion. + add_idf_sdkconfig_option("CONFIG_COMPILER_DISABLE_DEFAULT_ERRORS", False) + cg.add_build_flag("-Wno-error") # -Wno- (not -Wno-error=): suppress entirely, too noisy on C++ aggregates cg.add_build_flag("-Wno-missing-field-initializers") @@ -2658,13 +2658,29 @@ def copy_files(): def _decode_pc(config, addr): - from esphome.platformio import toolchain + # _decode_pc runs from the api log processor's asyncio callback, which + # only catches EsphomeError. Any other exception escaping here tears down + # the protocol and triggers an infinite reconnect/replay loop. Convert + # toolchain-resolution errors (e.g. missing build dir / cmake cache) into + # EsphomeError so the caller can disable decoding cleanly. + if CORE.using_toolchain_esp_idf: + from esphome.espidf import toolchain as idf_toolchain - idedata = toolchain.get_idedata(config) - if not idedata.addr2line_path or not idedata.firmware_elf_path: + try: + addr2line_path = idf_toolchain.get_addr2line_path() + firmware_elf_path = idf_toolchain.get_elf_path() + except RuntimeError as err: + raise EsphomeError(f"ESP-IDF toolchain not available: {err}") from err + else: + from esphome.platformio import toolchain + + idedata = toolchain.get_idedata(config) + addr2line_path = idedata.addr2line_path + firmware_elf_path = idedata.firmware_elf_path + if not addr2line_path or not firmware_elf_path: _LOGGER.debug("decode_pc no addr2line") return - command = [idedata.addr2line_path, "-pfiaC", "-e", idedata.firmware_elf_path, addr] + command = [str(addr2line_path), "-pfiaC", "-e", str(firmware_elf_path), addr] try: translation = subprocess.check_output(command, close_fds=False).decode().strip() except Exception: # pylint: disable=broad-except diff --git a/esphome/components/esp32_ble_client/ble_client_base.cpp b/esphome/components/esp32_ble_client/ble_client_base.cpp index 0c79eecd3a..3fb9632e9a 100644 --- a/esphome/components/esp32_ble_client/ble_client_base.cpp +++ b/esphome/components/esp32_ble_client/ble_client_base.cpp @@ -72,7 +72,7 @@ void BLEClientBase::loop() { // never delivered CLOSE_EVT/DISCONNECT_EVT, services would leak without this call. this->release_services(); this->set_idle_(); - this->on_disconnect_complete_(ESP_GATT_CONN_TIMEOUT); + this->on_disconnect_complete(ESP_GATT_CONN_TIMEOUT); } } @@ -419,7 +419,7 @@ bool BLEClientBase::gattc_event_handler(esp_gattc_cb_event_t event, esp_gatt_if_ this->log_gattc_lifecycle_event_("CLOSE"); this->release_services(); this->set_idle_(); - this->on_disconnect_complete_(param->close.reason); + this->on_disconnect_complete(param->close.reason); break; } case ESP_GATTC_SEARCH_RES_EVT: { diff --git a/esphome/components/esp32_ble_client/ble_client_base.h b/esphome/components/esp32_ble_client/ble_client_base.h index f7e87e667c..0291a4b993 100644 --- a/esphome/components/esp32_ble_client/ble_client_base.h +++ b/esphome/components/esp32_ble_client/ble_client_base.h @@ -145,7 +145,7 @@ class BLEClientBase : public espbt::ESPBTClient, public Component { /// Subclasses with extra per-connection accounting (e.g. bluetooth_proxy slot state) /// override this to release that state. `reason` is the controller reason code, or /// ESP_GATT_CONN_TIMEOUT for the safety-timeout path. - virtual void on_disconnect_complete_(esp_err_t reason) {} + virtual void on_disconnect_complete(esp_err_t reason) {} /// Transition to IDLE and reset conn_id — call when the connection is fully dead. void set_idle_() { this->set_state(espbt::ClientState::IDLE); diff --git a/esphome/components/esp32_hosted/__init__.py b/esphome/components/esp32_hosted/__init__.py index 71d1fd3ac1..94e20ea6c9 100644 --- a/esphome/components/esp32_hosted/__init__.py +++ b/esphome/components/esp32_hosted/__init__.py @@ -3,6 +3,7 @@ from pathlib import Path from esphome import pins from esphome.components import esp32 +from esphome.components.const import CONF_USE_PSRAM import esphome.config_validation as cv from esphome.const import ( CONF_CLK_PIN, @@ -39,6 +40,7 @@ BASE_SCHEMA = cv.Schema( cv.Required(CONF_VARIANT): cv.one_of(*esp32.VARIANTS, upper=True), cv.Required(CONF_ACTIVE_HIGH): cv.boolean, cv.Required(CONF_RESET_PIN): pins.internal_gpio_output_pin_number, + cv.Optional(CONF_USE_PSRAM, default=False): cv.boolean, } ) @@ -242,6 +244,12 @@ async def to_code(config): else: _configure_spi(config) + # Place the transport mempool in PSRAM. Required on memory-tight host + # configurations (e.g. P4 with a large LVGL UI) where the internal-RAM + # mempool allocation fails at boot with `sdio_mempool_create` assert. + if config[CONF_USE_PSRAM]: + esp32.add_idf_sdkconfig_option("CONFIG_ESP_HOSTED_MEMPOOL_PREFER_SPIRAM", True) + # Library versions idf_ver = esp32.idf_version() os.environ["ESP_IDF_VERSION"] = f"{idf_ver.major}.{idf_ver.minor}" @@ -249,7 +257,7 @@ async def to_code(config): esp32.add_idf_component(name="espressif/esp_wifi_remote", ref="1.5.1") esp32.add_idf_component(name="espressif/wifi_remote_over_eppp", ref="0.3.2") esp32.add_idf_component(name="espressif/eppp_link", ref="1.1.5") - esp32.add_idf_component(name="espressif/esp_hosted", ref="2.12.7") + esp32.add_idf_component(name="espressif/esp_hosted", ref="2.12.8") else: esp32.add_idf_component(name="espressif/esp_wifi_remote", ref="0.13.0") esp32.add_idf_component(name="espressif/eppp_link", ref="0.2.0") diff --git a/esphome/components/esp32_hosted/update/__init__.py b/esphome/components/esp32_hosted/update/__init__.py index b258a26b08..202df21ab5 100644 --- a/esphome/components/esp32_hosted/update/__init__.py +++ b/esphome/components/esp32_hosted/update/__init__.py @@ -75,7 +75,7 @@ def _validate_firmware(config: dict[str, Any]) -> None: return path = CORE.relative_config_path(config[CONF_PATH]) - with open(path, "rb") as f: + with path.open("rb") as f: firmware_data = f.read() calculated = hashlib.sha256(firmware_data).hexdigest() expected = config[CONF_SHA256].lower() @@ -93,7 +93,7 @@ async def to_code(config: dict[str, Any]) -> None: if config[CONF_TYPE] == TYPE_EMBEDDED: path = config[CONF_PATH] - with open(CORE.relative_config_path(path), "rb") as f: + with CORE.relative_config_path(path).open("rb") as f: firmware_data = f.read() rhs = [HexInt(x) for x in firmware_data] arr_id = ID(f"{config[CONF_ID]}_data", is_declaration=True, type=cg.uint8) diff --git a/esphome/components/esphome/ota/__init__.py b/esphome/components/esphome/ota/__init__.py index f7793b1493..66a33e1935 100644 --- a/esphome/components/esphome/ota/__init__.py +++ b/esphome/components/esphome/ota/__init__.py @@ -133,7 +133,7 @@ CONFIG_SCHEMA = cv.All( host=8082, ): cv.port, cv.Optional(CONF_ALLOW_PARTITION_ACCESS, default=False): cv.boolean, - cv.Optional(CONF_PASSWORD): cv.string, + cv.Optional(CONF_PASSWORD): cv.sensitive(), cv.Optional(CONF_NUM_ATTEMPTS): cv.invalid( f"'{CONF_SAFE_MODE}' (and its related configuration variables) has moved from 'ota' to its own component. See https://esphome.io/components/safe_mode" ), diff --git a/esphome/components/espnow/__init__.py b/esphome/components/espnow/__init__.py index 7861c0affa..13f278d3bc 100644 --- a/esphome/components/espnow/__init__.py +++ b/esphome/components/espnow/__init__.py @@ -17,7 +17,7 @@ from esphome.core import HexInt from esphome.types import ConfigType CODEOWNERS = ["@jesserockz"] - +AUTO_LOAD = ["network"] byte_vector = cg.std_vector.template(cg.uint8) peer_address_t = cg.std_ns.class_("array").template(cg.uint8, 6) diff --git a/esphome/components/espnow/espnow_component.cpp b/esphome/components/espnow/espnow_component.cpp index 91d44394e8..403e6f4944 100644 --- a/esphome/components/espnow/espnow_component.cpp +++ b/esphome/components/espnow/espnow_component.cpp @@ -149,12 +149,6 @@ bool ESPNowComponent::is_wifi_enabled() { } void ESPNowComponent::setup() { -#ifndef USE_WIFI - // Initialize LwIP stack for wake_loop_threadsafe() socket support - // When WiFi component is present, it handles esp_netif_init() - ESP_ERROR_CHECK(esp_netif_init()); -#endif - if (this->enable_on_boot_) { this->enable_(); } else { @@ -174,8 +168,6 @@ void ESPNowComponent::enable() { void ESPNowComponent::enable_() { if (!this->is_wifi_enabled()) { - esp_event_loop_create_default(); - wifi_init_config_t cfg = WIFI_INIT_CONFIG_DEFAULT(); ESP_ERROR_CHECK(esp_wifi_init(&cfg)); diff --git a/esphome/components/ethernet/ethernet_component_esp32.cpp b/esphome/components/ethernet/ethernet_component_esp32.cpp index d4585bf100..6481c8c1f4 100644 --- a/esphome/components/ethernet/ethernet_component_esp32.cpp +++ b/esphome/components/ethernet/ethernet_component_esp32.cpp @@ -5,6 +5,7 @@ #include "esphome/core/application.h" #include "esphome/core/helpers.h" #include "esphome/core/log.h" +#include "w5500_custom_spi.h" #include #include @@ -163,11 +164,7 @@ void EthernetComponent::setup() { err = spi_bus_initialize(host, &buscfg, SPI_DMA_CH_AUTO); ESPHL_ERROR_CHECK(err, "SPI bus initialize error"); #endif - - err = esp_netif_init(); - ESPHL_ERROR_CHECK(err, "ETH netif init error"); - err = esp_event_loop_create_default(); - ESPHL_ERROR_CHECK(err, "ETH event loop error"); + // Network interface setup handled by network component esp_netif_config_t cfg = ESP_NETIF_DEFAULT_ETH(); this->eth_netif_ = esp_netif_new(&cfg); @@ -207,6 +204,10 @@ void EthernetComponent::setup() { #ifdef USE_ETHERNET_SPI_POLLING_SUPPORT w5500_config.poll_period_ms = this->polling_interval_; #endif + // Install the custom SPI driver that offloads the bulk RX/TX frame transfers off the busy-wait + // path. w5500_config (and the devcfg it references) outlives esp_eth_mac_new_w5500() below, which + // runs the driver's init(). + install_w5500_async_spi(w5500_config); #elif defined(USE_ETHERNET_DM9051) dm9051_config.int_gpio_num = this->interrupt_pin_; #ifdef USE_ETHERNET_SPI_POLLING_SUPPORT diff --git a/esphome/components/ethernet/w5500_custom_spi.cpp b/esphome/components/ethernet/w5500_custom_spi.cpp new file mode 100644 index 0000000000..ed4f149738 --- /dev/null +++ b/esphome/components/ethernet/w5500_custom_spi.cpp @@ -0,0 +1,118 @@ +#include "w5500_custom_spi.h" + +#if defined(USE_ESP32) && defined(USE_ETHERNET_W5500) + +#include +#include +#include +#include +#include + +namespace esphome::ethernet { + +namespace { + +// Per-device context returned by init() and handed back to read/write/deinit. +struct W5500CustomSpiContext { + spi_device_handle_t handle; + SemaphoreHandle_t lock; +}; + +// Transfers up to the ESP32 SPI hardware FIFO size (64 bytes) stay on the polling path; larger +// transfers (the frame payloads) use the blocking, DMA-backed transmit. +constexpr uint32_t W5500_SPI_BULK_THRESHOLD = 64; +constexpr uint32_t W5500_SPI_LOCK_TIMEOUT_MS = 50; + +void *w5500_custom_spi_init(const void *spi_config) { + const auto *config = static_cast(spi_config); + auto *ctx = new (std::nothrow) W5500CustomSpiContext{}; + if (ctx == nullptr) { + return nullptr; + } + // The W5500 SPI frame carries the 16-bit address in the command phase and the 8-bit control + // byte in the address phase; mirror what the stock driver configures. + spi_device_interface_config_t devcfg = *config->spi_devcfg; + devcfg.command_bits = 16; + devcfg.address_bits = 8; + if (spi_bus_add_device(config->spi_host_id, &devcfg, &ctx->handle) != ESP_OK) { + delete ctx; + return nullptr; + } + ctx->lock = xSemaphoreCreateMutex(); + if (ctx->lock == nullptr) { + spi_bus_remove_device(ctx->handle); + delete ctx; + return nullptr; + } + return ctx; +} + +esp_err_t w5500_custom_spi_deinit(void *spi_ctx) { + auto *ctx = static_cast(spi_ctx); + spi_bus_remove_device(ctx->handle); + vSemaphoreDelete(ctx->lock); + delete ctx; + return ESP_OK; +} + +// Runs one transaction under the device lock, choosing the polling vs blocking transmit by size. +// Bulk payloads (> FIFO size) block so the calling task sleeps while DMA runs; small register +// accesses stay on the cheaper polling path. Used by both read and write. +esp_err_t w5500_custom_spi_transfer(W5500CustomSpiContext *ctx, spi_transaction_t *trans, uint32_t len) { + if (xSemaphoreTake(ctx->lock, pdMS_TO_TICKS(W5500_SPI_LOCK_TIMEOUT_MS)) != pdTRUE) { + return ESP_ERR_TIMEOUT; + } + esp_err_t ret; + if (len > W5500_SPI_BULK_THRESHOLD) { + ret = spi_device_transmit(ctx->handle, trans); + } else { + ret = spi_device_polling_transmit(ctx->handle, trans); + } + xSemaphoreGive(ctx->lock); + return ret; +} + +esp_err_t w5500_custom_spi_write(void *spi_ctx, uint32_t cmd, uint32_t addr, const void *data, uint32_t len) { + auto *ctx = static_cast(spi_ctx); + spi_transaction_t trans = {}; + trans.cmd = static_cast(cmd); + trans.addr = addr; + trans.length = 8 * len; + trans.tx_buffer = data; + return w5500_custom_spi_transfer(ctx, &trans, len); +} + +esp_err_t w5500_custom_spi_read(void *spi_ctx, uint32_t cmd, uint32_t addr, void *data, uint32_t len) { + auto *ctx = static_cast(spi_ctx); + spi_transaction_t trans = {}; + // Reads of <= 4 bytes use the transaction's inline RX buffer to avoid 4-byte boundary + // overwrites of adjacent registers (same guard the stock driver uses). + const bool use_rxdata = len <= 4; + trans.flags = use_rxdata ? SPI_TRANS_USE_RXDATA : 0; + trans.cmd = static_cast(cmd); + trans.addr = addr; + trans.length = 8 * len; + trans.rx_buffer = data; + esp_err_t ret = w5500_custom_spi_transfer(ctx, &trans, len); + if (use_rxdata && (ret == ESP_OK)) { + memcpy(data, trans.rx_data, len); + } + return ret; +} + +} // namespace + +void install_w5500_async_spi(eth_w5500_config_t &config) { + // Point the custom driver's config at the W5500 config itself; init() reads spi_host_id and + // spi_devcfg back out of it. The self-reference is valid because both the config and the + // spi_devcfg it points at outlive the esp_eth_mac_new_w5500() call that runs init(). + config.custom_spi_driver.config = &config; + config.custom_spi_driver.init = w5500_custom_spi_init; + config.custom_spi_driver.deinit = w5500_custom_spi_deinit; + config.custom_spi_driver.read = w5500_custom_spi_read; + config.custom_spi_driver.write = w5500_custom_spi_write; +} + +} // namespace esphome::ethernet + +#endif // USE_ESP32 && USE_ETHERNET_W5500 diff --git a/esphome/components/ethernet/w5500_custom_spi.h b/esphome/components/ethernet/w5500_custom_spi.h new file mode 100644 index 0000000000..8756a149af --- /dev/null +++ b/esphome/components/ethernet/w5500_custom_spi.h @@ -0,0 +1,35 @@ +#pragma once + +#include "esphome/core/defines.h" + +#if defined(USE_ESP32) && defined(USE_ETHERNET_W5500) + +#include +// IDF 6.0 moved the per-chip SPI MAC drivers to the Espressif Component Registry; eth_w5500_config_t +// is no longer reachable through esp_eth.h and needs the explicit header. +#if ESP_IDF_VERSION >= ESP_IDF_VERSION_VAL(6, 0, 0) +#include +#else +#include +#endif + +namespace esphome::ethernet { + +// Installs a custom W5500 SPI driver that offloads the bulk frame transfers off the busy-wait path. +// +// The stock W5500 driver runs every SPI transfer through spi_device_polling_transmit(), which +// busy-waits the CPU for the whole transfer. The frame payload (one large read per received frame, +// one large write per transmitted frame) is by far the biggest transfer, so the RX task and the TX +// caller each spin for hundreds of microseconds per frame. This driver sends payload transfers +// through the blocking, interrupt-driven spi_device_transmit() instead, so the calling task sleeps +// while DMA moves the bytes. Small register accesses stay on the polling path, where the busy-wait +// is cheaper than an interrupt round-trip. +// +// Must be called before esp_eth_mac_new_w5500(). The driver reads spi_host_id and spi_devcfg back +// out of `config` in its init() callback, so `config` (and the spi_devcfg it points at) must stay +// alive until esp_eth_mac_new_w5500() returns. +void install_w5500_async_spi(eth_w5500_config_t &config); + +} // namespace esphome::ethernet + +#endif // USE_ESP32 && USE_ETHERNET_W5500 diff --git a/esphome/components/external_components/__init__.py b/esphome/components/external_components/__init__.py index 6eb577e5ad..c892ec1112 100644 --- a/esphome/components/external_components/__init__.py +++ b/esphome/components/external_components/__init__.py @@ -81,7 +81,7 @@ def _process_single_config(config: dict[str, Any]) -> None: elif conf[CONF_TYPE] == TYPE_LOCAL: components_dir = Path(CORE.relative_config_path(conf[CONF_PATH])) else: - raise NotImplementedError() + raise NotImplementedError if config[CONF_COMPONENTS] == "all": num_components = len(list(components_dir.glob("*/__init__.py"))) diff --git a/esphome/components/font/__init__.py b/esphome/components/font/__init__.py index a10c45a9d7..7510f2f8b6 100644 --- a/esphome/components/font/__init__.py +++ b/esphome/components/font/__init__.py @@ -401,7 +401,7 @@ def validate_file_shorthand(value): data[CONF_WEIGHT] = weight[1:] return font_file_schema(data) - if value.startswith("http://") or value.startswith("https://"): + if value.startswith(("http://", "https://")): return font_file_schema( { CONF_TYPE: TYPE_WEB, @@ -563,13 +563,13 @@ async def to_code(config): point_set.update(flatten(config[CONF_GLYPHS])) # Create the codepoint to font file map base_font = FONT_CACHE[config[CONF_FILE]] - point_font_map: dict[str, Face] = {c: base_font for c in point_set} + point_font_map: dict[str, Face] = dict.fromkeys(point_set, base_font) # process extras, updating the map and extending the codepoint list for extra in config[CONF_EXTRAS]: extra_points = flatten(extra[CONF_GLYPHS]) point_set.update(extra_points) extra_font = FONT_CACHE[extra[CONF_FILE]] - point_font_map.update({c: extra_font for c in extra_points}) + point_font_map.update(dict.fromkeys(extra_points, extra_font)) codepoints = list(point_set) codepoints.sort(key=functools.cmp_to_key(glyph_comparator)) @@ -594,7 +594,9 @@ async def to_code(config): x.height, ] for (x, y) in zip( - glyph_args, list(accumulate([len(x.bitmap_data) for x in glyph_args])) + glyph_args, + list(accumulate([len(x.bitmap_data) for x in glyph_args])), + strict=True, ) ] diff --git a/esphome/components/homeassistant/time/__init__.py b/esphome/components/homeassistant/time/__init__.py index 62cb96a25a..05ca86a26e 100644 --- a/esphome/components/homeassistant/time/__init__.py +++ b/esphome/components/homeassistant/time/__init__.py @@ -1,7 +1,7 @@ import esphome.codegen as cg from esphome.components import time as time_ import esphome.config_validation as cv -from esphome.const import CONF_ID +from esphome.const import CONF_ID, CONF_TIMEZONE from .. import homeassistant_ns @@ -21,3 +21,5 @@ async def to_code(config): await time_.register_time(var, config) await cg.register_component(var, config) cg.add_define("USE_HOMEASSISTANT_TIME") + if CONF_TIMEZONE not in config: + cg.add_define("USE_HOMEASSISTANT_TIMEZONE") diff --git a/esphome/components/host/__init__.py b/esphome/components/host/__init__.py index 8adbfb02ec..50deb1acf6 100644 --- a/esphome/components/host/__init__.py +++ b/esphome/components/host/__init__.py @@ -14,7 +14,7 @@ from esphome.core import CORE from .const import KEY_HOST # force import gpio to register pin schema -from .gpio import host_pin_to_code # noqa +from .gpio import host_pin_to_code # noqa: F401 CODEOWNERS = ["@esphome/core", "@clydebarrow"] AUTO_LOAD = ["network", "preferences"] diff --git a/esphome/components/http_request/__init__.py b/esphome/components/http_request/__init__.py index 90879c459e..fd033dac7f 100644 --- a/esphome/components/http_request/__init__.py +++ b/esphome/components/http_request/__init__.py @@ -1,3 +1,5 @@ +from pathlib import Path + from esphome import automation import esphome.codegen as cg from esphome.components import esp32 @@ -63,7 +65,7 @@ CONF_JSON = "json" def validate_url(value): value = cv.url(value) - if value.startswith("http://") or value.startswith("https://"): + if value.startswith(("http://", "https://")): return value raise cv.Invalid("URL must start with 'http://' or 'https://'") @@ -174,7 +176,7 @@ async def to_code(config): if config.get(CONF_VERIFY_SSL): if ca_cert_path := config.get(CONF_CA_CERTIFICATE_PATH): - with open(ca_cert_path, encoding="utf-8") as f: + with Path(ca_cert_path).open(encoding="utf-8") as f: ca_cert_content = f.read() cg.add(var.set_ca_certificate(ca_cert_content)) else: diff --git a/esphome/components/http_request/ota/__init__.py b/esphome/components/http_request/ota/__init__.py index fb59e51943..1bb54599dc 100644 --- a/esphome/components/http_request/ota/__init__.py +++ b/esphome/components/http_request/ota/__init__.py @@ -57,7 +57,7 @@ OTA_HTTP_REQUEST_FLASH_ACTION_SCHEMA = cv.All( cv.Optional(CONF_MD5): cv.templatable( cv.All(cv.string, cv.Length(min=32, max=32)) ), - cv.Optional(CONF_PASSWORD): cv.templatable(cv.string), + cv.Optional(CONF_PASSWORD): cv.sensitive(cv.templatable(cv.string)), cv.Optional(CONF_USERNAME): cv.templatable(cv.string), cv.Required(CONF_URL): cv.templatable(cv.url), } diff --git a/esphome/components/image/__init__.py b/esphome/components/image/__init__.py index 365554f7d2..5f8e5ca132 100644 --- a/esphome/components/image/__init__.py +++ b/esphome/components/image/__init__.py @@ -395,7 +395,7 @@ def download_image(value): def is_svg_file(file): if not file: return False - with open(file, "rb") as f: + with Path(file).open("rb") as f: return " RAM0 AT> FLASH), after KEEP(*(.vectors)) +// so the Cortex-M4 vector table stays 512-byte-aligned for VTOR. // // BK72xx (all variants) are left as a no-op: their SDK wraps flash // operations in GLOBAL_INT_DISABLE() which masks FIQ + IRQ at the CPU for @@ -26,13 +34,7 @@ // layer. #if defined(USE_BK72XX) #define IRAM_ATTR -#elif defined(USE_LIBRETINY_VARIANT_RTL8710B) -// Stock linker consumes *(.image2.ram.text*) into .ram_image2.text (> BD_RAM). -#define IRAM_ATTR __attribute__((noinline, section(".image2.ram.text"))) #else -// RTL8720C: stock linker consumes *(.sram.text*) into .ram.code_text. -// LN882H: patch_linker.py.script injects *(.sram.text*) into -// .flash_copysection (> RAM0 AT> FLASH). #define IRAM_ATTR __attribute__((noinline, section(".sram.text"))) #endif #define PROGMEM diff --git a/esphome/components/libretiny/patch_linker.py.script b/esphome/components/libretiny/patch_linker.py.script index 3a8a4787ed..dfeaaa57d1 100644 --- a/esphome/components/libretiny/patch_linker.py.script +++ b/esphome/components/libretiny/patch_linker.py.script @@ -6,12 +6,18 @@ import re import subprocess # ESPHome marks ISR code IRAM_ATTR, which on LibreTiny maps to a per-family -# section routed into RAM-executable memory (see esphome/core/hal.h). +# section routed into RAM-executable memory (see esphome/core/hal.h). The +# input section name is always .sram.text; only the output section it lands +# in differs per family. # # This script is NOT loaded on BK72xx (IRAM_ATTR is a no-op there; the SDK # masks FIQ+IRQ around flash writes). On the remaining families: -# - RTL8710B: hal.h uses section(".image2.ram.text"); stock linker consumes it. -# - RTL8720C: hal.h uses section(".sram.text"); stock linker consumes it. +# - RTL8720C: stock linker consumes *(.sram.text*) into .ram.code_text. +# - RTL8710B: stock linker has KEEP(*(.image2.ram.text*)) in .ram_image2.text, +# but ltchiptool's AmebaZ elf2bin (soc/ambz/binary.py) does NOT list +# .ram_image2.text in sections_ram, so code there is silently dropped from +# the flashed image. Inject KEEP(*(.sram.text*)) at the top of +# .ram_image2.data (which IS extracted) instead. # - LN882H: stock linker has no glob for ".sram.text", so we inject # KEEP(*(.sram.text*)) into ".flash_copysection" (> RAM0 AT> FLASH) # immediately after KEEP(*(.vectors)), so the vector table stays at @@ -34,6 +40,20 @@ _KEEP_LINE = ( # aligned address; injecting before the vectors would push them to an # unaligned offset and mis-route every IRQ handler. _LN_COPY = re.compile(r"(KEEP\(\*\(\.vectors\)\)[^\n]*\n)") +# Inject at the top of .ram_image2.data, before __data_start__ so our code +# does not fall inside the data range markers. .ram_image2.data is one of the +# sections ltchiptool's AmebaZ elf2bin extracts; BD_RAM is rwx so the code is +# executable. AmbZ has no C runtime .data copy loop (the bootloader loads +# image2 into BD_RAM whole) so the inline code is not clobbered after boot. +# +# The regex is intentionally strict (no attribute / ALIGN between the section +# name and the opening brace, brace on its own line). If a future AmbZ SDK +# linker template changes this format, _pre_link raises RuntimeError on the +# unpatched .ld file(s), and the RTL8710B CI compile job in +# tests/test_build_components fails on the PR, surfacing the mismatch loudly +# rather than silently shipping a binary with IRAM_ATTR code dropped from +# one or both OTA slots. +_AMBZ_DATA = re.compile(r"(\.ram_image2\.data\s*:\s*\n?\s*\{\s*\n)") def _detect(env): @@ -71,12 +91,11 @@ def _inject_keep(host_section): # Variants not listed here intentionally have no .ld patcher: -# - RTL8710B: hal.h uses section(".image2.ram.text") which the stock linker -# already routes into .ram_image2.text (> BD_RAM). -# - RTL8720C: stock linker already consumes *(.sram.text*). +# - RTL8720C: stock linker already consumes *(.sram.text*) into .ram.code_text. # - BK72xx (all): SDK masks FIQ+IRQ around flash writes, IRAM_ATTR is no-op. _PATCHERS_BY_VARIANT = { "LN882H": (_inject_keep(_LN_COPY),), + "RTL8710B": (_inject_keep(_AMBZ_DATA),), } @@ -87,13 +106,14 @@ def _patchers_for(variant): def _pre_link(target, source, env): build_dir = env.subst("$BUILD_DIR") ld_files = [f for f in os.listdir(build_dir) if f.endswith(".ld")] - patched = 0 + patched = [] + unpatched = [] for name in ld_files: path = os.path.join(build_dir, name) with open(path, "r", encoding="utf-8") as fh: original = fh.read() if _MARKER in original: - patched += 1 + patched.append(name) continue content = original for fn in _patchers: @@ -102,7 +122,9 @@ def _pre_link(target, source, env): with open(path, "w", encoding="utf-8") as fh: fh.write(content) print("ESPHome: patched {} for IRAM_ATTR placement".format(name)) - patched += 1 + patched.append(name) + else: + unpatched.append(name) if not patched: raise RuntimeError( "ESPHome: no .ld in {} was patched for IRAM_ATTR. Update the " @@ -110,6 +132,20 @@ def _pre_link(target, source, env): build_dir ) ) + # Every .ld in the build must be patched. RTL8710B generates one .ld per + # OTA slot (xip1, xip2); if only one matches, the unpatched slot would + # ship with IRAM_ATTR code dropped to zeros and brick the device on the + # boot after an OTA into that slot. + if unpatched: + raise RuntimeError( + "ESPHome: {} of {} .ld file(s) in {} were not patched for " + "IRAM_ATTR: {}. The regex in patch_linker.py.script " + "(_PATCHERS_BY_VARIANT[{!r}]) matched the others but not " + "these. Update the regex to cover all linker scripts.".format( + len(unpatched), len(ld_files), build_dir, + ", ".join(unpatched), _variant, + ) + ) # Substrings matched against demangled names as a fallback on RTL8720C, diff --git a/esphome/components/light/__init__.py b/esphome/components/light/__init__.py index 68d9f85af2..7c4d7ed431 100644 --- a/esphome/components/light/__init__.py +++ b/esphome/components/light/__init__.py @@ -58,7 +58,7 @@ from .effects import ( RGB_EFFECTS, validate_effects, ) -from .types import ( # noqa +from .types import ( # noqa: F401 AddressableLight, AddressableLightState, ColorMode, diff --git a/esphome/components/logger/__init__.py b/esphome/components/logger/__init__.py index c6c440564a..5f160352cc 100644 --- a/esphome/components/logger/__init__.py +++ b/esphome/components/logger/__init__.py @@ -514,7 +514,7 @@ def validate_printf(value): (?:hh|h|ll|l|j|z|t|L|w|I|I32|I64)? # size [cCdiouxXeEfgGaAnpsSZ] # type ) - """ # noqa + """ matches = re.findall(cfmt, value[CONF_FORMAT], flags=re.VERBOSE) if len(matches) != len(value[CONF_ARGS]): raise cv.Invalid( diff --git a/esphome/components/lvgl/__init__.py b/esphome/components/lvgl/__init__.py index 4277c14dd7..6e005f897e 100644 --- a/esphome/components/lvgl/__init__.py +++ b/esphome/components/lvgl/__init__.py @@ -1,3 +1,4 @@ +import functools import importlib from pathlib import Path import pkgutil @@ -79,7 +80,7 @@ from .schemas import ( WIDGET_TYPES, any_widget_schema, container_schema, - obj_schema, + obj_dict, ) from .styles import styles_to_code, theme_to_code from .touchscreens import touchscreen_schema, touchscreens_to_code @@ -173,7 +174,7 @@ def generate_lv_conf_h(): if clashes: LOGGER.warning( "Some defines are set both by ESPHome build flags and by LVGL configuration which may lead to unexpected behavior: %s", - sorted(list(clashes)), + sorted(clashes), ) unused_defines = all_defines - lv_defines.keys() - defines_from_flags @@ -518,16 +519,32 @@ def add_hello_world(config): return config -def _theme_schema(value): +@functools.cache +def _build_theme_schema( + widget_types: tuple[tuple[str, widgets.WidgetType], ...], +) -> cv.Schema: + # The theme schema is value-independent: it depends only on the set of + # registered widget types. Key the cache on a snapshot of WIDGET_TYPES so + # that an external component registering a new widget after the first + # validation (legal per any_widget_schema's lazy-evaluation contract) + # produces a fresh tuple, a cache miss, and a rebuilt schema -- the cache + # self-heals instead of stale-rejecting valid themes. See obj_dict() in + # schemas.py for why chained .extend() is avoided here. return cv.Schema( { cv.Optional(df.CONF_DARK_MODE, default=False): cv.boolean, **{ - cv.Optional(name): obj_schema(w).extend(FULL_STYLE_SCHEMA) - for name, w in WIDGET_TYPES.items() + cv.Optional(name): cv.Schema( + {**obj_dict(w), **FULL_STYLE_SCHEMA.schema} + ) + for name, w in widget_types }, } - )(value) + ) + + +def _theme_schema(value: dict) -> dict: + return _build_theme_schema(tuple(WIDGET_TYPES.items()))(value) FINAL_VALIDATE_SCHEMA = final_validation diff --git a/esphome/components/lvgl/defines.py b/esphome/components/lvgl/defines.py index 15a24f1ad2..d9be881a7f 100644 --- a/esphome/components/lvgl/defines.py +++ b/esphome/components/lvgl/defines.py @@ -335,7 +335,7 @@ TYPE_NONE = "none" DIRECTIONS = LvConstant("LV_DIR_", "LEFT", "RIGHT", "BOTTOM", "TOP") -LV_FONTS = list(f"montserrat_{s}" for s in range(8, 50, 2)) + [ +LV_FONTS = [f"montserrat_{s}" for s in range(8, 50, 2)] + [ "dejavu_16_persian_hebrew", "simsun_16_cjk", "unscii_8", diff --git a/esphome/components/lvgl/helpers.py b/esphome/components/lvgl/helpers.py index 6f70a1e3bd..3da8643308 100644 --- a/esphome/components/lvgl/helpers.py +++ b/esphome/components/lvgl/helpers.py @@ -6,7 +6,6 @@ from esphome.const import CONF_ARGS, CONF_FORMAT CONF_IF_NAN = "if_nan" -# noqa f_regex = re.compile( r""" ( # start of capture group 1 @@ -20,7 +19,6 @@ f_regex = re.compile( """, flags=re.VERBOSE, ) -# noqa c_regex = re.compile( r""" ( # start of capture group 1 diff --git a/esphome/components/lvgl/lv_validation.py b/esphome/components/lvgl/lv_validation.py index a1b75182eb..27cbfff694 100644 --- a/esphome/components/lvgl/lv_validation.py +++ b/esphome/components/lvgl/lv_validation.py @@ -239,7 +239,7 @@ def color_retmapper(value): else: r, g, b, _ = from_rgbw(cval) return literal(f"lv_color_make({r}, {g}, {b})") - assert False + raise AssertionError(f"Unhandled lv_color value: {value!r}") def option_string(value): diff --git a/esphome/components/lvgl/schemas.py b/esphome/components/lvgl/schemas.py index 58ef88d6a8..bdaa91f15c 100644 --- a/esphome/components/lvgl/schemas.py +++ b/esphome/components/lvgl/schemas.py @@ -22,6 +22,7 @@ from esphome.const import ( ) from esphome.core import TimePeriod from esphome.core.config import StartupTrigger +from esphome.schema_extractors import EnableSchemaExtraction from . import defines as df, lv_validation as lvalid from .defines import ( @@ -378,18 +379,63 @@ TRIGGER_EVENT_MAP = { } -def part_schema(parts): +def part_dict(parts: tuple[str, ...] | list[str]) -> dict[Any, Any]: + """ + Return the raw mapping used by part_schema, so callers can merge it into a + larger dict and avoid chained .extend() calls (each .extend() recompiles the + whole mapping, turning the build into O(N^2)). + + Invariant: the source schemas spread here (STATE_SCHEMA, FLAG_SCHEMA, the + nested STATE_SCHEMA values) must use the default extra=PREVENT_EXTRA and + required=False and must not register any add_extra/prepend_extra + validators. Reaching into .schema and rebuilding via cv.Schema(...) keeps + only the mapping; non-default extra/required and any _extra_schemas would + be silently dropped. + """ + return { + **STATE_SCHEMA.schema, + **FLAG_SCHEMA.schema, + **{cv.Optional(part): STATE_SCHEMA for part in parts}, + } + + +def part_schema(parts: tuple[str, ...] | list[str]) -> cv.Schema: """ Generate a schema for the various parts (e.g. main:, indicator:) of a widget type :param parts: The parts to include :return: The schema """ - return STATE_SCHEMA.extend(FLAG_SCHEMA).extend( - {cv.Optional(part): STATE_SCHEMA for part in parts} - ) + return cv.Schema(part_dict(parts)) -def automation_schema(typ: LvType): +def _lazy_validate_automation(extra_schema: dict) -> Callable[[Any], Any]: + """Return a validator that defers building the validate_automation schema. + + validate_automation() runs AUTOMATION_SCHEMA.extend(extra_schema), which + voluptuous compiles eagerly. automation_schema() builds ~60 of these per + widget type, and the vast majority of slots are never invoked by a given + user config. Deferring the build to first use removes that work from + schema-construction time. + + When EnableSchemaExtraction is set (build_language_schema.py), fall back + to eager construction so the @schema_extractor("automation") decoration + inside validate_automation is registered. + """ + if EnableSchemaExtraction: + return validate_automation(extra_schema) + + cached: Callable[[Any], Any] | None = None + + def validator(value: Any) -> Any: + nonlocal cached + if cached is None: + cached = validate_automation(extra_schema) + return cached(value) + + return validator + + +def automation_schema(typ: LvType) -> dict[Any, Any]: events = df.LV_EVENT_TRIGGERS + df.SWIPE_TRIGGERS if typ.has_on_value: events = events + (CONF_ON_VALUE, CONF_ON_UPDATE) @@ -404,7 +450,7 @@ def automation_schema(typ: LvType): return { **{ - cv.Optional(event): validate_automation( + cv.Optional(event): _lazy_validate_automation( { cv.GenerateID(CONF_TRIGGER_ID): cv.declare_id( Trigger.template(*get_trigger_args(event)) @@ -413,7 +459,7 @@ def automation_schema(typ: LvType): ) for event in events }, - cv.Optional(CONF_ON_BOOT): validate_automation( + cv.Optional(CONF_ON_BOOT): _lazy_validate_automation( {cv.GenerateID(CONF_TRIGGER_ID): cv.declare_id(StartupTrigger)} ), } @@ -462,23 +508,62 @@ def base_update_schema(widget_type: WidgetType | LvType, parts): return schema -def obj_schema(widget_type: WidgetType): +# Memoize obj_dict() the same way _OBJ_SCHEMA_CACHE memoizes obj_schema(). +# automation_schema(w.w_type) builds fresh Trigger.template(...) objects on +# every call, so without this cache _theme_schema pays that cost per widget +# per validation. Callers must treat the returned dict as immutable. The +# _theme_schema caller spreads it into a fresh dict, which is safe; the +# obj_schema caller passes it directly to cv.Schema(...) -- voluptuous stores +# the mapping by reference but never mutates it (.extend() copies first), so +# the alias is also safe today. Adding in-place mutation of obj_schema(w).schema +# would corrupt this cache. +_OBJ_DICT_CACHE: dict[int, tuple[WidgetType, dict[Any, Any]]] = {} + + +def obj_dict(widget_type: WidgetType) -> dict[Any, Any]: + """ + Return the raw mapping used by obj_schema, so callers can merge it into a + larger dict and avoid chained .extend() calls. + + Inherits the same source-schema invariant documented on part_dict: any + schema spread into this mapping must use the default extra=PREVENT_EXTRA + and required=False and must carry no add_extra/prepend_extra validators. + + The returned mapping is cached and must be treated as immutable by callers. + """ + cached = _OBJ_DICT_CACHE.get(id(widget_type)) + if cached is not None and cached[0] is widget_type: + return cached[1] + built = { + **part_dict(widget_type.parts), + **ALIGN_TO_SCHEMA, + **automation_schema(widget_type.w_type), + cv.Optional(CONF_STATE): SET_STATE_SCHEMA, + cv.Optional(CONF_GROUP): cv.use_id(lv_group_t), + } + _OBJ_DICT_CACHE[id(widget_type)] = (widget_type, built) + return built + + +# Widget types are module-level singletons populated at import time, so we +# can cache compiled obj_schemas by widget_type identity for the lifetime of +# the process. The strong reference in the value keeps the key (an id() +# target) from being recycled. +_OBJ_SCHEMA_CACHE: dict[int, tuple[WidgetType, cv.Schema]] = {} + + +def obj_schema(widget_type: WidgetType) -> cv.Schema: """ Create a schema for a widget type itself i.e. no allowance for children :param widget_type: :return: """ - return ( - part_schema(widget_type.parts) - .extend(ALIGN_TO_SCHEMA) - .extend(automation_schema(widget_type.w_type)) - .extend( - { - cv.Optional(CONF_STATE): SET_STATE_SCHEMA, - cv.Optional(CONF_GROUP): cv.use_id(lv_group_t), - } - ) - ) + cached = _OBJ_SCHEMA_CACHE.get(id(widget_type)) + if cached is not None and cached[0] is widget_type: + return cached[1] + schema = cv.Schema(obj_dict(widget_type)) + _OBJ_SCHEMA_CACHE[id(widget_type)] = (widget_type, schema) + return schema ALIGN_TO_SCHEMA = { diff --git a/esphome/components/lvgl/widgets/meter.py b/esphome/components/lvgl/widgets/meter.py index 62ea14bdda..e2407fad5a 100644 --- a/esphome/components/lvgl/widgets/meter.py +++ b/esphome/components/lvgl/widgets/meter.py @@ -184,6 +184,7 @@ INDICATOR_ARC_SCHEMA = cv.Schema( cv.Optional(CONF_START_VALUE): lv_float, cv.Optional(CONF_END_VALUE): lv_float, cv.Optional(CONF_OPA, default=1.0): opacity, + cv.Optional(CONF_ROUNDED, default=False): cv.boolean, } ).add_extra(cv.has_at_most_one_key(CONF_VALUE, CONF_START_VALUE)) @@ -417,7 +418,7 @@ class MeterType(WidgetType): "arc_width": v[CONF_WIDTH], "arc_color": v[CONF_COLOR], "arc_opa": v[CONF_OPA], - "arc_rounded": v.get("arc_rounded", False), + "arc_rounded": v[CONF_ROUNDED], } if CONF_R_MOD in v: get_warnings().add( diff --git a/esphome/components/lvgl/widgets/tabview.py b/esphome/components/lvgl/widgets/tabview.py index 5e9e0494dd..ee252ecf0b 100644 --- a/esphome/components/lvgl/widgets/tabview.py +++ b/esphome/components/lvgl/widgets/tabview.py @@ -97,7 +97,7 @@ class TabviewType(WidgetType): tab_bar = Widget(bar_obj, obj_spec) await set_obj_properties(tab_bar, tab_style) if tab_items_style: - for index, tab_conf in enumerate(config[CONF_TABS]): + for index, _tab_conf in enumerate(config[CONF_TABS]): await set_obj_properties( Widget(lv_obj.get_child(bar_obj, index), button_spec), tab_items_style, diff --git a/esphome/components/micro_wake_word/__init__.py b/esphome/components/micro_wake_word/__init__.py index 38926fce99..cba6bcfa50 100644 --- a/esphome/components/micro_wake_word/__init__.py +++ b/esphome/components/micro_wake_word/__init__.py @@ -7,7 +7,7 @@ from urllib.parse import urljoin from esphome import automation, external_files, git from esphome.automation import register_action, register_condition import esphome.codegen as cg -from esphome.components import esp32, microphone, ota +from esphome.components import esp32, microphone, ota, psram import esphome.config_validation as cv from esphome.const import ( CONF_FILE, @@ -20,6 +20,7 @@ from esphome.const import ( CONF_RAW_DATA_ID, CONF_REF, CONF_REFRESH, + CONF_TASK_STACK_IN_PSRAM, CONF_TYPE, CONF_URL, CONF_USERNAME, @@ -358,6 +359,7 @@ CONFIG_SCHEMA = cv.All( ), cv.Optional(CONF_VAD): _maybe_empty_vad_schema, cv.Optional(CONF_STOP_AFTER_DETECTION, default=True): cv.boolean, + cv.Optional(CONF_TASK_STACK_IN_PSRAM): psram.validate_task_stack_in_psram, cv.Optional(CONF_MODEL): cv.invalid( f"The {CONF_MODEL} parameter has moved to be a list element under the {CONF_MODELS} parameter." ), @@ -374,14 +376,14 @@ CONFIG_SCHEMA = cv.All( def _load_model_data(manifest_path: Path): - with open(manifest_path, encoding="utf-8") as f: + with manifest_path.open(encoding="utf-8") as f: manifest = json.load(f) _validate_manifest_version(manifest) model_path = manifest_path.parent / manifest[CONF_MODEL] - with open(model_path, "rb") as f: + with model_path.open("rb") as f: model = f.read() if manifest.get(KEY_VERSION) == 1: @@ -451,6 +453,10 @@ async def to_code(config): cg.add_define("USE_MICRO_WAKE_WORD") ota.request_ota_state_listeners() + if config.get(CONF_TASK_STACK_IN_PSRAM): + cg.add(var.set_task_stack_in_psram(True)) + psram.request_external_task_stack() + esp32.add_idf_component(name="espressif/esp-tflite-micro", ref="1.3.3~1") # Pin esp-nn for stable future builds (esp-tflite-micro depends on esp-nn) esp32.add_idf_component(name="espressif/esp-nn", ref="1.1.2") diff --git a/esphome/components/micro_wake_word/micro_wake_word.cpp b/esphome/components/micro_wake_word/micro_wake_word.cpp index 6877e9e5df..237d72229d 100644 --- a/esphome/components/micro_wake_word/micro_wake_word.cpp +++ b/esphome/components/micro_wake_word/micro_wake_word.cpp @@ -33,7 +33,8 @@ static const uint32_t INFERENCE_TASK_STACK_SIZE = 3072; static const UBaseType_t INFERENCE_TASK_PRIORITY = 3; enum EventGroupBits : uint32_t { - COMMAND_STOP = (1 << 0), // Signals the inference task should stop + COMMAND_STOP = (1 << 0), // Signals the inference task should stop + COMMAND_RESET_RING_BUFFER = (1 << 1), // Signals the inference task to discard buffered audio TASK_STARTING = (1 << 3), TASK_RUNNING = (1 << 4), @@ -114,13 +115,13 @@ void MicroWakeWord::setup() { } std::shared_ptr temp_ring_buffer = this->ring_buffer_.lock(); if (this->ring_buffer_.use_count() > 1) { - size_t bytes_free = temp_ring_buffer->free(); - - if (bytes_free < data.size()) { - xEventGroupSetBits(this->event_group_, EventGroupBits::WARNING_FULL_RING_BUFFER); - temp_ring_buffer->reset(); + // Producer-only write: never touches consumer state. If the buffer is full, ask the inference task + // to drain it - reset() is a consumer operation and must run on the inference task's thread. + // Disable partial writes so audio chunks are either fully accepted or rejected and handled below. + if (temp_ring_buffer->write_without_replacement(data.data(), data.size(), 0, false) == 0) { + xEventGroupSetBits(this->event_group_, + EventGroupBits::WARNING_FULL_RING_BUFFER | EventGroupBits::COMMAND_RESET_RING_BUFFER); } - temp_ring_buffer->write((void *) data.data(), data.size()); } }); @@ -146,56 +147,65 @@ void MicroWakeWord::inference_task(void *params) { { // Ensures any C++ objects fall out of scope to deallocate before deleting the task - const size_t new_bytes_to_process = - this_mww->microphone_source_->get_audio_stream_info().ms_to_bytes(this_mww->features_step_size_); - std::unique_ptr audio_buffer; + const auto &stream_info = this_mww->microphone_source_->get_audio_stream_info(); + const size_t bytes_per_frame = stream_info.frames_to_bytes(1); + const size_t max_fill_bytes = stream_info.ms_to_bytes(this_mww->features_step_size_); + std::unique_ptr audio_source; int8_t features_buffer[PREPROCESSOR_FEATURE_SIZE]; if (!(xEventGroupGetBits(this_mww->event_group_) & ERROR_BITS)) { - // Allocate audio transfer buffer - audio_buffer = audio::AudioSourceTransferBuffer::create(new_bytes_to_process); - - if (audio_buffer == nullptr) { + // Round ring buffer size down to a frame multiple so the wrap boundary never splits an int16 sample. + const size_t ring_buffer_size = + (stream_info.ms_to_bytes(RING_BUFFER_DURATION_MS) / bytes_per_frame) * bytes_per_frame; + std::shared_ptr temp_ring_buffer = ring_buffer::RingBuffer::create(ring_buffer_size); + if (temp_ring_buffer == nullptr) { xEventGroupSetBits(this_mww->event_group_, EventGroupBits::ERROR_MEMORY); + } else { + audio_source = audio::RingBufferAudioSource::create(temp_ring_buffer, max_fill_bytes, + static_cast(bytes_per_frame)); + if (audio_source == nullptr) { + xEventGroupSetBits(this_mww->event_group_, EventGroupBits::ERROR_MEMORY); + } else { + this_mww->ring_buffer_ = temp_ring_buffer; + } } } - if (!(xEventGroupGetBits(this_mww->event_group_) & ERROR_BITS)) { - // Allocate ring buffer - std::shared_ptr temp_ring_buffer = ring_buffer::RingBuffer::create( - this_mww->microphone_source_->get_audio_stream_info().ms_to_bytes(RING_BUFFER_DURATION_MS)); - if (temp_ring_buffer.use_count() == 0) { - xEventGroupSetBits(this_mww->event_group_, EventGroupBits::ERROR_MEMORY); - } - audio_buffer->set_source(temp_ring_buffer); - this_mww->ring_buffer_ = temp_ring_buffer; - } - if (!(xEventGroupGetBits(this_mww->event_group_) & ERROR_BITS)) { this_mww->microphone_source_->start(); xEventGroupSetBits(this_mww->event_group_, EventGroupBits::TASK_RUNNING); - while (!(xEventGroupGetBits(this_mww->event_group_) & COMMAND_STOP)) { - audio_buffer->transfer_data_from_source(pdMS_TO_TICKS(DATA_TIMEOUT_MS)); - - if (audio_buffer->available() < new_bytes_to_process) { - // Insufficient data to generate new spectrogram features, read more next iteration - continue; + while (!(xEventGroupGetBits(this_mww->event_group_) & (COMMAND_STOP | ERROR_BITS))) { + if (xEventGroupGetBits(this_mww->event_group_) & EventGroupBits::COMMAND_RESET_RING_BUFFER) { + // Producer asked us to drain; run the consumer-side reset from this thread. + audio_source->clear_buffered_data(); + xEventGroupClearBits(this_mww->event_group_, EventGroupBits::COMMAND_RESET_RING_BUFFER); } - // Generate new spectrogram features - uint32_t processed_samples = this_mww->generate_features_( - (int16_t *) audio_buffer->get_buffer_start(), audio_buffer->available() / sizeof(int16_t), features_buffer); - audio_buffer->decrease_buffer_length(processed_samples * sizeof(int16_t)); + audio_source->fill(pdMS_TO_TICKS(DATA_TIMEOUT_MS), false); - // Run inference using the new spectorgram features - if (!this_mww->update_model_probabilities_(features_buffer)) { - xEventGroupSetBits(this_mww->event_group_, EventGroupBits::ERROR_INFERENCE); - break; + // The frontend buffers samples internally and only emits a feature once it has a full window, so we can + // hand it whatever the source exposes. The frontend consumes at least one sample per call, so available() + // strictly decreases and this loop always terminates. + while (audio_source->available() >= sizeof(int16_t)) { + const size_t samples_available = audio_source->available() / sizeof(int16_t); + const int16_t *audio_data = reinterpret_cast(audio_source->data()); + + size_t processed_samples = 0; + const bool feature_generated = + this_mww->generate_features_(audio_data, samples_available, features_buffer, &processed_samples); + audio_source->consume(processed_samples * sizeof(int16_t)); + + if (feature_generated) { + if (!this_mww->update_model_probabilities_(features_buffer)) { + xEventGroupSetBits(this_mww->event_group_, EventGroupBits::ERROR_INFERENCE); + break; + } + + // Process each model's probabilities and possibly send a Detection Event to the queue + this_mww->process_probabilities_(); + } } - - // Process each model's probabilities and possibly send a Detection Event to the queue - this_mww->process_probabilities_(); } } } @@ -207,10 +217,7 @@ void MicroWakeWord::inference_task(void *params) { FrontendFreeStateContents(&this_mww->frontend_state_); xEventGroupSetBits(this_mww->event_group_, EventGroupBits::TASK_STOPPED); - while (true) { - // Continuously delay until the main loop deletes the task - delay(10); - } + vTaskSuspend(nullptr); // Suspend this task indefinitely until the loop method deletes it } std::vector MicroWakeWord::get_wake_words() { @@ -233,14 +240,14 @@ void MicroWakeWord::add_vad_model(const uint8_t *model_start, uint8_t probabilit #endif void MicroWakeWord::suspend_task_() { - if (this->inference_task_handle_ != nullptr) { - vTaskSuspend(this->inference_task_handle_); + if (this->inference_task_.is_created()) { + vTaskSuspend(this->inference_task_.get_handle()); } } void MicroWakeWord::resume_task_() { - if (this->inference_task_handle_ != nullptr) { - vTaskResume(this->inference_task_handle_); + if (this->inference_task_.is_created()) { + vTaskResume(this->inference_task_.get_handle()); } } @@ -282,8 +289,7 @@ void MicroWakeWord::loop() { if ((event_group_bits & EventGroupBits::TASK_STOPPED)) { ESP_LOGD(TAG, "Inference task is finished, freeing task resources"); - vTaskDelete(this->inference_task_handle_); - this->inference_task_handle_ = nullptr; + this->inference_task_.deallocate(); xEventGroupClearBits(this->event_group_, ALL_BITS); xQueueReset(this->detection_queue_); this->set_state_(State::STOPPED); @@ -301,7 +307,7 @@ void MicroWakeWord::loop() { switch (this->state_) { case State::STARTING: - if ((this->inference_task_handle_ == nullptr) && !this->status_has_error()) { + if (!this->inference_task_.is_created() && !this->status_has_error()) { // Setup preprocesor feature generator. If done in the task, it would lock the task to its initial core, as it // uses floating point operations. if (!FrontendPopulateState(&this->frontend_config_, &this->frontend_state_, @@ -310,10 +316,8 @@ void MicroWakeWord::loop() { return; } - xTaskCreate(MicroWakeWord::inference_task, "mww", INFERENCE_TASK_STACK_SIZE, (void *) this, - INFERENCE_TASK_PRIORITY, &this->inference_task_handle_); - - if (this->inference_task_handle_ == nullptr) { + if (!this->inference_task_.create(MicroWakeWord::inference_task, "mww", INFERENCE_TASK_STACK_SIZE, + (void *) this, INFERENCE_TASK_PRIORITY, this->task_stack_in_psram_)) { FrontendFreeStateContents(&this->frontend_state_); // Deallocate frontend state this->status_momentary_error("task_start", 1000); } @@ -386,11 +390,15 @@ void MicroWakeWord::set_state_(State state) { } } -size_t MicroWakeWord::generate_features_(int16_t *audio_buffer, size_t samples_available, - int8_t features_buffer[PREPROCESSOR_FEATURE_SIZE]) { - size_t processed_samples = 0; +bool MicroWakeWord::generate_features_(const int16_t *audio_buffer, size_t samples_available, + int8_t features_buffer[PREPROCESSOR_FEATURE_SIZE], size_t *processed_samples) { + *processed_samples = 0; struct FrontendOutput frontend_output = - FrontendProcessSamples(&this->frontend_state_, audio_buffer, samples_available, &processed_samples); + FrontendProcessSamples(&this->frontend_state_, audio_buffer, samples_available, processed_samples); + + if (frontend_output.size == 0) { + return false; + } for (size_t i = 0; i < frontend_output.size; ++i) { // These scaling values are set to match the TFLite audio frontend int8 output. @@ -415,7 +423,7 @@ size_t MicroWakeWord::generate_features_(int16_t *audio_buffer, size_t samples_a features_buffer[i] = static_cast(clamp(value, INT8_MIN, INT8_MAX)); } - return processed_samples; + return true; } void MicroWakeWord::process_probabilities_() { diff --git a/esphome/components/micro_wake_word/micro_wake_word.h b/esphome/components/micro_wake_word/micro_wake_word.h index 5c0c056ac0..e4c590a423 100644 --- a/esphome/components/micro_wake_word/micro_wake_word.h +++ b/esphome/components/micro_wake_word/micro_wake_word.h @@ -11,6 +11,7 @@ #include "esphome/core/automation.h" #include "esphome/core/component.h" #include "esphome/core/defines.h" +#include "esphome/core/static_task.h" #ifdef USE_OTA_STATE_LISTENER #include "esphome/components/ota/ota_backend.h" @@ -59,6 +60,8 @@ class MicroWakeWord : public Component void set_stop_after_detection(bool stop_after_detection) { this->stop_after_detection_ = stop_after_detection; } + void set_task_stack_in_psram(bool task_stack_in_psram) { this->task_stack_in_psram_ = task_stack_in_psram; } + Trigger *get_wake_word_detected_trigger() { return &this->wake_word_detected_trigger_; } void add_wake_word_model(WakeWordModel *model); @@ -93,6 +96,8 @@ class MicroWakeWord : public Component bool stop_after_detection_; + bool task_stack_in_psram_{false}; + uint8_t features_step_size_; // Audio frontend handles generating spectrogram features @@ -105,8 +110,9 @@ class MicroWakeWord : public Component // Used to send messages about the models' states to the main loop QueueHandle_t detection_queue_; + StaticTask inference_task_; + static void inference_task(void *params); - TaskHandle_t inference_task_handle_{nullptr}; /// @brief Suspends the inference task void suspend_task_(); @@ -115,13 +121,16 @@ class MicroWakeWord : public Component void set_state_(State state); - /// @brief Generates spectrogram features from an input buffer of audio samples - /// @param audio_buffer (int16_t *) Buffer containing input audio samples - /// @param samples_available (size_t) Number of samples avaiable in the input buffer - /// @param features_buffer (int8_t *) Buffer to store generated features - /// @return (size_t) Number of samples processed from the input buffer - size_t generate_features_(int16_t *audio_buffer, size_t samples_available, - int8_t features_buffer[PREPROCESSOR_FEATURE_SIZE]); + /// @brief Generates a spectrogram feature from an input buffer of audio samples. The frontend buffers samples + /// internally, so callers may stream arbitrary-sized chunks; a feature is only emitted once enough samples have + /// accumulated to fill a full analysis window. + /// @param audio_buffer (const int16_t *) Buffer containing input audio samples + /// @param samples_available (size_t) Number of samples available in the input buffer + /// @param features_buffer (int8_t *) Buffer to store the generated feature, valid only when the return value is true + /// @param processed_samples (size_t *) Set to the number of samples consumed from the input buffer + /// @return True if a new feature was generated; false if more samples are required + bool generate_features_(const int16_t *audio_buffer, size_t samples_available, + int8_t features_buffer[PREPROCESSOR_FEATURE_SIZE], size_t *processed_samples); /// @brief Processes any new probabilities for each model. If any wake word is detected, it will send a DetectionEvent /// to the detection_queue_. diff --git a/esphome/components/mitsubishi_cn105/climate.py b/esphome/components/mitsubishi_cn105/climate.py index cc44494d89..522b9218fc 100644 --- a/esphome/components/mitsubishi_cn105/climate.py +++ b/esphome/components/mitsubishi_cn105/climate.py @@ -1,8 +1,14 @@ from esphome import automation import esphome.codegen as cg from esphome.components import climate, uart +from esphome.components.climate import validate_climate_swing_mode import esphome.config_validation as cv -from esphome.const import CONF_ID, CONF_TEMPERATURE, CONF_UPDATE_INTERVAL +from esphome.const import ( + CONF_ID, + CONF_SUPPORTED_SWING_MODES, + CONF_TEMPERATURE, + CONF_UPDATE_INTERVAL, +) from esphome.core import ID from esphome.cpp_generator import MockObj from esphome.types import ConfigType, TemplateArgsType @@ -43,6 +49,9 @@ CONFIG_SCHEMA = ( cv.Optional( CONF_CURRENT_TEMPERATURE_MIN_INTERVAL, default="60s" ): cv.update_interval, + cv.Optional( + CONF_SUPPORTED_SWING_MODES, default="OFF" + ): validate_climate_swing_mode, } ) ) @@ -63,6 +72,7 @@ async def to_code(config: ConfigType) -> None: var = await climate.new_climate(config) await cg.register_component(var, config) await uart.register_uart_device(var, config) + cg.add(var.set_supported_swing_mode(config[CONF_SUPPORTED_SWING_MODES])) cg.add( var.set_current_temperature_min_interval( config[CONF_CURRENT_TEMPERATURE_MIN_INTERVAL] diff --git a/esphome/components/mitsubishi_cn105/mitsubishi_cn105.h b/esphome/components/mitsubishi_cn105/mitsubishi_cn105.h index dbeb43068e..742d8e18a9 100644 --- a/esphome/components/mitsubishi_cn105/mitsubishi_cn105.h +++ b/esphome/components/mitsubishi_cn105/mitsubishi_cn105.h @@ -1,5 +1,6 @@ #pragma once +#include #include #include "esphome/components/uart/uart.h" #include "esphome/core/finite_set_mask.h" diff --git a/esphome/components/mitsubishi_cn105/mitsubishi_cn105_climate.cpp b/esphome/components/mitsubishi_cn105/mitsubishi_cn105_climate.cpp index 67a561397a..afffe7ea5e 100644 --- a/esphome/components/mitsubishi_cn105/mitsubishi_cn105_climate.cpp +++ b/esphome/components/mitsubishi_cn105/mitsubishi_cn105_climate.cpp @@ -84,6 +84,8 @@ climate::ClimateTraits MitsubishiCN105Climate::traits() { traits.add_supported_fan_mode(p.second); } + traits.set_supported_swing_modes(this->supported_swing_modes_); + traits.set_visual_min_temperature(16.0f); traits.set_visual_max_temperature(31.0f); traits.set_visual_temperature_step(1.0f); @@ -114,6 +116,37 @@ void MitsubishiCN105Climate::control(const climate::ClimateCall &call) { this->hp_.set_fan_mode(*fan_mode); } + if (const auto swing_mode = call.get_swing_mode()) { + auto vane = this->last_non_swing_vane_mode_; + auto wide = this->last_non_swing_wide_vane_mode_; + + switch (*swing_mode) { + case climate::CLIMATE_SWING_BOTH: + vane = MitsubishiCN105::VaneMode::SWING; + wide = MitsubishiCN105::WideVaneMode::SWING; + break; + + case climate::CLIMATE_SWING_VERTICAL: + vane = MitsubishiCN105::VaneMode::SWING; + break; + + case climate::CLIMATE_SWING_HORIZONTAL: + wide = MitsubishiCN105::WideVaneMode::SWING; + break; + + case climate::CLIMATE_SWING_OFF: + default: + break; + } + + if (this->supported_swing_modes_.count(climate::CLIMATE_SWING_VERTICAL)) { + this->hp_.set_vane_mode(vane); + } + if (this->supported_swing_modes_.count(climate::CLIMATE_SWING_HORIZONTAL)) { + this->hp_.set_wide_vane_mode(wide); + } + } + if (this->hp_.is_status_initialized()) { this->apply_values_(); } @@ -143,7 +176,64 @@ void MitsubishiCN105Climate::apply_values_() { ESP_LOGD(TAG, "Unable to map fan mode"); } + if (!this->supported_swing_modes_.empty()) { + bool vertical_swinging = false; + bool horizontal_swinging = false; + + if (this->supported_swing_modes_.count(climate::CLIMATE_SWING_VERTICAL)) { + if (status.vane_mode == MitsubishiCN105::VaneMode::SWING) { + vertical_swinging = true; + } else if (status.vane_mode != MitsubishiCN105::VaneMode::UNKNOWN) { + this->last_non_swing_vane_mode_ = status.vane_mode; + } + } + + if (this->supported_swing_modes_.count(climate::CLIMATE_SWING_HORIZONTAL)) { + if (status.wide_vane_mode == MitsubishiCN105::WideVaneMode::SWING) { + horizontal_swinging = true; + } else if (status.wide_vane_mode != MitsubishiCN105::WideVaneMode::UNKNOWN) { + this->last_non_swing_wide_vane_mode_ = status.wide_vane_mode; + } + } + + if (vertical_swinging && horizontal_swinging) { + this->swing_mode = climate::CLIMATE_SWING_BOTH; + } else if (vertical_swinging) { + this->swing_mode = climate::CLIMATE_SWING_VERTICAL; + } else if (horizontal_swinging) { + this->swing_mode = climate::CLIMATE_SWING_HORIZONTAL; + } else { + this->swing_mode = climate::CLIMATE_SWING_OFF; + } + } + this->publish_state(); } +void MitsubishiCN105Climate::set_supported_swing_mode(climate::ClimateSwingMode mode) { + this->supported_swing_modes_.clear(); + switch (mode) { + case climate::CLIMATE_SWING_VERTICAL: + this->supported_swing_modes_.insert(climate::CLIMATE_SWING_OFF); + this->supported_swing_modes_.insert(climate::CLIMATE_SWING_VERTICAL); + break; + + case climate::CLIMATE_SWING_HORIZONTAL: + this->supported_swing_modes_.insert(climate::CLIMATE_SWING_OFF); + this->supported_swing_modes_.insert(climate::CLIMATE_SWING_HORIZONTAL); + break; + + case climate::CLIMATE_SWING_BOTH: + this->supported_swing_modes_.insert(climate::CLIMATE_SWING_OFF); + this->supported_swing_modes_.insert(climate::CLIMATE_SWING_VERTICAL); + this->supported_swing_modes_.insert(climate::CLIMATE_SWING_HORIZONTAL); + this->supported_swing_modes_.insert(climate::CLIMATE_SWING_BOTH); + break; + + case climate::CLIMATE_SWING_OFF: + default: + break; + } +} + } // namespace esphome::mitsubishi_cn105 diff --git a/esphome/components/mitsubishi_cn105/mitsubishi_cn105_climate.h b/esphome/components/mitsubishi_cn105/mitsubishi_cn105_climate.h index e09158bfcf..c83a5519c1 100644 --- a/esphome/components/mitsubishi_cn105/mitsubishi_cn105_climate.h +++ b/esphome/components/mitsubishi_cn105/mitsubishi_cn105_climate.h @@ -25,10 +25,15 @@ class MitsubishiCN105Climate : public climate::Climate, public Component, public void set_remote_temperature(float temperature) { this->hp_.set_remote_temperature(temperature); } void clear_remote_temperature() { this->hp_.clear_remote_temperature(); } + void set_supported_swing_mode(climate::ClimateSwingMode mode); + protected: void apply_values_(); MitsubishiCN105 hp_; + climate::ClimateSwingModeMask supported_swing_modes_{}; + MitsubishiCN105::VaneMode last_non_swing_vane_mode_{MitsubishiCN105::VaneMode::AUTO}; + MitsubishiCN105::WideVaneMode last_non_swing_wide_vane_mode_{MitsubishiCN105::WideVaneMode::CENTER}; }; template diff --git a/esphome/components/mixer/speaker/__init__.py b/esphome/components/mixer/speaker/__init__.py index 59a80d9297..47164a9997 100644 --- a/esphome/components/mixer/speaker/__init__.py +++ b/esphome/components/mixer/speaker/__init__.py @@ -1,6 +1,6 @@ from esphome import automation import esphome.codegen as cg -from esphome.components import audio, esp32, speaker +from esphome.components import audio, psram, speaker import esphome.config_validation as cv from esphome.const import ( CONF_BITS_PER_SAMPLE, @@ -44,20 +44,10 @@ SOURCE_SPEAKER_SCHEMA = speaker.SPEAKER_SCHEMA.extend( cv.positive_time_period_milliseconds, cv.one_of(CONF_NEVER, lower=True), ), - cv.Optional(CONF_BITS_PER_SAMPLE, default=16): cv.int_range(16, 16), } ) -def _set_stream_limits(config): - audio.set_stream_limits( - min_bits_per_sample=16, - max_bits_per_sample=16, - )(config) - - return config - - def _validate_source_speaker(config): fconf = fv.full_config.get() @@ -67,15 +57,25 @@ def _validate_source_speaker(config): output_speaker_id = fconf.get_config_for_path(path) config[CONF_OUTPUT_SPEAKER] = output_speaker_id + inherit_property_from(CONF_BITS_PER_SAMPLE, CONF_OUTPUT_SPEAKER)(config) inherit_property_from(CONF_NUM_CHANNELS, CONF_OUTPUT_SPEAKER)(config) inherit_property_from(CONF_SAMPLE_RATE, CONF_OUTPUT_SPEAKER)(config) + audio.final_validate_audio_schema( + "mixer", + audio_device=CONF_OUTPUT_SPEAKER, + sample_rate=config.get(CONF_SAMPLE_RATE), + )(config) + + return config + + +def _validate_output_speaker(config): audio.final_validate_audio_schema( "mixer", audio_device=CONF_OUTPUT_SPEAKER, bits_per_sample=config.get(CONF_BITS_PER_SAMPLE), channels=config.get(CONF_NUM_CHANNELS), - sample_rate=config.get(CONF_SAMPLE_RATE), )(config) return config @@ -89,24 +89,26 @@ CONFIG_SCHEMA = cv.All( cv.Required(CONF_SOURCE_SPEAKERS): cv.All( cv.ensure_list(SOURCE_SPEAKER_SCHEMA), cv.Length(min=2, max=8), - [_set_stream_limits], ), + cv.Optional(CONF_BITS_PER_SAMPLE): cv.one_of(8, 16, 24, 32, int=True), cv.Optional(CONF_NUM_CHANNELS): cv.int_range(min=1, max=2), cv.Optional(CONF_QUEUE_MODE, default=False): cv.boolean, - cv.Optional(CONF_TASK_STACK_IN_PSRAM, default=False): cv.boolean, + cv.Optional(CONF_TASK_STACK_IN_PSRAM): psram.validate_task_stack_in_psram, } ), cv.only_on([PLATFORM_ESP32]), ) FINAL_VALIDATE_SCHEMA = cv.All( + inherit_property_from(CONF_BITS_PER_SAMPLE, CONF_OUTPUT_SPEAKER), + inherit_property_from(CONF_NUM_CHANNELS, CONF_OUTPUT_SPEAKER), cv.Schema( { cv.Optional(CONF_SOURCE_SPEAKERS): [_validate_source_speaker], }, extra=cv.ALLOW_EXTRA, ), - inherit_property_from(CONF_NUM_CHANNELS, CONF_OUTPUT_SPEAKER), + _validate_output_speaker, ) @@ -116,16 +118,14 @@ async def to_code(config): spkr = await cg.get_variable(config[CONF_OUTPUT_SPEAKER]) + cg.add(var.set_output_bits_per_sample(config[CONF_BITS_PER_SAMPLE])) cg.add(var.set_output_channels(config[CONF_NUM_CHANNELS])) cg.add(var.set_output_speaker(spkr)) cg.add(var.set_queue_mode(config[CONF_QUEUE_MODE])) - if task_stack_in_psram := config.get(CONF_TASK_STACK_IN_PSRAM): - cg.add(var.set_task_stack_in_psram(task_stack_in_psram)) - if task_stack_in_psram and config[CONF_TASK_STACK_IN_PSRAM]: - esp32.add_idf_sdkconfig_option( - "CONFIG_SPIRAM_ALLOW_STACK_EXTERNAL_MEMORY", True - ) + if config.get(CONF_TASK_STACK_IN_PSRAM): + cg.add(var.set_task_stack_in_psram(True)) + psram.request_external_task_stack() # Initialize FixedVector with exact count of source speakers cg.add(var.init_source_speakers(len(config[CONF_SOURCE_SPEAKERS]))) diff --git a/esphome/components/mixer/speaker/mixer_speaker.cpp b/esphome/components/mixer/speaker/mixer_speaker.cpp index 1a995a6edf..6128dc3767 100644 --- a/esphome/components/mixer/speaker/mixer_speaker.cpp +++ b/esphome/components/mixer/speaker/mixer_speaker.cpp @@ -7,8 +7,10 @@ #include "esphome/core/helpers.h" #include "esphome/core/log.h" +#include // esp-audio-libs +#include // esp-audio-libs + #include -#include #include namespace esphome::mixer_speaker { @@ -22,19 +24,8 @@ static const uint32_t MIXER_AUTO_STOP_DEBOUNCE_MS = 200; static const size_t TASK_STACK_SIZE = 4096; -static const int16_t MAX_AUDIO_SAMPLE_VALUE = INT16_MAX; -static const int16_t MIN_AUDIO_SAMPLE_VALUE = INT16_MIN; - static const char *const TAG = "speaker_mixer"; -// Gives the Q15 fixed point scaling factor to reduce by 0 dB, 1dB, ..., 50 dB -// dB to PCM scaling factor formula: floating_point_scale_factor = 2^(-db/6.014) -// float to Q15 fixed point formula: q15_scale_factor = floating_point_scale_factor * 2^(15) -static const std::array DECIBEL_REDUCTION_TABLE = { - 32767, 29201, 26022, 23189, 20665, 18415, 16410, 14624, 13032, 11613, 10349, 9222, 8218, 7324, 6527, 5816, 5183, - 4619, 4116, 3668, 3269, 2913, 2596, 2313, 2061, 1837, 1637, 1459, 1300, 1158, 1032, 920, 820, 731, - 651, 580, 517, 461, 411, 366, 326, 291, 259, 231, 206, 183, 163, 146, 130, 116, 103}; - // Event bits for SourceSpeaker command processing enum SourceSpeakerEventBits : uint32_t { SOURCE_SPEAKER_COMMAND_START = (1 << 0), @@ -315,97 +306,17 @@ size_t SourceSpeaker::process_data_from_source(std::shared_ptraudio_stream_info_.bytes_to_samples(bytes_read); if (samples_to_duck > 0) { - int16_t *current_buffer = reinterpret_cast(audio_source->mutable_data()); - - duck_samples(current_buffer, samples_to_duck, &this->current_ducking_db_reduction_, - &this->ducking_transition_samples_remaining_, this->samples_per_ducking_step_, - this->db_change_per_ducking_step_); + esp_audio_libs::ducking::apply(audio_source->mutable_data(), + static_cast(this->audio_stream_info_.get_bits_per_sample() / 8), + samples_to_duck, this->ducking_state_); } return bytes_read; } void SourceSpeaker::apply_ducking(uint8_t decibel_reduction, uint32_t duration) { - if (this->target_ducking_db_reduction_ != decibel_reduction) { - // Start transition from the previous target (which becomes the new current level) - this->current_ducking_db_reduction_ = this->target_ducking_db_reduction_; - - this->target_ducking_db_reduction_ = decibel_reduction; - - // Calculate the number of intermediate dB steps for the transition timing. - // Subtract 1 because the first step is taken immediately after this calculation. - uint8_t total_ducking_steps = 0; - if (this->target_ducking_db_reduction_ > this->current_ducking_db_reduction_) { - // The dB reduction level is increasing (which results in quieter audio) - total_ducking_steps = this->target_ducking_db_reduction_ - this->current_ducking_db_reduction_ - 1; - this->db_change_per_ducking_step_ = 1; - } else { - // The dB reduction level is decreasing (which results in louder audio) - total_ducking_steps = this->current_ducking_db_reduction_ - this->target_ducking_db_reduction_ - 1; - this->db_change_per_ducking_step_ = -1; - } - if ((duration > 0) && (total_ducking_steps > 0)) { - this->ducking_transition_samples_remaining_ = this->audio_stream_info_.ms_to_samples(duration); - - this->samples_per_ducking_step_ = this->ducking_transition_samples_remaining_ / total_ducking_steps; - this->ducking_transition_samples_remaining_ = - this->samples_per_ducking_step_ * total_ducking_steps; // adjust for integer division rounding - - this->current_ducking_db_reduction_ += this->db_change_per_ducking_step_; - } else { - this->ducking_transition_samples_remaining_ = 0; - this->current_ducking_db_reduction_ = this->target_ducking_db_reduction_; - } - } -} - -void SourceSpeaker::duck_samples(int16_t *input_buffer, uint32_t input_samples_to_duck, - int8_t *current_ducking_db_reduction, uint32_t *ducking_transition_samples_remaining, - uint32_t samples_per_ducking_step, int8_t db_change_per_ducking_step) { - if (*ducking_transition_samples_remaining > 0) { - // Ducking level is still transitioning - - // Takes the ceiling of input_samples_to_duck/samples_per_ducking_step - uint32_t ducking_steps_in_batch = - input_samples_to_duck / samples_per_ducking_step + (input_samples_to_duck % samples_per_ducking_step != 0); - - for (uint32_t i = 0; i < ducking_steps_in_batch; ++i) { - uint32_t samples_left_in_step = *ducking_transition_samples_remaining % samples_per_ducking_step; - - if (samples_left_in_step == 0) { - samples_left_in_step = samples_per_ducking_step; - } - - uint32_t samples_to_duck = std::min(input_samples_to_duck, samples_left_in_step); - samples_to_duck = std::min(samples_to_duck, *ducking_transition_samples_remaining); - - // Ensure we only point to valid index in the Q15 scaling factor table - uint8_t safe_db_reduction_index = - clamp(*current_ducking_db_reduction, 0, DECIBEL_REDUCTION_TABLE.size() - 1); - int16_t q15_scale_factor = DECIBEL_REDUCTION_TABLE[safe_db_reduction_index]; - - audio::scale_audio_samples(input_buffer, input_buffer, q15_scale_factor, samples_to_duck); - - if (samples_left_in_step - samples_to_duck == 0) { - // After scaling the current samples, we are ready to transition to the next step - *current_ducking_db_reduction += db_change_per_ducking_step; - } - - input_buffer += samples_to_duck; - *ducking_transition_samples_remaining -= samples_to_duck; - input_samples_to_duck -= samples_to_duck; - } - } - - if ((*current_ducking_db_reduction > 0) && (input_samples_to_duck > 0)) { - // Audio is ducked, but its not in the middle of a transition step - - uint8_t safe_db_reduction_index = - clamp(*current_ducking_db_reduction, 0, DECIBEL_REDUCTION_TABLE.size() - 1); - int16_t q15_scale_factor = DECIBEL_REDUCTION_TABLE[safe_db_reduction_index]; - - audio::scale_audio_samples(input_buffer, input_buffer, q15_scale_factor, input_samples_to_duck); - } + const uint32_t transition_samples = duration > 0 ? this->audio_stream_info_.ms_to_samples(duration) : 0; + esp_audio_libs::ducking::set_target(this->ducking_state_, decibel_reduction, transition_samples); } void SourceSpeaker::enter_stopping_state_() { @@ -417,8 +328,9 @@ void SourceSpeaker::enter_stopping_state_() { void MixerSpeaker::dump_config() { ESP_LOGCONFIG(TAG, "Speaker Mixer:\n" - " Number of output channels: %u", - this->output_channels_); + " Number of output channels: %" PRIu8 "\n" + " Output bits per sample: %" PRIu8, + this->output_channels_, this->output_bits_per_sample_); } void MixerSpeaker::setup() { @@ -512,13 +424,8 @@ void MixerSpeaker::loop() { esp_err_t MixerSpeaker::start(audio::AudioStreamInfo &stream_info) { if (!this->audio_stream_info_.has_value()) { - if (stream_info.get_bits_per_sample() != 16) { - // Audio streams that don't have 16 bits per sample are not supported - return ESP_ERR_NOT_SUPPORTED; - } - - this->audio_stream_info_ = audio::AudioStreamInfo(stream_info.get_bits_per_sample(), this->output_channels_, - stream_info.get_sample_rate()); + this->audio_stream_info_ = + audio::AudioStreamInfo(this->output_bits_per_sample_, this->output_channels_, stream_info.get_sample_rate()); this->output_speaker_->set_audio_stream_info(this->audio_stream_info_.value()); } else { if (!this->queue_mode_ && (stream_info.get_sample_rate() != this->audio_stream_info_.value().get_sample_rate())) { @@ -542,57 +449,6 @@ esp_err_t MixerSpeaker::start(audio::AudioStreamInfo &stream_info) { return ESP_OK; } -void MixerSpeaker::copy_frames(const int16_t *input_buffer, audio::AudioStreamInfo input_stream_info, - int16_t *output_buffer, audio::AudioStreamInfo output_stream_info, - uint32_t frames_to_transfer) { - uint8_t input_channels = input_stream_info.get_channels(); - uint8_t output_channels = output_stream_info.get_channels(); - const uint8_t max_input_channel_index = input_channels - 1; - - if (input_channels == output_channels) { - size_t bytes_to_copy = input_stream_info.frames_to_bytes(frames_to_transfer); - memcpy(output_buffer, input_buffer, bytes_to_copy); - - return; - } - - for (uint32_t frame_index = 0; frame_index < frames_to_transfer; ++frame_index) { - for (uint8_t output_channel_index = 0; output_channel_index < output_channels; ++output_channel_index) { - uint8_t input_channel_index = std::min(output_channel_index, max_input_channel_index); - output_buffer[output_channels * frame_index + output_channel_index] = - input_buffer[input_channels * frame_index + input_channel_index]; - } - } -} - -void MixerSpeaker::mix_audio_samples(const int16_t *primary_buffer, audio::AudioStreamInfo primary_stream_info, - const int16_t *secondary_buffer, audio::AudioStreamInfo secondary_stream_info, - int16_t *output_buffer, audio::AudioStreamInfo output_stream_info, - uint32_t frames_to_mix) { - const uint8_t primary_channels = primary_stream_info.get_channels(); - const uint8_t secondary_channels = secondary_stream_info.get_channels(); - const uint8_t output_channels = output_stream_info.get_channels(); - - const uint8_t max_primary_channel_index = primary_channels - 1; - const uint8_t max_secondary_channel_index = secondary_channels - 1; - - for (uint32_t frames_index = 0; frames_index < frames_to_mix; ++frames_index) { - for (uint8_t output_channel_index = 0; output_channel_index < output_channels; ++output_channel_index) { - const uint32_t secondary_channel_index = std::min(output_channel_index, max_secondary_channel_index); - const int32_t secondary_sample = secondary_buffer[frames_index * secondary_channels + secondary_channel_index]; - - const uint32_t primary_channel_index = std::min(output_channel_index, max_primary_channel_index); - const int32_t primary_sample = - static_cast(primary_buffer[frames_index * primary_channels + primary_channel_index]); - - const int32_t added_sample = secondary_sample + primary_sample; - - output_buffer[frames_index * output_channels + output_channel_index] = - static_cast(clamp(added_sample, MIN_AUDIO_SAMPLE_VALUE, MAX_AUDIO_SAMPLE_VALUE)); - } - } -} - // NOLINTBEGIN(bugprone-unchecked-optional-access) -- audio_stream_info_ always set before this task is created void MixerSpeaker::audio_mixer_task(void *params) { MixerSpeaker *this_mixer = static_cast(params); @@ -662,6 +518,10 @@ void MixerSpeaker::audio_mixer_task(void *params) { uint32_t frames_to_mix = output_frames_free; + const audio::AudioStreamInfo &output_info = this_mixer->audio_stream_info_.value(); + const uint8_t output_bps = output_info.get_bits_per_sample() / 8; + const uint8_t output_channels = output_info.get_channels(); + if ((audio_sources_with_data.size() == 1) || this_mixer->queue_mode_) { // Only one speaker has audio data, just copy samples over @@ -669,14 +529,15 @@ void MixerSpeaker::audio_mixer_task(void *params) { if (active_stream_info.get_sample_rate() == this_mixer->output_speaker_->get_audio_stream_info().get_sample_rate()) { - // Speaker's sample rate matches the output speaker's, copy directly + // Speaker's sample rate matches the output speaker's, convert directly into the output buffer const uint32_t frames_available_in_buffer = active_stream_info.bytes_to_frames(audio_sources_with_data[0]->available()); frames_to_mix = std::min(frames_to_mix, frames_available_in_buffer); - copy_frames(reinterpret_cast(audio_sources_with_data[0]->data()), active_stream_info, - reinterpret_cast(output_transfer_buffer->get_buffer_end()), - this_mixer->audio_stream_info_.value(), frames_to_mix); + esp_audio_libs::pcm_convert::copy_frames( + audio_sources_with_data[0]->data(), output_transfer_buffer->get_buffer_end(), + static_cast(active_stream_info.get_bits_per_sample() / 8), active_stream_info.get_channels(), + output_bps, output_channels, frames_to_mix); // Set playback delay for newly contributing source if (!speakers_with_data[0]->has_contributed_.load(std::memory_order_acquire)) { @@ -690,8 +551,7 @@ void MixerSpeaker::audio_mixer_task(void *params) { audio_sources_with_data[0]->consume(active_stream_info.frames_to_bytes(frames_to_mix)); // Update output transfer buffer length and pipeline frame count - output_transfer_buffer->increase_buffer_length( - this_mixer->audio_stream_info_.value().frames_to_bytes(frames_to_mix)); + output_transfer_buffer->increase_buffer_length(output_info.frames_to_bytes(frames_to_mix)); this_mixer->frames_in_pipeline_.fetch_add(frames_to_mix, std::memory_order_release); } else { // Speaker's stream info doesn't match the output speaker's, so it's a new source speaker @@ -703,7 +563,7 @@ void MixerSpeaker::audio_mixer_task(void *params) { } else { // Speaker has finished writing the current audio, update the stream information and restart the speaker this_mixer->audio_stream_info_ = - audio::AudioStreamInfo(active_stream_info.get_bits_per_sample(), this_mixer->output_channels_, + audio::AudioStreamInfo(this_mixer->output_bits_per_sample_, this_mixer->output_channels_, active_stream_info.get_sample_rate()); this_mixer->output_speaker_->set_audio_stream_info(this_mixer->audio_stream_info_.value()); this_mixer->output_speaker_->start(); @@ -719,21 +579,22 @@ void MixerSpeaker::audio_mixer_task(void *params) { speakers_with_data[i]->get_audio_stream_info().bytes_to_frames(audio_sources_with_data[i]->available()); frames_to_mix = std::min(frames_to_mix, frames_available_in_buffer); } - const int16_t *primary_buffer = reinterpret_cast(audio_sources_with_data[0]->data()); + const uint8_t *primary_buffer = audio_sources_with_data[0]->data(); audio::AudioStreamInfo primary_stream_info = speakers_with_data[0]->get_audio_stream_info(); - // Mix two streams together + // Mix two streams together at a time, accumulating into the output buffer. for (size_t i = 1; i < audio_sources_with_data.size(); ++i) { - mix_audio_samples(primary_buffer, primary_stream_info, - reinterpret_cast(audio_sources_with_data[i]->data()), - speakers_with_data[i]->get_audio_stream_info(), - reinterpret_cast(output_transfer_buffer->get_buffer_end()), - this_mixer->audio_stream_info_.value(), frames_to_mix); + esp_audio_libs::mixer::mix_frames( + primary_buffer, static_cast(primary_stream_info.get_bits_per_sample() / 8), + primary_stream_info.get_channels(), audio_sources_with_data[i]->data(), + static_cast(speakers_with_data[i]->get_audio_stream_info().get_bits_per_sample() / 8), + speakers_with_data[i]->get_audio_stream_info().get_channels(), output_transfer_buffer->get_buffer_end(), + output_bps, output_channels, frames_to_mix); if (i != audio_sources_with_data.size() - 1) { // Need to mix more streams together, point primary buffer and stream info to the already mixed output - primary_buffer = reinterpret_cast(output_transfer_buffer->get_buffer_end()); - primary_stream_info = this_mixer->audio_stream_info_.value(); + primary_buffer = output_transfer_buffer->get_buffer_end(); + primary_stream_info = output_info; } } @@ -754,8 +615,7 @@ void MixerSpeaker::audio_mixer_task(void *params) { } // Update output transfer buffer length and pipeline frame count (once, not per source) - output_transfer_buffer->increase_buffer_length( - this_mixer->audio_stream_info_.value().frames_to_bytes(frames_to_mix)); + output_transfer_buffer->increase_buffer_length(output_info.frames_to_bytes(frames_to_mix)); this_mixer->frames_in_pipeline_.fetch_add(frames_to_mix, std::memory_order_release); } } diff --git a/esphome/components/mixer/speaker/mixer_speaker.h b/esphome/components/mixer/speaker/mixer_speaker.h index f57bead679..f1ae919b50 100644 --- a/esphome/components/mixer/speaker/mixer_speaker.h +++ b/esphome/components/mixer/speaker/mixer_speaker.h @@ -11,6 +11,8 @@ #include "esphome/core/helpers.h" #include "esphome/core/static_task.h" +#include // esp-audio-libs + #include #include @@ -22,7 +24,8 @@ namespace esphome::mixer_speaker { * - Source speaker commands are signaled via event group bits and processed in its loop function to ensure thread * safety * - Directly handles pausing at the SourceSpeaker level; pause state is not passed through to the output speaker. - * - Audio sent to the SourceSpeaker must have 16 bits per sample. + * - Audio sent to the SourceSpeaker can have 8, 16, 24, or 32 bits per sample. Each source is converted to the output + * speaker's bit depth as it is mixed (or copied) into the output buffer. * - Audio sent to the SourceSpeaker can have any number of channels. They are duplicated or ignored as needed to match * the number of channels required for the output speaker. * - In queue mode, the audio sent to the SourceSpeakers can have different sample rates. @@ -93,19 +96,6 @@ class SourceSpeaker : public speaker::Speaker, public Component { void enter_stopping_state_(); void send_command_(uint32_t command_bit, bool wake_loop = false); - /// @brief Ducks audio samples by a specified amount. When changing the ducking amount, it can transition gradually - /// over a specified amount of samples. - /// @param input_buffer buffer with audio samples to be ducked in place - /// @param input_samples_to_duck number of samples to process in ``input_buffer`` - /// @param current_ducking_db_reduction pointer to the current dB reduction - /// @param ducking_transition_samples_remaining pointer to the total number of samples left before the - /// transition is finished - /// @param samples_per_ducking_step total number of samples per ducking step for the transition - /// @param db_change_per_ducking_step the change in dB reduction per step - static void duck_samples(int16_t *input_buffer, uint32_t input_samples_to_duck, int8_t *current_ducking_db_reduction, - uint32_t *ducking_transition_samples_remaining, uint32_t samples_per_ducking_step, - int8_t db_change_per_ducking_step); - MixerSpeaker *parent_; std::shared_ptr audio_source_; @@ -118,11 +108,7 @@ class SourceSpeaker : public speaker::Speaker, public Component { bool pause_state_{false}; - int8_t target_ducking_db_reduction_{0}; - int8_t current_ducking_db_reduction_{0}; - int8_t db_change_per_ducking_step_{1}; - uint32_t ducking_transition_samples_remaining_{0}; - uint32_t samples_per_ducking_step_{0}; + esp_audio_libs::ducking::DuckingState ducking_state_{}; std::atomic pending_playback_frames_{0}; std::atomic playback_delay_frames_{0}; // Frames in output pipeline when this source started contributing @@ -143,12 +129,14 @@ class MixerSpeaker : public Component { /// @brief Starts the mixer task. Called by a source speaker giving the current audio stream information /// @param stream_info The calling source speaker's audio stream information - /// @return ESP_ERR_NOT_SUPPORTED if the incoming stream is incompatible due to unsupported bits per sample - /// ESP_ERR_INVALID_ARG if the incoming stream is incompatible to be mixed with the other input audio stream + /// @return ESP_ERR_INVALID_ARG if the incoming stream is incompatible to be mixed with the other input audio stream /// ESP_OK if the incoming stream is compatible and the mixer task starts esp_err_t start(audio::AudioStreamInfo &stream_info); void set_output_channels(uint8_t output_channels) { this->output_channels_ = output_channels; } + void set_output_bits_per_sample(uint8_t output_bits_per_sample) { + this->output_bits_per_sample_ = output_bits_per_sample; + } void set_output_speaker(speaker::Speaker *speaker) { this->output_speaker_ = speaker; } void set_queue_mode(bool queue_mode) { this->queue_mode_ = queue_mode; } void set_task_stack_in_psram(bool task_stack_in_psram) { this->task_stack_in_psram_ = task_stack_in_psram; } @@ -159,33 +147,6 @@ class MixerSpeaker : public Component { uint32_t get_frames_in_pipeline() const { return this->frames_in_pipeline_.load(std::memory_order_acquire); } protected: - /// @brief Copies audio frames from the input buffer to the output buffer taking into account the number of channels - /// in each stream. If the output stream has more channels, the input samples are duplicated. If the output stream has - /// less channels, the extra channel input samples are dropped. - /// @param input_buffer - /// @param input_stream_info - /// @param output_buffer - /// @param output_stream_info - /// @param frames_to_transfer number of frames (consisting of a sample for each channel) to copy from the input buffer - static void copy_frames(const int16_t *input_buffer, audio::AudioStreamInfo input_stream_info, int16_t *output_buffer, - audio::AudioStreamInfo output_stream_info, uint32_t frames_to_transfer); - - /// @brief Mixes the primary and secondary streams taking into account the number of channels in each stream. Primary - /// and secondary samples are duplicated or dropped as necessary to ensure the output stream has the configured number - /// of channels. Output samples are clamped to the corresponding int16 min or max values if the mixed sample - /// overflows. - /// @param primary_buffer samples buffer for the primary stream - /// @param primary_stream_info stream info for the primary stream - /// @param secondary_buffer samples buffer for secondary stream - /// @param secondary_stream_info stream info for the secondary stream - /// @param output_buffer buffer for the mixed samples - /// @param output_stream_info stream info for the output buffer - /// @param frames_to_mix number of frames in the primary and secondary buffers to mix together - static void mix_audio_samples(const int16_t *primary_buffer, audio::AudioStreamInfo primary_stream_info, - const int16_t *secondary_buffer, audio::AudioStreamInfo secondary_stream_info, - int16_t *output_buffer, audio::AudioStreamInfo output_stream_info, - uint32_t frames_to_mix); - static void audio_mixer_task(void *params); EventGroupHandle_t event_group_{nullptr}; @@ -193,6 +154,7 @@ class MixerSpeaker : public Component { FixedVector source_speakers_; speaker::Speaker *output_speaker_{nullptr}; + uint8_t output_bits_per_sample_; uint8_t output_channels_; bool queue_mode_; bool task_stack_in_psram_{false}; diff --git a/esphome/components/mqtt/__init__.py b/esphome/components/mqtt/__init__.py index cb6b9d144f..86bba11a60 100644 --- a/esphome/components/mqtt/__init__.py +++ b/esphome/components/mqtt/__init__.py @@ -232,7 +232,7 @@ CONFIG_SCHEMA = cv.All( cv.Optional(CONF_ENABLE_ON_BOOT, default=True): cv.boolean, cv.Optional(CONF_PORT, default=1883): cv.port, cv.Optional(CONF_USERNAME, default=""): cv.string, - cv.Optional(CONF_PASSWORD, default=""): cv.string, + cv.Optional(CONF_PASSWORD, default=""): cv.sensitive(), cv.Optional(CONF_CLEAN_SESSION, default=False): cv.boolean, cv.Optional(CONF_CLIENT_ID): cv.string, cv.SplitDefault(CONF_IDF_SEND_ASYNC, esp32=False): cv.All( diff --git a/esphome/components/msa3xx/binary_sensor.py b/esphome/components/msa3xx/binary_sensor.py index 793d5190af..732a0ed291 100644 --- a/esphome/components/msa3xx/binary_sensor.py +++ b/esphome/components/msa3xx/binary_sensor.py @@ -26,7 +26,7 @@ CONFIG_SCHEMA = MSA_SENSOR_SCHEMA.extend( ), key=CONF_NAME, ) - for event, icon in zip(EVENT_SENSORS, ICONS) + for event, icon in zip(EVENT_SENSORS, ICONS, strict=True) } ) diff --git a/esphome/components/neopixelbus/light.py b/esphome/components/neopixelbus/light.py index 943fd141f6..2e18688af0 100644 --- a/esphome/components/neopixelbus/light.py +++ b/esphome/components/neopixelbus/light.py @@ -1,3 +1,5 @@ +import logging + from esphome import pins import esphome.codegen as cg from esphome.components import light @@ -22,6 +24,7 @@ from esphome.const import ( Framework, ) from esphome.core import CORE +from esphome.types import ConfigType from ._methods import ( METHOD_BIT_BANG, @@ -34,6 +37,8 @@ from ._methods import ( ) from .const import CHIP_TYPES, CONF_ASYNC, CONF_BUS, ONE_WIRE_CHIPS +_LOGGER = logging.getLogger(__name__) + neopixelbus_ns = cg.esphome_ns.namespace("neopixelbus") NeoPixelBusLightOutputBase = neopixelbus_ns.class_( "NeoPixelBusLightOutputBase", light.AddressableLight @@ -134,6 +139,17 @@ def _validate(config): return config +def _warn_esp32_deprecated(config: ConfigType) -> ConfigType: + if CORE.is_esp32: + _LOGGER.warning( + "'neopixelbus' on ESP32 is deprecated. The upstream library " + "(makuna/NeoPixelBus) is no longer actively maintained. Migrate " + "to 'esp32_rmt_led_strip'. Removal is targeted for 2027.1 but " + "may happen sooner once ESPHome moves to ESP-IDF 6." + ) + return config + + def _validate_method(value): if value is None: # default method is determined afterwards because it depends on the chip type chosen @@ -195,6 +211,7 @@ CONFIG_SCHEMA = cv.All( ).extend(cv.COMPONENT_SCHEMA), _choose_default_method, _validate, + _warn_esp32_deprecated, ) diff --git a/esphome/components/network/__init__.py b/esphome/components/network/__init__.py index 811e7c875a..2818b8c93e 100644 --- a/esphome/components/network/__init__.py +++ b/esphome/components/network/__init__.py @@ -5,8 +5,9 @@ import esphome.codegen as cg from esphome.components.esp32 import add_idf_sdkconfig_option from esphome.components.psram import is_guaranteed as psram_is_guaranteed import esphome.config_validation as cv -from esphome.const import CONF_ENABLE_IPV6, CONF_MIN_IPV6_ADDR_COUNT +from esphome.const import CONF_ENABLE_IPV6, CONF_ID, CONF_MIN_IPV6_ADDR_COUNT from esphome.core import CORE, CoroPriority, coroutine_with_priority +from esphome.types import ConfigType CODEOWNERS = ["@esphome/core"] AUTO_LOAD = ["mdns"] @@ -19,6 +20,7 @@ KEY_HIGH_PERFORMANCE_NETWORKING = "high_performance_networking" CONF_ENABLE_HIGH_PERFORMANCE = "enable_high_performance" network_ns = cg.esphome_ns.namespace("network") +NetworkComponent = network_ns.class_("NetworkComponent", cg.Component) IPAddress = network_ns.class_("IPAddress") @@ -107,6 +109,7 @@ def has_high_performance_networking() -> bool: CONFIG_SCHEMA = cv.Schema( { + cv.GenerateID(): cv.declare_id(NetworkComponent), cv.SplitDefault( CONF_ENABLE_IPV6, bk72xx=False, @@ -224,3 +227,15 @@ async def to_code(config): cg.add_build_flag("-DPIO_FRAMEWORK_ARDUINO_LWIP2_IPV6_LOW_MEMORY") if CORE.is_rp2040: cg.add_build_flag("-DPIO_FRAMEWORK_ARDUINO_ENABLE_IPV6") + # Pvariable creation lives in a separate coroutine at NETWORK_SERVICES so it + # emits after wifi/ethernet at COMMUNICATION. This keeps compile-time config + # (above) separate from C++ object lifecycle and allows wiring in interface + # pointers via get_variable(). + if CORE.is_esp32: + CORE.add_job(network_component_to_code, config) + + +@coroutine_with_priority(CoroPriority.NETWORK_SERVICES) +async def network_component_to_code(config: ConfigType) -> None: + var = cg.new_Pvariable(config[CONF_ID]) + await cg.register_component(var, config) diff --git a/esphome/components/network/network_component.cpp b/esphome/components/network/network_component.cpp new file mode 100644 index 0000000000..40cf64906c --- /dev/null +++ b/esphome/components/network/network_component.cpp @@ -0,0 +1,33 @@ +#include "network_component.h" + +#include "esphome/core/defines.h" +#if defined(USE_NETWORK) && defined(USE_ESP32) +#include "esphome/core/log.h" +#include "esp_err.h" +#include "esp_netif.h" +#include "esp_event.h" +namespace esphome::network { + +static const char *const TAG = "network"; + +void NetworkComponent::setup() { + // Initialize ESP-IDF network interfaces and ensure the default event loop exists + esp_err_t err; + err = esp_netif_init(); + if (err != ESP_OK) { + ESP_LOGE(TAG, "esp_netif_init failed: (%d) %s", err, esp_err_to_name(err)); + this->mark_failed(); + return; + } + err = esp_event_loop_create_default(); + // ESP_ERR_INVALID_STATE is returned if the default loop already exists, + // which is fine since we just want to make sure it exists + if (err != ESP_OK && err != ESP_ERR_INVALID_STATE) { + ESP_LOGE(TAG, "esp_event_loop_create_default failed: (%d) %s", err, esp_err_to_name(err)); + this->mark_failed(); + return; + } +} + +} // namespace esphome::network +#endif diff --git a/esphome/components/network/network_component.h b/esphome/components/network/network_component.h new file mode 100644 index 0000000000..dde15940e4 --- /dev/null +++ b/esphome/components/network/network_component.h @@ -0,0 +1,14 @@ +#pragma once +#include "esphome/core/defines.h" +#if defined(USE_NETWORK) && defined(USE_ESP32) +#include "esphome/core/component.h" + +namespace esphome::network { +class NetworkComponent : public Component { + public: + void setup() override; + // AFTER_BLUETOOTH: BLE controller must initialize before esp_netif_init per IDF guidance. + float get_setup_priority() const override { return setup_priority::AFTER_BLUETOOTH; } +}; +} // namespace esphome::network +#endif diff --git a/esphome/components/nfc/nfc.cpp b/esphome/components/nfc/nfc.cpp index 99e476dbdf..76a391f1de 100644 --- a/esphome/components/nfc/nfc.cpp +++ b/esphome/components/nfc/nfc.cpp @@ -15,17 +15,6 @@ char *format_bytes_to(char *buffer, std::span bytes) { return format_hex_pretty_to(buffer, FORMAT_BYTES_BUFFER_SIZE, bytes.data(), bytes.size(), ' '); } -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wdeprecated-declarations" -// Deprecated wrappers intentionally use heap-allocating version for backward compatibility -std::string format_uid(std::span uid) { - return format_hex_pretty(uid.data(), uid.size(), '-', false); // NOLINT -} -std::string format_bytes(std::span bytes) { - return format_hex_pretty(bytes.data(), bytes.size(), ' ', false); // NOLINT -} -#pragma GCC diagnostic pop - uint8_t guess_tag_type(uint8_t uid_length) { if (uid_length == 4) { return TAG_TYPE_MIFARE_CLASSIC; diff --git a/esphome/components/nfc/nfc.h b/esphome/components/nfc/nfc.h index 42ef993913..36b27ce5f6 100644 --- a/esphome/components/nfc/nfc.h +++ b/esphome/components/nfc/nfc.h @@ -63,13 +63,6 @@ static constexpr size_t FORMAT_BYTES_BUFFER_SIZE = 192; /// Format bytes to buffer with ' ' separator (e.g., "04 11 22 33"). Returns buffer for inline use. char *format_bytes_to(char *buffer, std::span bytes); -// Remove before 2026.6.0 -ESPDEPRECATED("Use format_uid_to() with stack buffer instead. Removed in 2026.6.0", "2025.12.0") -std::string format_uid(std::span uid); -// Remove before 2026.6.0 -ESPDEPRECATED("Use format_bytes_to() with stack buffer instead. Removed in 2026.6.0", "2025.12.0") -std::string format_bytes(std::span bytes); - uint8_t guess_tag_type(uint8_t uid_length); int8_t get_mifare_classic_ndef_start_index(std::vector &data); bool decode_mifare_classic_tlv(std::vector &data, uint32_t &message_length, uint8_t &message_start_index); diff --git a/esphome/components/nrf52/__init__.py b/esphome/components/nrf52/__init__.py index 2aba208af7..4ba1ab5d4d 100644 --- a/esphome/components/nrf52/__init__.py +++ b/esphome/components/nrf52/__init__.py @@ -65,7 +65,7 @@ from .const import ( ) # force import gpio to register pin schema -from .gpio import nrf52_pin_to_code # noqa +from .gpio import nrf52_pin_to_code # noqa: F401 CODEOWNERS = ["@tomaszduda23"] AUTO_LOAD = ["zephyr", "preferences"] diff --git a/esphome/components/nrf52/ota.py b/esphome/components/nrf52/ota.py index eb1caa5595..5d608acbac 100644 --- a/esphome/components/nrf52/ota.py +++ b/esphome/components/nrf52/ota.py @@ -139,7 +139,7 @@ async def _smpmgr_upload_connected( already_uploaded = True if not already_uploaded: - with open(firmware, "rb") as file: + with firmware.open("rb") as file: image = file.read() upload_size = len(image) progress = ProgressBar("Uploading") diff --git a/esphome/components/opentherm/generate.py b/esphome/components/opentherm/generate.py index 0b39895798..1c0de329e5 100644 --- a/esphome/components/opentherm/generate.py +++ b/esphome/components/opentherm/generate.py @@ -16,7 +16,7 @@ def define_has_component(component_type: str, keys: list[str]) -> None: cg.add_define( f"OPENTHERM_{component_type.upper()}_LIST(F, sep)", cg.RawExpression( - " sep ".join(map(lambda key: f"F({key}_{component_type.lower()})", keys)) + " sep ".join(f"F({key}_{component_type.lower()})" for key in keys) ), ) for key in keys: @@ -30,12 +30,8 @@ def define_has_settings(keys: list[str], schemas: dict[str, SettingSchema]) -> N "OPENTHERM_SETTING_LIST(F, sep)", cg.RawExpression( " sep ".join( - map( - lambda key: ( - f"F({schemas[key].backing_type}, {key}_setting, {schemas[key].default_value})" - ), - keys, - ) + f"F({schemas[key].backing_type}, {key}_setting, {schemas[key].default_value})" + for key in keys ) ), ) diff --git a/esphome/components/opentherm/output/__init__.py b/esphome/components/opentherm/output/__init__.py index 87307eb051..68977b9e34 100644 --- a/esphome/components/opentherm/output/__init__.py +++ b/esphome/components/opentherm/output/__init__.py @@ -21,7 +21,7 @@ async def new_openthermoutput( var = cg.new_Pvariable(config[CONF_ID]) await cg.register_component(var, config) await output.register_output(var, config) - cg.add(getattr(var, "set_id")(cg.RawExpression(f'"{key}_{config[CONF_ID]}"'))) + cg.add(var.set_id(cg.RawExpression(f'"{key}_{config[CONF_ID]}"'))) input.generate_setters(var, config) return var diff --git a/esphome/components/openthread/openthread_esp.cpp b/esphome/components/openthread/openthread_esp.cpp index 27712bd86a..787f2f5de8 100644 --- a/esphome/components/openthread/openthread_esp.cpp +++ b/esphome/components/openthread/openthread_esp.cpp @@ -35,9 +35,8 @@ void OpenThreadComponent::setup() { esp_vfs_eventfd_config_t eventfd_config = { .max_fds = 3, }; + // Network interface setup handled by network component ESP_ERROR_CHECK(nvs_flash_init()); - ESP_ERROR_CHECK(esp_event_loop_create_default()); - ESP_ERROR_CHECK(esp_netif_init()); ESP_ERROR_CHECK(esp_vfs_eventfd_register(&eventfd_config)); xTaskCreate( diff --git a/esphome/components/packages/__init__.py b/esphome/components/packages/__init__.py index 06a64208b6..c1c5bd2ae3 100644 --- a/esphome/components/packages/__init__.py +++ b/esphome/components/packages/__init__.py @@ -112,7 +112,7 @@ def expand_file_to_files(config: dict): def validate_yaml_filename(value): value = cv.string(value) - if not (value.endswith(".yaml") or value.endswith(".yml")): + if not value.endswith((".yaml", ".yml")): raise cv.Invalid("Only YAML (.yaml / .yml) files are supported.") return value @@ -215,7 +215,7 @@ def _process_remote_package(config: dict[str, Any]) -> dict[str, Any]: If loading fails after cloning, attempts a revert and retry in case a prior cached checkout is stale. """ - repo_dir, revert = git.clone_or_update( + repo_root, revert = git.clone_or_update( url=config[CONF_URL], ref=config.get(CONF_REF), refresh=config[CONF_REFRESH], @@ -225,6 +225,10 @@ def _process_remote_package(config: dict[str, Any]) -> dict[str, Any]: ) files: list[dict[str, Any]] = [] + # ``repo_root`` is the directory containing ``.git`` and must be passed + # to git for symlink-stub resolution. ``repo_dir`` may be narrowed to a + # subdirectory via the user's CONF_PATH and is used for file lookups. + repo_dir = repo_root if base_path := config.get(CONF_PATH): repo_dir = repo_dir / base_path @@ -236,13 +240,37 @@ def _process_remote_package(config: dict[str, Any]) -> dict[str, Any]: def _load_package_yaml(yaml_file: Path, filename: str) -> dict: """Load a YAML file from a remote package, validating min_version.""" - try: - new_yaml = yaml_util.load_yaml(yaml_file) - except EsphomeError as e: + + def _load(path: Path) -> dict | str | None: + try: + return yaml_util.load_yaml(path) + except EsphomeError as e: + raise cv.Invalid( + f"{filename} is not a valid YAML file." + f" Please check the file contents.\n{e}" + ) from e + + new_yaml = _load(yaml_file) + if not isinstance(new_yaml, dict): + # On Windows, git defaults to core.symlinks=false unless the user + # has Developer Mode enabled or is running elevated. Files stored + # in the repo as symlinks (tree mode 120000) are then checked out + # as plain text files containing the symlink target path, so + # parsing them as YAML yields a bare scalar instead of a mapping. + # Best-effort: follow the symlink target ourselves and re-load. + target = git.resolve_symlink_stub(repo_root, yaml_file) + if target is not None: + new_yaml = _load(target) + if not isinstance(new_yaml, dict): raise cv.Invalid( - f"{filename} is not a valid YAML file." - f" Please check the file contents.\n{e}" - ) from e + f"{filename} does not contain a YAML mapping at the top level " + f"(got {type(new_yaml).__name__}). " + f"If this file is a git symlink in the source repository, it " + f"may not have been materialized correctly on your platform " + f"(this is a known issue with git on Windows without Developer " + f"Mode enabled). Try pointing your package at the real file " + f"path instead." + ) esphome_config = new_yaml.get(CONF_ESPHOME) or {} min_version = esphome_config.get(CONF_MIN_VERSION) if min_version is not None and cv.Version.parse(min_version) > cv.Version.parse( diff --git a/esphome/components/psram/__init__.py b/esphome/components/psram/__init__.py index 86c17ce9ca..d36d900997 100644 --- a/esphome/components/psram/__init__.py +++ b/esphome/components/psram/__init__.py @@ -1,5 +1,6 @@ import logging import textwrap +from typing import Any import esphome.codegen as cg from esphome.components.const import CONF_IGNORE_NOT_FOUND @@ -94,6 +95,27 @@ def is_guaranteed() -> bool: return CORE.data.get(KEY_PSRAM_GUARANTEED, False) +def request_external_task_stack() -> None: + """Allow FreeRTOS task stacks to be allocated in external RAM (PSRAM). + + Components that expose a ``task_stack_in_psram`` option should call this from their + ``to_code`` when the option is enabled. The sdkconfig option only permits external + stacks; it does not move any stack into PSRAM on its own, so it stays opt-in per task. + """ + add_idf_sdkconfig_option("CONFIG_SPIRAM_ALLOW_STACK_EXTERNAL_MEMORY", True) + + +def validate_task_stack_in_psram(value: Any) -> bool: + """Validate a ``task_stack_in_psram`` boolean, requiring the psram component only when enabled. + + Validating the boolean first means an explicit ``false`` does not pull in the psram + requirement, so the option can still be set to false on devices without PSRAM. + """ + if value := cv.boolean(value): + return cv.requires_component(DOMAIN)(value) + return value + + def validate_psram_mode(config): esp32_config = fv.full_config.get()[PLATFORM_ESP32] if config[CONF_SPEED] == "120MHZ": diff --git a/esphome/components/resampler/speaker/__init__.py b/esphome/components/resampler/speaker/__init__.py index 3134cf7646..8a13110631 100644 --- a/esphome/components/resampler/speaker/__init__.py +++ b/esphome/components/resampler/speaker/__init__.py @@ -1,5 +1,5 @@ import esphome.codegen as cg -from esphome.components import audio, esp32, speaker +from esphome.components import audio, psram, speaker import esphome.config_validation as cv from esphome.const import ( CONF_BITS_PER_SAMPLE, @@ -63,7 +63,7 @@ CONFIG_SCHEMA = cv.All( cv.Optional( CONF_BUFFER_DURATION, default="100ms" ): cv.positive_time_period_milliseconds, - cv.Optional(CONF_TASK_STACK_IN_PSRAM, default=False): cv.boolean, + cv.Optional(CONF_TASK_STACK_IN_PSRAM): psram.validate_task_stack_in_psram, cv.Optional(CONF_FILTERS, default=16): cv.int_range(min=2, max=1024), cv.Optional(CONF_TAPS, default=16): _validate_taps, } @@ -88,9 +88,7 @@ async def to_code(config): if config.get(CONF_TASK_STACK_IN_PSRAM): cg.add(var.set_task_stack_in_psram(True)) - esp32.add_idf_sdkconfig_option( - "CONFIG_SPIRAM_ALLOW_STACK_EXTERNAL_MEMORY", True - ) + psram.request_external_task_stack() cg.add(var.set_target_bits_per_sample(config[CONF_BITS_PER_SAMPLE])) cg.add(var.set_target_sample_rate(config[CONF_SAMPLE_RATE])) diff --git a/esphome/components/router/__init__.py b/esphome/components/router/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/esphome/components/router/speaker/__init__.py b/esphome/components/router/speaker/__init__.py new file mode 100644 index 0000000000..2b2dc56433 --- /dev/null +++ b/esphome/components/router/speaker/__init__.py @@ -0,0 +1,123 @@ +from esphome import automation, core +import esphome.codegen as cg +from esphome.components import audio, speaker +import esphome.config_validation as cv +from esphome.const import ( + CONF_BITS_PER_SAMPLE, + CONF_ID, + CONF_NUM_CHANNELS, + CONF_OUTPUT_SPEAKER, + CONF_SAMPLE_RATE, +) +from esphome.core import ID +from esphome.cpp_generator import MockObj +from esphome.types import ConfigType, TemplateArgsType + +CODEOWNERS = ["@kahrendt"] + +CONF_OUTPUT_SPEAKERS = "output_speakers" +CONF_TARGET_SPEAKER = "target_speaker" + +router_ns = cg.esphome_ns.namespace("router") +Router = router_ns.class_("Router", cg.Component, speaker.Speaker) +SwitchOutputAction = router_ns.class_("SwitchOutputAction", automation.Action) + +SpeakerPtr = speaker.Speaker.operator("ptr") + + +def _set_stream_limits(config: ConfigType) -> ConfigType: + # Lock the router's stream limits to the user-declared format. Limits are set + # at CONFIG_SCHEMA time so they're visible to other components' FINAL_VALIDATE + # (which has no guaranteed ordering vs. ours). + audio.set_stream_limits( + min_bits_per_sample=config[CONF_BITS_PER_SAMPLE], + max_bits_per_sample=config[CONF_BITS_PER_SAMPLE], + min_channels=config[CONF_NUM_CHANNELS], + max_channels=config[CONF_NUM_CHANNELS], + min_sample_rate=config[CONF_SAMPLE_RATE], + max_sample_rate=config[CONF_SAMPLE_RATE], + )(config) + return config + + +CONFIG_SCHEMA = cv.All( + cv.Schema( + { + cv.GenerateID(): cv.declare_id(Router), + cv.Required(CONF_OUTPUT_SPEAKERS): cv.All( + cv.ensure_list(cv.use_id(speaker.Speaker)), + cv.Length(min=2, max=8), + ), + # All outputs must agree on a single format so the producer can keep + # streaming through a switch without reconfiguring. These are required + # rather than inherited because downstream components (e.g. mixer) + # read them from the router's declaration during FINAL_VALIDATE, + # which can't depend on our FINAL_VALIDATE running first. + cv.Required(CONF_BITS_PER_SAMPLE): cv.int_range(8, 32), + cv.Required(CONF_NUM_CHANNELS): cv.int_range(1, 2), + cv.Required(CONF_SAMPLE_RATE): cv.int_range(8000, 96000), + } + ).extend(cv.COMPONENT_SCHEMA), + cv.only_on_esp32, + _set_stream_limits, +) + + +def _final_validate(config: ConfigType) -> ConfigType: + # Validate every configured output speaker can accept the router's format. + # Switching to an output that can't reproduce the format the producer is + # already sending would otherwise fail silently at runtime. + for spk_id in config[CONF_OUTPUT_SPEAKERS]: + proxy = {**config, CONF_OUTPUT_SPEAKER: spk_id} + audio.final_validate_audio_schema( + "router", + audio_device=CONF_OUTPUT_SPEAKER, + bits_per_sample=config[CONF_BITS_PER_SAMPLE], + channels=config[CONF_NUM_CHANNELS], + sample_rate=config[CONF_SAMPLE_RATE], + )(proxy) + return config + + +FINAL_VALIDATE_SCHEMA = _final_validate + + +async def to_code(config: ConfigType) -> None: + var = cg.new_Pvariable(config[CONF_ID]) + await cg.register_component(var, config) + + # The first configured output is the default active output on boot. + speakers = config[CONF_OUTPUT_SPEAKERS] + cg.add(var.set_output_count(len(speakers))) + for spk_id in speakers: + spk = await cg.get_variable(spk_id) + cg.add(var.add_output(spk)) + + +@automation.register_action( + "router.speaker.switch_output", + SwitchOutputAction, + cv.Schema( + { + cv.GenerateID(CONF_ID): cv.use_id(Router), + cv.Required(CONF_TARGET_SPEAKER): cv.templatable( + cv.use_id(speaker.Speaker) + ), + } + ), + synchronous=True, +) +async def switch_output_to_code( + config: ConfigType, + action_id: ID, + template_arg: cg.TemplateArguments, + args: TemplateArgsType, +) -> MockObj: + parent = await cg.get_variable(config[CONF_ID]) + var = cg.new_Pvariable(action_id, template_arg, parent) + target = config[CONF_TARGET_SPEAKER] + if not isinstance(target, core.Lambda): + target = await cg.get_variable(target) + template_ = await cg.templatable(target, args, SpeakerPtr) + cg.add(var.set_target(template_)) + return var diff --git a/esphome/components/router/speaker/router_speaker.cpp b/esphome/components/router/speaker/router_speaker.cpp new file mode 100644 index 0000000000..f4bf7420ab --- /dev/null +++ b/esphome/components/router/speaker/router_speaker.cpp @@ -0,0 +1,236 @@ +#include "router_speaker.h" + +#ifdef USE_ESP32 + +#include "esphome/core/log.h" + +#include "esp_timer.h" + +#include + +namespace esphome::router { + +static const char *const TAG = "router.speaker"; + +static inline uint32_t atomic_subtract_clamped(std::atomic &var, uint32_t amount) { + uint32_t current = var.load(std::memory_order_acquire); + uint32_t subtracted = 0; + if (current > 0) { + uint32_t new_value; + do { + subtracted = std::min(amount, current); + new_value = current - subtracted; + } while (!var.compare_exchange_weak(current, new_value, std::memory_order_release, std::memory_order_acquire)); + } + return subtracted; +} + +void Router::setup() { + // Register a callback on every configured output. Each lambda captures its own + // index and only forwards when that output is the active one. This is required + // because CallbackManager has no remove() API. + for (size_t i = 0; i < this->outputs_.size(); i++) { + this->outputs_[i]->add_audio_output_callback([this, i](uint32_t frames, int64_t timestamp_us) { + // Always suppress the draining previous output during a switch, even if it's + // also the reselected active output (switching back to the bus holder). + // loop() fires one synthetic credit for its in-flight frames instead. + if (this->pending_start_prev_idx_.load(std::memory_order_relaxed) == static_cast(i)) { + return; + } + if (this->active_output_idx_.load(std::memory_order_relaxed) != static_cast(i)) { + return; + } + atomic_subtract_clamped(this->frames_in_pipeline_, frames); + this->audio_output_callback_.call(frames, timestamp_us); + }); + } +} + +void Router::loop() { + speaker::Speaker *active = this->get_active_output(); + + // Mid-switch: the new output's start() is deferred until the previous output + // fully releases shared hardware (e.g. a single i2s_audio bus driving two + // speakers). Starting earlier produces "Parent bus is busy" retries. The + // synthetic-credit callback is also deferred until prev is fully stopped, so + // that once its task has drained no natural callbacks can race ours. + const int8_t pending_prev_idx = this->pending_start_prev_idx_.load(std::memory_order_relaxed); + if (pending_prev_idx >= 0) { + speaker::Speaker *prev = this->outputs_[pending_prev_idx]; + if (prev->is_stopped()) { + this->pending_start_prev_idx_.store(-1, std::memory_order_relaxed); + + // Credit any frames left in prev's ring buffer / DMA so producer frame + // accounting (SpeakerSourceMediaPlayer pending_frames, sendspin/AEC + // clocks) clears cleanly. The leftover audio is intentionally dropped and + // the producer is told it played "now", giving a clean discontinuity that + // keeps frame accounting consistent across the switch. + const uint32_t in_flight = this->frames_in_pipeline_.exchange(0, std::memory_order_acq_rel); + if (in_flight > 0) { + this->audio_output_callback_.call(in_flight, esp_timer_get_time()); + } + + this->apply_cached_state_to_active_(); + this->state_ = speaker::STATE_STARTING; + active->start(); + } + return; + } + + // Mirror the active output's running/stopped state into our own state_ so that + // is_running() / is_stopped() stay accurate from the producer's perspective. + // Also catch the active output self-stopping (e.g. i2s_audio silence timeout): + // without this, our state_ would stay RUNNING forever and the next play() would + // skip start(). The output retains its own volume/mute across a restart (and we + // forward those live regardless), but stream info arrives via the non-virtual + // set_audio_stream_info() and never reaches the output on its own; if the format + // changed while stopped, only start()'s apply_cached_state_to_active_() pushes it + // down before the output's play()-side auto-start locks in the stale format. + if (active->is_stopped()) { + this->state_ = speaker::STATE_STOPPED; + } else if (this->state_ == speaker::STATE_STARTING && active->is_running()) { + this->state_ = speaker::STATE_RUNNING; + } +} + +void Router::dump_config() { + ESP_LOGCONFIG(TAG, + "Router Speaker:\n" + " Outputs: %u", + static_cast(this->outputs_.size())); +} + +size_t Router::play(const uint8_t *data, size_t length, TickType_t ticks_to_wait) { + speaker::Speaker *active = this->get_active_output(); + + // Drop frames during a mid-switch until the old output releases shared hardware; + // forwarding now would trigger the new output's play()-side auto-start while + // the bus is still busy. + if (this->pending_start_prev_idx_.load(std::memory_order_relaxed) >= 0) { + vTaskDelay(ticks_to_wait); + return 0; + } + + // Producers (e.g. mixer) set stream info on us and then drive play() from a + // task without ever calling our start(). i2s_audio's play() auto-starts the + // underlying driver, so we must push our cached stream info to the active + // output before that auto-start, or it locks to its default (16k mono). + if (this->state_ == speaker::STATE_STOPPED) { + this->start(); + vTaskDelay(ticks_to_wait); + ticks_to_wait = 0; + } + + size_t written = active->play(data, length, ticks_to_wait); + if (written > 0) { + const uint32_t frames = this->audio_stream_info_.bytes_to_frames(written); + this->frames_in_pipeline_.fetch_add(frames, std::memory_order_release); + } + return written; +} + +void Router::start() { + this->frames_in_pipeline_.store(0, std::memory_order_release); + this->apply_cached_state_to_active_(); + this->state_ = speaker::STATE_STARTING; + this->get_active_output()->start(); +} + +void Router::stop() { + // Cancel any pending mid-switch start; the producer wants us stopped. + this->pending_start_prev_idx_.store(-1, std::memory_order_relaxed); + this->state_ = speaker::STATE_STOPPING; + this->get_active_output()->stop(); +} + +void Router::finish() { + this->pending_start_prev_idx_.store(-1, std::memory_order_relaxed); + this->state_ = speaker::STATE_STOPPING; + this->get_active_output()->finish(); +} + +bool Router::has_buffered_data() const { return this->get_active_output()->has_buffered_data(); } + +void Router::set_pause_state(bool pause_state) { + this->cached_pause_ = pause_state; + this->get_active_output()->set_pause_state(pause_state); +} + +void Router::set_volume(float volume) { + this->volume_ = volume; + this->get_active_output()->set_volume(volume); +} + +void Router::set_mute_state(bool mute_state) { + this->mute_state_ = mute_state; + this->get_active_output()->set_mute_state(mute_state); +} + +bool Router::switch_to_output(speaker::Speaker *target) { + if (target == nullptr) { + return false; + } + + int8_t new_idx = -1; + for (size_t i = 0; i < this->outputs_.size(); i++) { + if (this->outputs_[i] == target) { + new_idx = static_cast(i); + break; + } + } + if (new_idx < 0) { + ESP_LOGW(TAG, "Switch target is not a configured output"); + return false; + } + if (new_idx == this->active_output_idx_.load(std::memory_order_relaxed)) { + return true; + } + + // A switch is already in flight: pending_start_prev_idx_ is still releasing the + // shared bus and the current active output's start() is still deferred (it never + // started). Just redirect which output we start once the bus frees. Leave the bus + // holder (pending_start_prev_idx_), the in-flight frame counter (loop() still owes one + // synthetic credit for the bus holder's in-flight frames), and state_ alone, and + // don't stop the current active output, which never started. + if (this->pending_start_prev_idx_.load(std::memory_order_relaxed) >= 0) { + this->active_output_idx_.store(new_idx, std::memory_order_relaxed); + return true; + } + + const bool was_active = (this->state_ == speaker::STATE_STARTING || this->state_ == speaker::STATE_RUNNING); + const int8_t old_idx = this->active_output_idx_.load(std::memory_order_relaxed); + + if (was_active) { + this->outputs_[old_idx]->stop(); + } + + this->active_output_idx_.store(new_idx, std::memory_order_relaxed); + + if (was_active) { + // Defer start and the synthetic-credit callback until the old output's + // task is fully stopped; loop() handles both. Firing the synthetic credit + // here would race the old task's still-in-flight natural callbacks, + // dispatching audio_output_callback_ concurrently from two threads, which + // some consumers (e.g. sendspin's progress sync) aren't reentrant-safe for. + // STATE_STOPPING keeps producers from observing a transient stopped state + // and lets our play() short-circuit so the new output's play() doesn't + // auto-start it while the shared bus is still being released. + this->state_ = speaker::STATE_STOPPING; + this->pending_start_prev_idx_.store(old_idx, std::memory_order_relaxed); + } else { + this->frames_in_pipeline_.store(0, std::memory_order_release); + } + return true; +} + +void Router::apply_cached_state_to_active_() { + speaker::Speaker *active = this->get_active_output(); + active->set_audio_stream_info(this->audio_stream_info_); + active->set_volume(this->volume_); + active->set_mute_state(this->mute_state_); + active->set_pause_state(this->cached_pause_); +} + +} // namespace esphome::router + +#endif // USE_ESP32 diff --git a/esphome/components/router/speaker/router_speaker.h b/esphome/components/router/speaker/router_speaker.h new file mode 100644 index 0000000000..13b58a1c72 --- /dev/null +++ b/esphome/components/router/speaker/router_speaker.h @@ -0,0 +1,92 @@ +#pragma once + +#ifdef USE_ESP32 + +#include "esphome/components/speaker/speaker.h" +#include "esphome/core/automation.h" +#include "esphome/core/component.h" +#include "esphome/core/helpers.h" + +#include + +#include + +namespace esphome::router { + +class Router : public Component, public speaker::Speaker { + public: + float get_setup_priority() const override { return setup_priority::DATA; } + + void setup() override; + void loop() override; + void dump_config() override; + + size_t play(const uint8_t *data, size_t length) override { return this->play(data, length, 0); } + size_t play(const uint8_t *data, size_t length, TickType_t ticks_to_wait) override; + + void start() override; + void stop() override; + void finish() override; + + bool has_buffered_data() const override; + + void set_pause_state(bool pause_state) override; + bool get_pause_state() const override { return this->cached_pause_; } + + void set_volume(float volume) override; + float get_volume() override { return this->volume_; } + + void set_mute_state(bool mute_state) override; + bool get_mute_state() override { return this->mute_state_; } + + // Allocate the output list to its final size. Must be called before add_output(). + void set_output_count(size_t count) { this->outputs_.init(count); } + void add_output(speaker::Speaker *spk) { this->outputs_.push_back(spk); } + + /// Switch the active output to the given speaker. Must be one of the configured outputs. + /// Returns false if `target` is not in the output list. + bool switch_to_output(speaker::Speaker *target); + + // Always valid: active_output_idx_ stays within [0, outputs_.size()) and at least + // two outputs are required (validated in Python), so this never returns null. + speaker::Speaker *get_active_output() const { + return this->outputs_[this->active_output_idx_.load(std::memory_order_relaxed)]; + } + + protected: + // Frames written to the active output but not yet played: incremented in play() and decremented + // (clamped at zero) by the active output's audio_output_callback. Mirrors mixer_speaker's + // frames_in_pipeline_. + std::atomic frames_in_pipeline_{0}; + + bool cached_pause_{false}; + + void apply_cached_state_to_active_(); + + // Index of the previously-active output we're waiting on to fully stop before + // starting the new one. -1 means no pending start. Set by switch_to_output() + // when switching mid-playback; cleared by loop() once the old output reports + // is_stopped(). Required because shared-bus drivers (e.g. two i2s_audio + // speakers on one i2s_bus) reject start() until the previous user releases. + std::atomic pending_start_prev_idx_{-1}; + + private: + FixedVector outputs_; + // Index into outputs_, always within [0, outputs_.size()). Defaults to the first + // configured output; updated by switch_to_output(). + std::atomic active_output_idx_{0}; +}; + +template class SwitchOutputAction : public Action { + public: + explicit SwitchOutputAction(Router *parent) : parent_(parent) {} + TEMPLATABLE_VALUE(speaker::Speaker *, target) + void play(const Ts &...x) override { this->parent_->switch_to_output(this->target_.value(x...)); } + + protected: + Router *parent_; +}; + +} // namespace esphome::router + +#endif // USE_ESP32 diff --git a/esphome/components/rp2040/__init__.py b/esphome/components/rp2040/__init__.py index fbeac907a2..85d8ec123f 100644 --- a/esphome/components/rp2040/__init__.py +++ b/esphome/components/rp2040/__init__.py @@ -1,8 +1,10 @@ +from collections.abc import Callable import logging from pathlib import Path import re from string import ascii_letters, digits import subprocess +from typing import Any import esphome.codegen as cg import esphome.config_validation as cv @@ -11,6 +13,7 @@ from esphome.const import ( CONF_FRAMEWORK, CONF_PLATFORM_VERSION, CONF_SOURCE, + CONF_VARIANT, CONF_VERSION, CONF_WATCHDOG_TIMEOUT, KEY_CORE, @@ -20,22 +23,33 @@ from esphome.const import ( PLATFORM_RP2040, ThreadModel, ) -from esphome.core import CORE, CoroPriority, EsphomeError, coroutine_with_priority +from esphome.core import ( + CORE, + CoroPriority, + EsphomeCore, + EsphomeError, + coroutine_with_priority, +) from esphome.core.config import BOARD_MAX_LENGTH from esphome.helpers import copy_file_if_changed, read_file, write_file_if_changed +from esphome.types import ConfigType from . import boards from .const import ( - CONF_ENABLE_FULL_PRINTF, KEY_BOARD, KEY_LWIP_OPTS, KEY_PIO_FILES, KEY_RP2040, + KEY_VARIANT, + MCU_TO_VARIANT, + STANDARD_BOARDS, + VARIANT_FRIENDLY, + VARIANTS, rp2040_ns, ) # force import gpio to register pin schema -from .gpio import rp2040_pin_to_code # noqa +from .gpio import rp2040_pin_to_code # noqa: F401 _LOGGER = logging.getLogger(__name__) CODEOWNERS = ["@jesserockz"] @@ -74,7 +88,7 @@ def board_id_has_wifi(board_id: str) -> bool: return board_info.get("wifi", False) -def set_core_data(config): +def set_core_data(config: ConfigType) -> ConfigType: CORE.data[KEY_RP2040] = {} CORE.data[KEY_CORE][KEY_TARGET_PLATFORM] = PLATFORM_RP2040 CORE.data[KEY_CORE][KEY_TARGET_FRAMEWORK] = "arduino" @@ -82,12 +96,46 @@ def set_core_data(config): config[CONF_FRAMEWORK][CONF_VERSION] ) CORE.data[KEY_RP2040][KEY_BOARD] = config[CONF_BOARD] + CORE.data[KEY_RP2040][KEY_VARIANT] = config[CONF_VARIANT] CORE.data[KEY_RP2040][KEY_PIO_FILES] = {} return config +def get_rp2040_variant(core_obj: EsphomeCore | None = None) -> str: + return (core_obj or CORE).data[KEY_RP2040][KEY_VARIANT] + + +def only_on_variant( + *, + supported: str | list[str] | None = None, + unsupported: str | list[str] | None = None, + msg_prefix: str = "This feature", +) -> Callable[[Any], Any]: + """Config validator for features only available on some RP2040 variants.""" + if supported is not None and not isinstance(supported, list): + supported = [supported] + if unsupported is not None and not isinstance(unsupported, list): + unsupported = [unsupported] + + def validator_(obj: Any) -> Any: + if not CORE.is_rp2040: + raise cv.Invalid(f"{msg_prefix} is only available on RP2040") + variant = get_rp2040_variant() + if supported is not None and variant not in supported: + raise cv.Invalid( + f"{msg_prefix} is only available on {', '.join(supported)}" + ) + if unsupported is not None and variant in unsupported: + raise cv.Invalid( + f"{msg_prefix} is not available on {', '.join(unsupported)}" + ) + return obj + + return validator_ + + def get_download_types(storage_json): """Binary-download entries for a built RP2040 firmware. @@ -198,12 +246,52 @@ ARDUINO_FRAMEWORK_SCHEMA = cv.All( _arduino_check_versions, ) + +def _detect_variant(value: ConfigType) -> ConfigType: + value = value.copy() + board: str | None = value.get(CONF_BOARD) + variant: str | None = value.get(CONF_VARIANT) + + if board is None: + # `cv.has_at_least_one_key` guarantees variant is set here. + board = STANDARD_BOARDS[variant] + value[CONF_BOARD] = board + + board_info = boards.BOARDS.get(board) + if board_info is None: + if variant is None: + raise cv.Invalid( + "This board is unknown; please specify the chip variant using " + f"the '{CONF_VARIANT}' option.", + path=[CONF_BOARD], + ) + _LOGGER.warning( + "This board is unknown; the specified variant '%s' will be used " + "but this may not work as expected.", + variant, + ) + else: + board_variant = MCU_TO_VARIANT[board_info["mcu"]] + if variant is None: + variant = board_variant + elif variant != board_variant: + raise cv.Invalid( + f"Option '{CONF_VARIANT}' ({variant}) does not match the " + f"selected board '{board}' ({board_variant}).", + path=[CONF_VARIANT], + ) + + value[CONF_VARIANT] = variant + return value + + CONFIG_SCHEMA = cv.All( cv.Schema( { - cv.Required(CONF_BOARD): cv.All( + cv.Optional(CONF_BOARD): cv.All( cv.string_strict, cv.ByteLength(max=BOARD_MAX_LENGTH) ), + cv.Optional(CONF_VARIANT): cv.one_of(*VARIANTS, upper=True), cv.Optional(CONF_FRAMEWORK, default={}): ARDUINO_FRAMEWORK_SCHEMA, cv.Optional(CONF_WATCHDOG_TIMEOUT, default="8388ms"): cv.All( cv.positive_time_period_milliseconds, @@ -212,6 +300,8 @@ CONFIG_SCHEMA = cv.All( cv.Optional(CONF_ENABLE_FULL_PRINTF, default=False): cv.boolean, } ), + cv.has_at_least_one_key(CONF_BOARD, CONF_VARIANT), + _detect_variant, set_core_data, ) @@ -229,7 +319,9 @@ async def to_code(config): cg.add_define("USE_NATIVE_64BIT_TIME") cg.set_cpp_standard("gnu++20") cg.add_define("ESPHOME_BOARD", config[CONF_BOARD]) - cg.add_define("ESPHOME_VARIANT", "RP2040") + variant = config[CONF_VARIANT] + cg.add_build_flag(f"-DUSE_RP2040_VARIANT_{variant}") + cg.add_define("ESPHOME_VARIANT", VARIANT_FRIENDLY[variant]) cg.add_define(ThreadModel.SINGLE) cg.add_platformio_option("extra_scripts", ["post:post_build.py"]) diff --git a/esphome/components/rp2040/const.py b/esphome/components/rp2040/const.py index a7953834e9..817e9136c9 100644 --- a/esphome/components/rp2040/const.py +++ b/esphome/components/rp2040/const.py @@ -5,5 +5,31 @@ KEY_BOARD = "board" KEY_LWIP_OPTS = "lwip_opts" KEY_RP2040 = "rp2040" KEY_PIO_FILES = "pio_files" +KEY_VARIANT = "variant" + +VARIANT_RP2040 = "RP2040" +VARIANT_RP2350 = "RP2350" +VARIANTS = [ + VARIANT_RP2040, + VARIANT_RP2350, +] + +VARIANT_FRIENDLY = { + VARIANT_RP2040: "RP2040", + VARIANT_RP2350: "RP2350", +} + +# Map BOARDS[board]["mcu"] (lowercase) to canonical variant constant +MCU_TO_VARIANT = { + "rp2040": VARIANT_RP2040, + "rp2350": VARIANT_RP2350, +} + +# Default board chosen when only `variant` is specified — the Raspberry Pi +# Foundation reference boards (Pico W / Pico 2 W). +STANDARD_BOARDS = { + VARIANT_RP2040: "rpipicow", + VARIANT_RP2350: "rpipico2w", +} rp2040_ns = cg.esphome_ns.namespace("rp2040") diff --git a/esphome/components/rp2040/generate_boards.py b/esphome/components/rp2040/generate_boards.py index 8af261396c..b1a0b17ca3 100644 --- a/esphome/components/rp2040/generate_boards.py +++ b/esphome/components/rp2040/generate_boards.py @@ -67,7 +67,7 @@ def load_boards(arduino_pico_path: Path) -> tuple[dict, dict]: for json_file in sorted(json_dir.glob("*.json")): board_name = json_file.stem - with open(json_file, encoding="utf-8") as f: + with json_file.open(encoding="utf-8") as f: data = json.load(f) build = data.get("build", {}) @@ -136,7 +136,7 @@ def _get_variant(json_file: Path) -> str | None: """Get variant name from a board JSON file.""" if not json_file.exists(): return None - with open(json_file, encoding="utf-8") as f: + with json_file.open(encoding="utf-8") as f: data = json.load(f) return data.get("build", {}).get("variant") diff --git a/esphome/components/sendspin/__init__.py b/esphome/components/sendspin/__init__.py index b670bd3c4d..e8c643f9b9 100644 --- a/esphome/components/sendspin/__init__.py +++ b/esphome/components/sendspin/__init__.py @@ -121,13 +121,6 @@ def register_player_config(config: ConfigType) -> None: data.player_config = config -def _validate_task_stack_in_psram(value): - value = cv.boolean(value) - if value: - return cv.requires_component(psram.DOMAIN)(value) - return value - - def _request_high_performance_networking(config: ConfigType) -> ConfigType: """Request high performance networking for Sendspin streaming. @@ -152,7 +145,7 @@ CONFIG_SCHEMA = cv.All( cv.Schema( { cv.GenerateID(): cv.declare_id(SendspinHub), - cv.Optional(CONF_TASK_STACK_IN_PSRAM): _validate_task_stack_in_psram, + cv.Optional(CONF_TASK_STACK_IN_PSRAM): psram.validate_task_stack_in_psram, } ), cv.only_on_esp32, @@ -201,9 +194,7 @@ async def to_code(config: ConfigType) -> None: if config.get(CONF_TASK_STACK_IN_PSRAM): cg.add(var.set_task_stack_in_psram(True)) - esp32.add_idf_sdkconfig_option( - "CONFIG_SPIRAM_ALLOW_STACK_EXTERNAL_MEMORY", True - ) + psram.request_external_task_stack() # sendspin-cpp library esp32.add_idf_component(name="sendspin/sendspin-cpp", ref="0.6.1") @@ -261,9 +252,7 @@ async def to_code(config: ConfigType) -> None: psram_stack = player_cfg.get(CONF_TASK_STACK_IN_PSRAM, False) if psram_stack: - esp32.add_idf_sdkconfig_option( - "CONFIG_SPIRAM_ALLOW_STACK_EXTERNAL_MEMORY", True - ) + psram.request_external_task_stack() # Library defaults: priority 18 (one above httpd_priority 17 so the decoder is not # starved by the HTTP server during the initial encoded-audio burst at stream start), diff --git a/esphome/components/sendspin/media_source/__init__.py b/esphome/components/sendspin/media_source/__init__.py index f689ab01cb..6af244d41f 100644 --- a/esphome/components/sendspin/media_source/__init__.py +++ b/esphome/components/sendspin/media_source/__init__.py @@ -1,6 +1,6 @@ from esphome import automation import esphome.codegen as cg -from esphome.components import media_source +from esphome.components import media_source, psram import esphome.config_validation as cv from esphome.const import ( CONF_BUFFER_SIZE, @@ -19,7 +19,6 @@ from .. import ( CONF_SENDSPIN_ID, MEMORY_LOCATIONS, SendspinHub, - _validate_task_stack_in_psram, register_player_config, request_controller_support, sendspin_ns, @@ -71,7 +70,7 @@ CONFIG_SCHEMA = cv.All( ).extend( { cv.GenerateID(CONF_SENDSPIN_ID): cv.use_id(SendspinHub), - cv.Optional(CONF_TASK_STACK_IN_PSRAM): _validate_task_stack_in_psram, + cv.Optional(CONF_TASK_STACK_IN_PSRAM): psram.validate_task_stack_in_psram, cv.Optional(CONF_BUFFER_SIZE, default=1000000): cv.int_range(min=25000), cv.Optional(CONF_INITIAL_STATIC_DELAY, default="0ms"): cv.All( cv.positive_time_period_milliseconds, diff --git a/esphome/components/sensor/__init__.py b/esphome/components/sensor/__init__.py index 6bbab76363..5a2ebf03c0 100644 --- a/esphome/components/sensor/__init__.py +++ b/esphome/components/sensor/__init__.py @@ -1192,7 +1192,7 @@ def _std(x): def _correlation_coeff(x, y): m_x, m_y = _mean(x), _mean(y) - s_xy = sum((x_ - m_x) * (y_ - m_y) for x_, y_ in zip(x, y)) + s_xy = sum((x_ - m_x) * (y_ - m_y) for x_, y_ in zip(x, y, strict=True)) s_sq_x = sum((x_ - m_x) ** 2 for x_ in x) s_sq_y = sum((y_ - m_y) ** 2 for y_ in y) return s_xy / math.sqrt(s_sq_x * s_sq_y) @@ -1228,7 +1228,7 @@ def _mat_copy(m): def _mat_transpose(m): - return _mat_copy(zip(*m)) + return _mat_copy(zip(*m, strict=True)) def _mat_identity(n): @@ -1237,7 +1237,10 @@ def _mat_identity(n): def _mat_dot(a, b): b_t = _mat_transpose(b) - return [[sum(x * y for x, y in zip(row_a, col_b)) for col_b in b_t] for row_a in a] + return [ + [sum(x * y for x, y in zip(row_a, col_b, strict=True)) for col_b in b_t] + for row_a in a + ] def _mat_inverse(m): diff --git a/esphome/components/speaker/media_player/__init__.py b/esphome/components/speaker/media_player/__init__.py index 094043c292..90eb19d73d 100644 --- a/esphome/components/speaker/media_player/__init__.py +++ b/esphome/components/speaker/media_player/__init__.py @@ -7,7 +7,6 @@ import esphome.codegen as cg from esphome.components import ( audio, audio_file, - esp32, media_player, network, ota, @@ -155,9 +154,7 @@ CONFIG_SCHEMA = cv.All( # Remove before 2026.10.0 cv.Optional(CONF_CODEC_SUPPORT_ENABLED): cv.Any(cv.boolean, cv.string), cv.Optional(CONF_FILES): audio_file.audio_files_schema(), - cv.Optional(CONF_TASK_STACK_IN_PSRAM): cv.All( - cv.boolean, cv.requires_component(psram.DOMAIN) - ), + cv.Optional(CONF_TASK_STACK_IN_PSRAM): psram.validate_task_stack_in_psram, cv.Optional(CONF_VOLUME_INCREMENT, default=0.05): cv.percentage, cv.Optional(CONF_VOLUME_INITIAL, default=0.5): cv.percentage, cv.Optional(CONF_VOLUME_MAX, default=1.0): cv.percentage, @@ -198,9 +195,7 @@ async def to_code(config): if config.get(CONF_TASK_STACK_IN_PSRAM): cg.add(var.set_task_stack_in_psram(True)) - esp32.add_idf_sdkconfig_option( - "CONFIG_SPIRAM_ALLOW_STACK_EXTERNAL_MEMORY", True - ) + psram.request_external_task_stack() cg.add(var.set_volume_increment(config[CONF_VOLUME_INCREMENT])) cg.add(var.set_volume_initial(config[CONF_VOLUME_INITIAL])) diff --git a/esphome/components/text_sensor/text_sensor.cpp b/esphome/components/text_sensor/text_sensor.cpp index 31543117b8..d2483619a6 100644 --- a/esphome/components/text_sensor/text_sensor.cpp +++ b/esphome/components/text_sensor/text_sensor.cpp @@ -39,16 +39,13 @@ void TextSensor::publish_state(const char *state, size_t len) { #ifdef USE_TEXT_SENSOR_FILTER } else { // Has filters: need separate raw storage -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wdeprecated-declarations" // Only assign if changed to avoid heap allocation - if (len != this->raw_state.size() || memcmp(state, this->raw_state.data(), len) != 0) { - this->raw_state.assign(state, len); + if (len != this->raw_state_.size() || memcmp(state, this->raw_state_.data(), len) != 0) { + this->raw_state_.assign(state, len); } - this->raw_callback_.call(this->raw_state); - ESP_LOGV(TAG, "'%s': Received new state %s", this->name_.c_str(), this->raw_state.c_str()); - this->filter_list_->input(this->raw_state); -#pragma GCC diagnostic pop + this->raw_callback_.call(this->raw_state_); + ESP_LOGV(TAG, "'%s': Received new state %s", this->name_.c_str(), this->raw_state_.c_str()); + this->filter_list_->input(this->raw_state_); } #endif } @@ -89,11 +86,7 @@ const std::string &TextSensor::get_state() const { return this->state; } const std::string &TextSensor::get_raw_state() const { #ifdef USE_TEXT_SENSOR_FILTER if (this->filter_list_ != nullptr) { - // Suppress deprecation warning - get_raw_state() is the replacement API -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wdeprecated-declarations" - return this->raw_state; -#pragma GCC diagnostic pop + return this->raw_state_; } #endif return this->state; // No filters, raw == filtered diff --git a/esphome/components/text_sensor/text_sensor.h b/esphome/components/text_sensor/text_sensor.h index 3f69e91c8d..aa48781f41 100644 --- a/esphome/components/text_sensor/text_sensor.h +++ b/esphome/components/text_sensor/text_sensor.h @@ -29,19 +29,12 @@ class TextSensor : public EntityBase { public: std::string state; -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wdeprecated-declarations" - /// @deprecated Use get_raw_state() instead. This member will be removed in ESPHome 2026.6.0. - ESPDEPRECATED("Use get_raw_state() instead of .raw_state. Will be removed in 2026.6.0", "2025.12.0") - std::string raw_state; - TextSensor() = default; ~TextSensor() = default; -#pragma GCC diagnostic pop /// Getter-syntax for .state. const std::string &get_state() const; - /// Getter-syntax for .raw_state + /// Returns the raw (pre-filter) state. const std::string &get_raw_state() const; void publish_state(const std::string &state); @@ -84,6 +77,7 @@ class TextSensor : public EntityBase { /// Notify frontend that state has changed (assumes this->state is already set) void notify_frontend_(); #ifdef USE_TEXT_SENSOR_FILTER + std::string raw_state_; ///< Backing storage for the raw (pre-filter) value. Only used when a filter is attached. LazyCallbackManager raw_callback_; ///< Storage for raw state callbacks. #endif LazyCallbackManager callback_; ///< Storage for filtered state callbacks. diff --git a/esphome/components/time/__init__.py b/esphome/components/time/__init__.py index 067a10898c..b3bf2d44d7 100644 --- a/esphome/components/time/__init__.py +++ b/esphome/components/time/__init__.py @@ -30,13 +30,21 @@ from esphome.const import ( CONF_SECONDS, CONF_TIMEZONE, CONF_TRIGGER_ID, + PLATFORM_BK72XX, + PLATFORM_ESP32, + PLATFORM_ESP8266, + PLATFORM_HOST, + PLATFORM_LN882X, + PLATFORM_RP2040, + PLATFORM_RTL87XX, ) -from esphome.core import CORE, CoroPriority, coroutine_with_priority +from esphome.core import CORE, CoroPriority, EsphomeError, coroutine_with_priority _LOGGER = logging.getLogger(__name__) CODEOWNERS = ["@esphome/core"] IS_PLATFORM_COMPONENT = True +DOMAIN = "time" time_ns = cg.esphome_ns.namespace("time") RealTimeClock = time_ns.class_("RealTimeClock", cg.PollingComponent) @@ -88,24 +96,38 @@ def _extract_tz_string(tzfile: bytes) -> str: return tzfile.split(b"\n")[-2].decode() except (IndexError, UnicodeDecodeError): _LOGGER.error("Could not determine TZ string. Please report this issue.") - _LOGGER.error("tzfile contents: %s", tzfile, exc_info=True) + _LOGGER.exception("tzfile contents: %s", tzfile) raise -def detect_tz() -> str: +def detect_tz() -> str | None: + if CORE.target_platform not in { + PLATFORM_ESP8266, + PLATFORM_ESP32, + PLATFORM_RP2040, + PLATFORM_BK72XX, + PLATFORM_RTL87XX, + PLATFORM_LN882X, + PLATFORM_HOST, + }: + return None + # Avoids duplicate logger messages when multiple time components are configured + if cached := CORE.data.setdefault(DOMAIN, {}).get(CONF_TIMEZONE): + return cached iana_key = tzlocal.get_localzone_name() if iana_key is None: - raise cv.Invalid( + raise EsphomeError( "Could not automatically determine timezone, please set timezone manually." ) - _LOGGER.info("Detected timezone '%s'", iana_key) tzfile = _load_tzdata(iana_key) if tzfile is None: - raise cv.Invalid( + raise EsphomeError( "Could not automatically determine timezone, please set timezone manually." ) ret = _extract_tz_string(tzfile) + _LOGGER.info("Detected timezone '%s'", iana_key) _LOGGER.debug(" -> TZ string %s", ret) + CORE.data.setdefault(DOMAIN, {})[CONF_TIMEZONE] = ret return ret @@ -182,7 +204,7 @@ def cron_expression_validator(name, min_value, max_value, special_mapping=None): raise cv.Invalid( f"{name} {v} is out of range (min={min_value} max={max_value})." ) - return list(sorted(value)) + return sorted(value) value = cv.string(value) values = set() for part in value.split(","): @@ -312,16 +334,7 @@ def validate_tz(value: str) -> str: TIME_SCHEMA = cv.Schema( { - cv.SplitDefault( - CONF_TIMEZONE, - esp8266=detect_tz, - esp32=detect_tz, - rp2040=detect_tz, - bk72xx=detect_tz, - rtl87xx=detect_tz, - ln882x=detect_tz, - host=detect_tz, - ): cv.All( + cv.Optional(CONF_TIMEZONE): cv.All( cv.only_with_framework(["arduino", "esp-idf", "host"]), validate_tz, ), @@ -384,26 +397,32 @@ def _emit_parsed_timezone_fields(parsed): async def setup_time_core_(time_var, config): - if timezone := config.get(CONF_TIMEZONE): + timezone = config.get(CONF_TIMEZONE) + # an empty timezone is treated as disabling timezones completely as before + if timezone is None: + timezone = detect_tz() + if timezone: cg.add_define("USE_TIME_TIMEZONE") if CORE.is_host: # Host platform needs setenv("TZ")/tzset() for libc compatibility - cg.add(cg.RawExpression(f'setenv("TZ", "{timezone}", 1)')) - cg.add(cg.RawExpression("tzset()")) - - # Pre-parse at codegen time, emit struct directly - parsed = parse_posix_tz_python(timezone) - _emit_parsed_timezone_fields(parsed) + cg.add(time_var.set_timezone(timezone)) + else: + # Embedded: pre-parse at codegen time, emit struct directly + try: + parsed = parse_posix_tz_python(timezone) + _emit_parsed_timezone_fields(parsed) + except ValueError as e: + raise EsphomeError(f"Invalid timezone: {timezone}") from e for conf in config.get(CONF_ON_TIME, []): trigger = cg.new_Pvariable(conf[CONF_TRIGGER_ID], time_var) - seconds = conf.get(CONF_SECONDS, list(range(0, 61))) + seconds = conf.get(CONF_SECONDS, list(range(61))) cg.add(trigger.add_seconds(seconds)) - minutes = conf.get(CONF_MINUTES, list(range(0, 60))) + minutes = conf.get(CONF_MINUTES, list(range(60))) cg.add(trigger.add_minutes(minutes)) - hours = conf.get(CONF_HOURS, list(range(0, 24))) + hours = conf.get(CONF_HOURS, list(range(24))) cg.add(trigger.add_hours(hours)) days_of_month = conf.get(CONF_DAYS_OF_MONTH, list(range(1, 32))) cg.add(trigger.add_days_of_month(days_of_month)) diff --git a/esphome/components/voice_assistant/__init__.py b/esphome/components/voice_assistant/__init__.py index 958d1cbf91..f41adfd8de 100644 --- a/esphome/components/voice_assistant/__init__.py +++ b/esphome/components/voice_assistant/__init__.py @@ -15,7 +15,7 @@ from esphome.const import ( CONF_SPEAKER, ) -AUTO_LOAD = ["ring_buffer", "socket"] +AUTO_LOAD = ["audio", "ring_buffer", "socket"] DEPENDENCIES = ["api", "microphone"] CODEOWNERS = ["@jesserockz", "@kahrendt"] diff --git a/esphome/components/voice_assistant/voice_assistant.cpp b/esphome/components/voice_assistant/voice_assistant.cpp index 286e6645d2..f13ea39fa2 100644 --- a/esphome/components/voice_assistant/voice_assistant.cpp +++ b/esphome/components/voice_assistant/voice_assistant.cpp @@ -4,6 +4,7 @@ #ifdef USE_VOICE_ASSISTANT #include "esphome/components/socket/socket.h" +#include "esphome/core/application.h" #include "esphome/core/log.h" #include @@ -26,11 +27,16 @@ static const size_t SEND_BUFFER_SIZE = SEND_BUFFER_SAMPLES * sizeof(int16_t); static const size_t RECEIVE_SIZE = 1024; static const size_t SPEAKER_BUFFER_SIZE = 16 * RECEIVE_SIZE; +// If one microphone channel keeps producing audio while another configured channel produces none for this +// long, treat the silent channel as failed and stop the stream. A working microphone exposes a chunk every +// SEND_BUFFER_SAMPLES (32 ms), so this is far longer than any legitimate gap between chunks. +static const uint32_t AUDIO_CHANNEL_STALL_TIMEOUT_MS = 2000; + VoiceAssistant::VoiceAssistant() { global_voice_assistant = this; } void VoiceAssistant::setup() { this->mic_source_->add_data_callback([this](const std::vector &data) { - std::shared_ptr temp_ring_buffer = this->ring_buffer_; + std::shared_ptr temp_ring_buffer = this->ring_buffer_.lock(); if (temp_ring_buffer != nullptr) { temp_ring_buffer->write((void *) data.data(), data.size()); } @@ -39,7 +45,7 @@ void VoiceAssistant::setup() { // Second microphone channel if (this->mic_source2_ != nullptr) { this->mic_source2_->add_data_callback([this](const std::vector &data) { - std::shared_ptr temp_ring_buffer = this->ring_buffer2_; + std::shared_ptr temp_ring_buffer = this->ring_buffer2_.lock(); if (temp_ring_buffer != nullptr) { temp_ring_buffer->write((void *) data.data(), data.size()); } @@ -125,63 +131,51 @@ bool VoiceAssistant::allocate_buffers_() { } #endif - if (this->ring_buffer_ == nullptr) { - this->ring_buffer_ = ring_buffer::RingBuffer::create(RING_BUFFER_SIZE); - if (this->ring_buffer_ == nullptr) { + if (this->audio_source_ == nullptr) { + std::shared_ptr temp_ring_buffer = ring_buffer::RingBuffer::create(RING_BUFFER_SIZE); + if (temp_ring_buffer == nullptr) { ESP_LOGE(TAG, "Could not allocate ring buffer"); return false; } - } - - if (this->send_buffer_ == nullptr) { - RAMAllocator send_allocator; - this->send_buffer_ = send_allocator.allocate(SEND_BUFFER_SIZE); - if (send_buffer_ == nullptr) { - ESP_LOGW(TAG, "Could not allocate send buffer"); + // Zero-copy source that reads directly from the ring buffer; frame-aligned to never split an int16 sample. + this->audio_source_ = audio::RingBufferAudioSource::create(temp_ring_buffer, SEND_BUFFER_SIZE, sizeof(int16_t)); + if (this->audio_source_ == nullptr) { + ESP_LOGE(TAG, "Could not allocate audio source"); return false; } + this->ring_buffer_ = temp_ring_buffer; } // Second microphone channel - if (this->mic_source2_ != nullptr) { - if (this->ring_buffer2_ == nullptr) { - this->ring_buffer2_ = ring_buffer::RingBuffer::create(RING_BUFFER_SIZE); - if (this->ring_buffer2_ == nullptr) { - ESP_LOGE(TAG, "Could not allocate second ring buffer"); - return false; - } + if ((this->mic_source2_ != nullptr) && (this->audio_source2_ == nullptr)) { + std::shared_ptr temp_ring_buffer = ring_buffer::RingBuffer::create(RING_BUFFER_SIZE); + if (temp_ring_buffer == nullptr) { + ESP_LOGE(TAG, "Could not allocate second ring buffer"); + return false; } - - if (this->send_buffer2_ == nullptr) { - RAMAllocator send_allocator; - this->send_buffer2_ = send_allocator.allocate(SEND_BUFFER_SIZE); - if (this->send_buffer2_ == nullptr) { - ESP_LOGW(TAG, "Could not allocate second send buffer"); - return false; - } + this->audio_source2_ = audio::RingBufferAudioSource::create(temp_ring_buffer, SEND_BUFFER_SIZE, sizeof(int16_t)); + if (this->audio_source2_ == nullptr) { + ESP_LOGE(TAG, "Could not allocate second audio source"); + return false; } + this->ring_buffer2_ = temp_ring_buffer; } return true; } void VoiceAssistant::clear_buffers_() { - if (this->send_buffer_ != nullptr) { - memset(this->send_buffer_, 0, SEND_BUFFER_SIZE); - } - - if (this->ring_buffer_ != nullptr) { - this->ring_buffer_->reset(); + if (this->audio_source_ != nullptr) { + this->audio_source_->clear_buffered_data(); } // Second microphone channel - if (this->send_buffer2_ != nullptr) { - memset(this->send_buffer2_, 0, SEND_BUFFER_SIZE); + if (this->audio_source2_ != nullptr) { + this->audio_source2_->clear_buffered_data(); } - if (this->ring_buffer2_ != nullptr) { - this->ring_buffer2_->reset(); - } + // Reset the multi-channel stall watchdog (see audio_channel_stall_start_). + this->audio_channel_stall_start_ = 0; #ifdef USE_SPEAKER if ((this->speaker_ != nullptr) && (this->speaker_buffer_ != nullptr)) { @@ -195,22 +189,11 @@ void VoiceAssistant::clear_buffers_() { } void VoiceAssistant::deallocate_buffers_() { - if (this->send_buffer_ != nullptr) { - RAMAllocator send_deallocator; - send_deallocator.deallocate(this->send_buffer_, SEND_BUFFER_SIZE); - this->send_buffer_ = nullptr; - } - - this->ring_buffer_.reset(); + // Destroying each source releases its ring buffer; the matching weak_ptr then expires automatically. + this->audio_source_.reset(); // Second microphone channel - if (this->send_buffer2_ != nullptr) { - RAMAllocator send_deallocator; - send_deallocator.deallocate(this->send_buffer2_, SEND_BUFFER_SIZE); - this->send_buffer2_ = nullptr; - } - - this->ring_buffer2_.reset(); + this->audio_source2_.reset(); #ifdef USE_SPEAKER if ((this->speaker_ != nullptr) && (this->speaker_buffer_ != nullptr)) { @@ -226,6 +209,79 @@ void VoiceAssistant::reset_conversation_id() { ESP_LOGD(TAG, "reset conversation ID"); } +void VoiceAssistant::stream_api_audio_() { + // Both microphone channels are sent together, if configured. Home Assistant feeds one of the + // channels to its speech-to-text stream and treats an empty payload on that channel as + // end-of-stream, and the device cannot know which channel it picked, so only send once every + // configured channel has audio exposed, and always send them together. We don't target any + // particular message size: Home Assistant re-chunks the audio, and each fill() exposes at most + // SEND_BUFFER_SIZE bytes. + while (true) { + // fill() exposes a new chunk, or returns 0 if a previous chunk is still exposed; available() + // reports the currently exposed bytes either way. + this->audio_source_->fill(0, false); + size_t available = this->audio_source_->available(); + size_t available2 = 0; + if (this->audio_source2_ != nullptr) { + this->audio_source2_->fill(0, false); + available2 = this->audio_source2_->available(); + } + + const bool channel_empty = (available == 0); + const bool channel2_empty = (this->audio_source2_ != nullptr) && (available2 == 0); + if (channel_empty || channel2_empty) { + // A configured channel has no audio yet, so keep any chunk exposed on the other channel for the + // next pass rather than sending an empty payload. + this->handle_channel_stall_(available, available2); + break; + } + + // Both channels have audio exposed; clear any in-progress stall timer. + this->audio_channel_stall_start_ = 0; + + api::VoiceAssistantAudio msg; + // Zero-copy: send_message() copies the data out before we consume it. + msg.data = this->audio_source_->data(); + msg.data_len = available; + if (this->audio_source2_ != nullptr) { + msg.data2 = this->audio_source2_->data(); + msg.data2_len = available2; + } + + this->api_client_->send_message(msg); + + this->audio_source_->consume(available); + if (this->audio_source2_ != nullptr) { + this->audio_source2_->consume(available2); + } + } +} + +void VoiceAssistant::handle_channel_stall_(size_t available, size_t available2) { + // Called when at least one configured channel has no audio exposed. When one channel has data and the + // other does not, watch how long the empty channel stays starved: Home Assistant has no stream timeout + // and would never tell us to stop, so a channel that fails outright would otherwise hang streaming + // forever with the live channel's chunk held. Stop the stream with an error after a prolonged imbalance. + if ((available == 0) && (available2 == 0)) { + // Both channels are idle (no audio buffered yet); normal, not a stalled channel. + this->audio_channel_stall_start_ = 0; + return; + } + + const uint32_t now = App.get_loop_component_start_time(); + if (this->audio_channel_stall_start_ == 0) { + this->audio_channel_stall_start_ = now; + } else if ((now - this->audio_channel_stall_start_) >= AUDIO_CHANNEL_STALL_TIMEOUT_MS) { + ESP_LOGW(TAG, "Mic channel %d stalled, stopping stream", (available == 0) ? 0 : 1); + this->audio_channel_stall_start_ = 0; + this->signal_stop_(); + this->set_state_(State::STOP_MICROPHONE, State::IDLE); + this->defer([this]() { + this->error_trigger_.trigger("mic-channel-stalled", "A microphone channel stopped producing audio"); + }); + } +} + void VoiceAssistant::loop() { if (this->api_client_ == nullptr && this->state_ != State::IDLE && this->state_ != State::STOP_MICROPHONE && this->state_ != State::STOPPING_MICROPHONE) { @@ -316,52 +372,27 @@ void VoiceAssistant::loop() { break; // State changed when udp server port received } case State::STREAMING_MICROPHONE: { + // pre_shift is ignored by RingBufferAudioSource (no intermediate transfer buffer to compact). if (this->audio_mode_ == AUDIO_MODE_API) { - // API audio - // Both microphone channels are sent, if configured - bool is_available = this->ring_buffer_->available() >= SEND_BUFFER_SIZE; - bool is_available2 = false; - if (this->mic_source2_) { - is_available2 = this->ring_buffer2_->available() >= SEND_BUFFER_SIZE; - } - - while (is_available || is_available2) { - api::VoiceAssistantAudio msg; - - if (is_available) { - size_t read_bytes = this->ring_buffer_->read((void *) this->send_buffer_, SEND_BUFFER_SIZE, 0); - msg.data = this->send_buffer_; - msg.data_len = read_bytes; - } - - // Second microphone channel - if (is_available2) { - size_t read_bytes = this->ring_buffer2_->read((void *) this->send_buffer2_, SEND_BUFFER_SIZE, 0); - msg.data2 = this->send_buffer2_; - msg.data2_len = read_bytes; - } - - this->api_client_->send_message(msg); - is_available = this->ring_buffer_->available() >= SEND_BUFFER_SIZE; - if (this->mic_source2_) { - is_available2 = this->ring_buffer2_->available() >= SEND_BUFFER_SIZE; - } else { - is_available2 = false; - } - } + this->stream_api_audio_(); } else { // UDP (will eventually be deprecated) // Only the primary microphone channel is used - while (this->ring_buffer_->available() >= SEND_BUFFER_SIZE) { - size_t read_bytes = this->ring_buffer_->read((void *) this->send_buffer_, SEND_BUFFER_SIZE, 0); + while (true) { + this->audio_source_->fill(0, false); + size_t available = this->audio_source_->available(); + if (available == 0) { + break; + } if (!this->udp_socket_running_) { if (!this->start_udp_socket_()) { this->set_state_(State::STOP_MICROPHONE, State::IDLE); break; } } - this->socket_->sendto(this->send_buffer_, read_bytes, 0, (struct sockaddr *) &this->dest_addr_, + this->socket_->sendto(this->audio_source_->data(), available, 0, (struct sockaddr *) &this->dest_addr_, sizeof(this->dest_addr_)); + this->audio_source_->consume(available); } } // audio mode break; @@ -862,8 +893,8 @@ void VoiceAssistant::on_event(const api::VoiceAssistantEventResponse &msg) { }); State new_state = this->local_output_ ? State::STREAMING_RESPONSE : State::IDLE; if (new_state != this->state_) { - // Don't needlessly change the state. The intent progress stage may have already changed the state to streaming - // response. + // Don't needlessly change the state. The intent progress stage may have already changed the state to + // streaming response. this->set_state_(new_state, new_state); } break; diff --git a/esphome/components/voice_assistant/voice_assistant.h b/esphome/components/voice_assistant/voice_assistant.h index c4fa7eb615..76b076a366 100644 --- a/esphome/components/voice_assistant/voice_assistant.h +++ b/esphome/components/voice_assistant/voice_assistant.h @@ -9,6 +9,7 @@ #include "esphome/core/helpers.h" #include "esphome/components/api/api_connection.h" +#include "esphome/components/audio/audio_transfer_buffer.h" #include "esphome/components/ring_buffer/ring_buffer.h" #include "esphome/components/api/api_pb2.h" #include "esphome/components/microphone/microphone_source.h" @@ -243,6 +244,12 @@ class VoiceAssistant : public Component { void signal_stop_(); void start_playback_timeout_(); + // Drains the exposed microphone audio and sends it to Home Assistant over the API in one loop() pass. + void stream_api_audio_(); + // Handles a pass where at least one configured channel has no audio exposed, timing out a channel that + // stalls. See audio_channel_stall_start_. + void handle_channel_stall_(size_t available, size_t available2); + std::unique_ptr socket_ = nullptr; struct sockaddr_storage dest_addr_; @@ -306,8 +313,20 @@ class VoiceAssistant : public Component { std::string wake_word_; - std::shared_ptr ring_buffer_; - std::shared_ptr ring_buffer2_; + // Zero-copy sources that read directly from each microphone channel's ring buffer internal storage. + // Each source owns its ring buffer; the matching ``ring_buffer_``/``ring_buffer2_`` weak_ptr is used by + // the microphone callback (a different thread) to write into it. + std::unique_ptr audio_source_; + std::unique_ptr audio_source2_; + std::weak_ptr ring_buffer_; + std::weak_ptr ring_buffer2_; + + // When streaming multiple channels, the send loop holds an exposed chunk on one channel until the other + // channel also has audio so the channels are always sent together (an empty payload looks like + // end-of-stream to Home Assistant). Home Assistant has no stream timeout, so a channel that stops + // producing entirely would hang streaming forever. This records when such an imbalance began so a + // prolonged one can be detected and stopped; 0 means no imbalance is currently being timed. + uint32_t audio_channel_stall_start_{0}; bool use_wake_word_; uint8_t noise_suppression_level_; @@ -315,9 +334,6 @@ class VoiceAssistant : public Component { float volume_multiplier_; uint32_t conversation_timeout_; - uint8_t *send_buffer_{nullptr}; - uint8_t *send_buffer2_{nullptr}; - bool continuous_{false}; bool silence_detection_; diff --git a/esphome/components/waveshare_epaper/display.py b/esphome/components/waveshare_epaper/display.py index 5db7a1fc3d..7ecc3b4a87 100644 --- a/esphome/components/waveshare_epaper/display.py +++ b/esphome/components/waveshare_epaper/display.py @@ -236,7 +236,7 @@ async def to_code(config): rhs = model.new() var = cg.Pvariable(config[CONF_ID], rhs, model) else: - raise NotImplementedError() + raise NotImplementedError await display.register_display(var, config) await spi.register_spi_device(var, config, write_only=True) diff --git a/esphome/components/web_server/__init__.py b/esphome/components/web_server/__init__.py index 84910b6f90..fd380a38dd 100644 --- a/esphome/components/web_server/__init__.py +++ b/esphome/components/web_server/__init__.py @@ -193,8 +193,8 @@ CONFIG_SCHEMA = cv.All( cv.Required(CONF_USERNAME): cv.All( cv.string_strict, cv.Length(min=1) ), - cv.Required(CONF_PASSWORD): cv.All( - cv.string_strict, cv.Length(min=1) + cv.Required(CONF_PASSWORD): cv.sensitive( + cv.All(cv.string_strict, cv.Length(min=1)) ), } ), @@ -326,12 +326,12 @@ async def to_code(config): if CONF_CSS_INCLUDE in config: cg.add_define("USE_WEBSERVER_CSS_INCLUDE") path = CORE.relative_config_path(config[CONF_CSS_INCLUDE]) - with open(file=path, encoding="utf-8") as css_file: + with path.open(encoding="utf-8") as css_file: add_resource_as_progmem("CSS_INCLUDE", css_file.read()) if CONF_JS_INCLUDE in config: cg.add_define("USE_WEBSERVER_JS_INCLUDE") path = CORE.relative_config_path(config[CONF_JS_INCLUDE]) - with open(file=path, encoding="utf-8") as js_file: + with path.open(encoding="utf-8") as js_file: add_resource_as_progmem("JS_INCLUDE", js_file.read()) cg.add(var.set_include_internal(config[CONF_INCLUDE_INTERNAL])) if CONF_LOCAL in config and config[CONF_LOCAL]: diff --git a/esphome/components/wifi/__init__.py b/esphome/components/wifi/__init__.py index f9cb391442..b7719c80d1 100644 --- a/esphome/components/wifi/__init__.py +++ b/esphome/components/wifi/__init__.py @@ -251,7 +251,7 @@ EAP_AUTH_SCHEMA = cv.All( { cv.Optional(CONF_IDENTITY): cv.string_strict, cv.Optional(CONF_USERNAME): cv.string_strict, - cv.Optional(CONF_PASSWORD): cv.string_strict, + cv.Optional(CONF_PASSWORD): cv.sensitive(cv.string_strict), cv.Optional(CONF_CERTIFICATE_AUTHORITY): wpa2_eap.validate_certificate, cv.SplitDefault(CONF_TTLS_PHASE_2, esp32="mschapv2"): cv.All( cv.enum(TTLS_PHASE_2), cv.only_on_esp32 @@ -271,8 +271,8 @@ EAP_AUTH_SCHEMA = cv.All( WIFI_NETWORK_BASE = cv.Schema( { cv.GenerateID(): cv.declare_id(WiFiAP), - cv.Optional(CONF_SSID): cv.ssid, - cv.Optional(CONF_PASSWORD): validate_password, + cv.Optional(CONF_SSID): cv.sensitive(cv.ssid), + cv.Optional(CONF_PASSWORD): cv.sensitive(validate_password), cv.Optional(CONF_CHANNEL): validate_channel, cv.Optional(CONF_MANUAL_IP): STA_MANUAL_IP_SCHEMA, } @@ -326,23 +326,9 @@ def validate_variant(_): def _apply_min_auth_mode_default(config): - """Apply platform-specific default for min_auth_mode and warn ESP8266 users.""" - # Only apply defaults for platforms that support min_auth_mode + """Apply platform-specific default for min_auth_mode.""" if CONF_MIN_AUTH_MODE not in config and (CORE.is_esp8266 or CORE.is_esp32): - if CORE.is_esp8266: - _LOGGER.warning( - "The minimum WiFi authentication mode (wifi -> min_auth_mode) is not set. " - "This controls the weakest encryption your device will accept when connecting to WiFi. " - "Currently defaults to WPA (less secure), but will change to WPA2 (more secure) in 2026.6.0. " - "WPA uses TKIP encryption which has known security vulnerabilities and should be avoided. " - "WPA2 uses AES encryption which is significantly more secure. " - "To silence this warning, explicitly set min_auth_mode under 'wifi:'. " - "If your router supports WPA2 or WPA3, set 'min_auth_mode: WPA2'. " - "If your router only supports WPA, set 'min_auth_mode: WPA'." - ) - config[CONF_MIN_AUTH_MODE] = VALIDATE_WIFI_MIN_AUTH_MODE("WPA") - elif CORE.is_esp32: - config[CONF_MIN_AUTH_MODE] = VALIDATE_WIFI_MIN_AUTH_MODE("WPA2") + config[CONF_MIN_AUTH_MODE] = VALIDATE_WIFI_MIN_AUTH_MODE("WPA2") return config @@ -448,8 +434,8 @@ CONFIG_SCHEMA = cv.All( cv.Optional(CONF_NETWORKS): cv.All( cv.ensure_list(WIFI_NETWORK_STA), cv.Length(max=MAX_WIFI_NETWORKS) ), - cv.Optional(CONF_SSID): cv.ssid, - cv.Optional(CONF_PASSWORD): validate_password, + cv.Optional(CONF_SSID): cv.sensitive(cv.ssid), + cv.Optional(CONF_PASSWORD): cv.sensitive(validate_password), cv.Optional(CONF_MANUAL_IP): STA_MANUAL_IP_SCHEMA, cv.Optional(CONF_EAP): EAP_AUTH_SCHEMA, cv.Optional(CONF_AP): wifi_network_ap, @@ -864,8 +850,8 @@ async def final_step(): WiFiConfigureAction, cv.Schema( { - cv.Required(CONF_SSID): cv.templatable(cv.ssid), - cv.Required(CONF_PASSWORD): cv.templatable(validate_password), + cv.Required(CONF_SSID): cv.sensitive(cv.templatable(cv.ssid)), + cv.Required(CONF_PASSWORD): cv.sensitive(cv.templatable(validate_password)), cv.Optional(CONF_SAVE, default=True): cv.templatable(cv.boolean), cv.Optional(CONF_TIMEOUT, default="30000ms"): cv.templatable( cv.positive_time_period_milliseconds diff --git a/esphome/components/wifi/wifi_component.cpp b/esphome/components/wifi/wifi_component.cpp index edfb93bba2..fdbd70bc61 100644 --- a/esphome/components/wifi/wifi_component.cpp +++ b/esphome/components/wifi/wifi_component.cpp @@ -634,9 +634,6 @@ void WiFiComponent::setup() { if (this->enable_on_boot_) { this->start(); } else { -#ifdef USE_ESP32 - esp_netif_init(); -#endif this->state_ = WIFI_COMPONENT_STATE_DISABLED; } } @@ -2193,7 +2190,15 @@ bool WiFiComponent::request_high_performance() { } // Give the semaphore (non-blocking). This increments the count. - return xSemaphoreGive(this->high_performance_semaphore_) == pdTRUE; + bool success = xSemaphoreGive(this->high_performance_semaphore_) == pdTRUE; + + // Wake the main loop so the switch to high-performance mode is applied on the + // next tick instead of waiting up to loop_interval. + if (success) { + App.wake_loop_threadsafe(); + } + + return success; } bool WiFiComponent::release_high_performance() { diff --git a/esphome/components/wifi/wifi_component_esp_idf.cpp b/esphome/components/wifi/wifi_component_esp_idf.cpp index 1e08aea72a..c1f8241350 100644 --- a/esphome/components/wifi/wifi_component_esp_idf.cpp +++ b/esphome/components/wifi/wifi_component_esp_idf.cpp @@ -146,23 +146,15 @@ void WiFiComponent::wifi_pre_setup_() { get_mac_address_raw(mac); set_mac_address(mac); } - esp_err_t err = esp_netif_init(); - if (err != ERR_OK) { - ESP_LOGE(TAG, "esp_netif_init failed: %s", esp_err_to_name(err)); - return; - } + // Network interface setup handled by network component s_wifi_event_group = xEventGroupCreate(); if (s_wifi_event_group == nullptr) { ESP_LOGE(TAG, "xEventGroupCreate failed"); return; } - err = esp_event_loop_create_default(); - if (err != ERR_OK) { - ESP_LOGE(TAG, "esp_event_loop_create_default failed: %s", esp_err_to_name(err)); - return; - } esp_event_handler_instance_t instance_wifi_id, instance_ip_id; - err = esp_event_handler_instance_register(WIFI_EVENT, ESP_EVENT_ANY_ID, &event_handler, nullptr, &instance_wifi_id); + esp_err_t err = + esp_event_handler_instance_register(WIFI_EVENT, ESP_EVENT_ANY_ID, &event_handler, nullptr, &instance_wifi_id); if (err != ERR_OK) { ESP_LOGE(TAG, "esp_event_handler_instance_register failed: %s", esp_err_to_name(err)); return; diff --git a/esphome/components/zigbee/zigbee_esp32.py b/esphome/components/zigbee/zigbee_esp32.py index 89efd583ab..a0fadbce8b 100644 --- a/esphome/components/zigbee/zigbee_esp32.py +++ b/esphome/components/zigbee/zigbee_esp32.py @@ -129,9 +129,8 @@ def final_validate_esp32(config: ConfigType) -> ConfigType: if CONF_PARTITIONS in fv.full_config.get() and not isinstance( fv.full_config.get()[CONF_PARTITIONS], list ): - with open( - CORE.relative_config_path(fv.full_config.get()[CONF_PARTITIONS]), - encoding="utf8", + with CORE.relative_config_path(fv.full_config.get()[CONF_PARTITIONS]).open( + encoding="utf8" ) as f: partitions_tab = f.read() for partition, types in [ diff --git a/esphome/components/zigbee/zigbee_zephyr.py b/esphome/components/zigbee/zigbee_zephyr.py index aa16bbef53..39ecadfddf 100644 --- a/esphome/components/zigbee/zigbee_zephyr.py +++ b/esphome/components/zigbee/zigbee_zephyr.py @@ -161,7 +161,10 @@ async def _attr_to_code(config: ConfigType) -> None: zigbee_set_string(basic_attrs.mf_name, "esphome"), zigbee_set_string(basic_attrs.model_id, config[CONF_MODEL]), zigbee_set_string( - basic_attrs.date_code, datetime.datetime.now().strftime("%Y%m%d %H%M%S") + basic_attrs.date_code, + # Local build time, matching the esp32 implementation + # (App.get_build_time() in C++). + datetime.datetime.now().astimezone().strftime("%Y%m%d %H%M%S"), ), zigbee_assign( basic_attrs.power_source, diff --git a/esphome/config.py b/esphome/config.py index 79d0d2b02b..9da39a387b 100644 --- a/esphome/config.py +++ b/esphome/config.py @@ -1005,7 +1005,6 @@ def validate_config( CORE.skip_external_update = skip_external_update loader.clear_component_meta_finders() - loader.install_custom_components_meta_finder() # 0. Load packages if CONF_PACKAGES in config: diff --git a/esphome/config_validation.py b/esphome/config_validation.py index c993c1dcc5..0ef6d212fe 100644 --- a/esphome/config_validation.py +++ b/esphome/config_validation.py @@ -101,7 +101,7 @@ from esphome.schema_extractors import ( ) from esphome.util import parse_esphome_version from esphome.voluptuous_schema import _Schema -from esphome.yaml_util import make_data_base +from esphome.yaml_util import SensitiveStr, make_data_base _LOGGER = logging.getLogger(__name__) @@ -487,6 +487,59 @@ def string_strict(value): ) +# Substring fallbacks for fields whose validator isn't explicitly wrapped in +# ``cv.sensitive``. Frontends and dump tooling should prefer the explicit +# marker; this list exists so we still mask obvious leaks in unmigrated or +# third-party schemas. Kept here as the single source of truth. +SENSITIVE_KEY_FRAGMENTS: frozenset[str] = frozenset( + { + "password", + "passcode", + "secret", + "token", + "api_key", + "apikey", + "psk", + } +) + + +class SensitiveValidator: + """Marker wrapper that flags a field as containing sensitive data (passwords, + encryption keys, PSKs, tokens). Frontends and dump tooling detect this marker + to mask the value; validation behavior is delegated to the inner validator. + """ + + def __init__(self, inner: Callable[[typing.Any], typing.Any]) -> None: + self.inner = inner + + def __call__(self, value: typing.Any) -> typing.Any: + validated = self.inner(value) + # Tag string results so yaml_util.dump can mask them. Non-string + # results pass through unchanged; already-tagged values are not + # re-wrapped to keep nested cv.sensitive applications idempotent. + if isinstance(validated, str) and not isinstance(validated, SensitiveStr): + return SensitiveStr(validated) + return validated + + def __repr__(self) -> str: + # Mirror the inner validator's repr so ``build_language_schema``'s + # ``known_schemas``/``extended_schemas`` dedup (keyed on ``repr(schema)``) + # treats two wrappers around the same inner as identical, and so + # voluptuous error messages stay readable. + return repr(self.inner) + + +def sensitive( + inner: Callable[[typing.Any], typing.Any] = string, +) -> SensitiveValidator: + """Mark a field as sensitive so that frontends mask it and dump tooling redacts it. + + Validation behavior is identical to ``inner`` (defaults to ``cv.string``). + """ + return SensitiveValidator(inner) + + def icon(value): """Validate that a given config value is a valid icon.""" from esphome.core.config import ICON_MAX_LENGTH @@ -810,16 +863,6 @@ only_on_rp2040 = only_on(PLATFORM_RP2040) only_with_arduino = only_with_framework(Framework.ARDUINO) -def only_with_esp_idf(obj): - """Deprecated: use only_on_esp32 instead.""" - _LOGGER.warning( - "cv.only_with_esp_idf was deprecated in 2026.1, will change behavior in 2026.6. " - "ESP32 Arduino builds on top of ESP-IDF, so ESP-IDF features are available in both frameworks. " - "Use cv.only_on_esp32 and/or cv.only_with_arduino instead." - ) - return only_with_framework(Framework.ESP_IDF)(obj) - - # Adapted from: # https://github.com/alecthomas/voluptuous/issues/115#issuecomment-144464666 def has_at_least_one_key(*keys): @@ -1136,7 +1179,9 @@ def date_time(date: bool, time: bool): format += "%p" try: - date_obj = datetime.strptime(value, format) + # The generated format never includes %z/%Z, so this parses a + # naive wall-clock date/time by design. + date_obj = datetime.strptime(value, format) # noqa: DTZ007 except ValueError as err: raise Invalid(f"Invalid {exc_message}: {err}") from err @@ -1862,7 +1907,7 @@ def extract_keys(schema): elif isinstance(skey, vol.Marker) and isinstance(skey.schema, str): keys.append(skey.schema) else: - raise ValueError() + raise ValueError keys.sort() return keys diff --git a/esphome/const.py b/esphome/const.py index 9dd77a7cb8..07f6bad771 100644 --- a/esphome/const.py +++ b/esphome/const.py @@ -199,6 +199,7 @@ CONF_BROKER = "broker" CONF_BSSID = "bssid" CONF_BUFFER_DURATION = "buffer_duration" CONF_BUFFER_SIZE = "buffer_size" +CONF_BUILD_FLAGS = "build_flags" CONF_BUILD_PATH = "build_path" CONF_BUS_VOLTAGE = "bus_voltage" CONF_BUSY_PIN = "busy_pin" diff --git a/esphome/core/__init__.py b/esphome/core/__init__.py index 3a7bdf5d4b..f73ede8a62 100644 --- a/esphome/core/__init__.py +++ b/esphome/core/__init__.py @@ -895,15 +895,6 @@ class EsphomeCore: def using_arduino(self): return self.target_framework == "arduino" - @property - def using_esp_idf(self): - _LOGGER.warning( - "CORE.using_esp_idf was deprecated in 2026.1, will change behavior in 2026.6. " - "ESP32 Arduino builds on top of ESP-IDF, so ESP-IDF features are available in both frameworks. " - "Use CORE.is_esp32 and/or CORE.using_arduino instead." - ) - return self.target_framework == "esp-idf" - @property def using_toolchain_esp_idf(self): return self.toolchain == Toolchain.ESP_IDF @@ -1132,7 +1123,7 @@ class EnumValue: @enum_value.setter def enum_value(self, value): - setattr(self, "_enum_value", value) + self._enum_value = value CORE = EsphomeCore() diff --git a/esphome/core/automation.h b/esphome/core/automation.h index 468ea3b382..ea522a4d2d 100644 --- a/esphome/core/automation.h +++ b/esphome/core/automation.h @@ -13,27 +13,6 @@ namespace esphome { -// C++20 std::index_sequence is now used for tuple unpacking -// Legacy seq<>/gens<> pattern deprecated but kept for backwards compatibility -// https://stackoverflow.com/questions/7858817/unpacking-a-tuple-to-call-a-matching-function-pointer/7858971#7858971 -// Remove before 2026.6.0 -// NOLINTBEGIN(readability-identifier-naming) -#if defined(__GNUC__) || defined(__clang__) -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wdeprecated-declarations" -#endif - -template struct ESPDEPRECATED("Use std::index_sequence instead. Removed in 2026.6.0", "2025.12.0") seq {}; -template -struct ESPDEPRECATED("Use std::make_index_sequence instead. Removed in 2026.6.0", "2025.12.0") gens - : gens {}; -template struct gens<0, S...> { using type = seq; }; - -#if defined(__GNUC__) || defined(__clang__) -#pragma GCC diagnostic pop -#endif -// NOLINTEND(readability-identifier-naming) - /// Function-pointer-only templatable storage (4 bytes on 32-bit). /// Used by the TEMPLATABLE_VALUE macro for codegen-managed fields. /// Codegen wraps constants in stateless lambdas so only a function pointer is needed. diff --git a/esphome/core/component.cpp b/esphome/core/component.cpp index e33652482e..2d80301897 100644 --- a/esphome/core/component.cpp +++ b/esphome/core/component.cpp @@ -32,10 +32,7 @@ static const char *const TAG = "component"; namespace { struct ComponentErrorMessage { const Component *component; - const char *message; - // Track if message is flash pointer (needs LOG_STR_ARG) or RAM pointer - // Remove before 2026.6.0 when deprecated const char* API is removed - bool is_flash_ptr; + const LogString *message; }; #ifdef USE_SETUP_PRIORITY_OVERRIDE @@ -56,9 +53,8 @@ std::vector *setup_priority_overrides = nullptr; // NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables) std::vector *component_error_messages = nullptr; -// Helper to store error messages - reduces duplication between deprecated and new API -// Remove before 2026.6.0 when deprecated const char* API is removed -void store_component_error_message(const Component *component, const char *message, bool is_flash_ptr) { +// Helper to store error messages +void store_component_error_message(const Component *component, const LogString *message) { // Lazy allocate the error messages vector if needed if (!component_error_messages) { component_error_messages = new std::vector(); @@ -67,12 +63,11 @@ void store_component_error_message(const Component *component, const char *messa for (auto &entry : *component_error_messages) { if (entry.component == component) { entry.message = message; - entry.is_flash_ptr = is_flash_ptr; return; } } // Add new error message - component_error_messages->emplace_back(ComponentErrorMessage{component, message, is_flash_ptr}); + component_error_messages->emplace_back(ComponentErrorMessage{component, message}); } } // namespace @@ -209,21 +204,17 @@ void Component::call_dump_config_() { this->dump_config(); if (this->is_failed()) { // Look up error message from global vector - const char *error_msg = nullptr; - bool is_flash_ptr = false; + const LogString *error_msg = nullptr; if (component_error_messages) { for (const auto &entry : *component_error_messages) { if (entry.component == this) { error_msg = entry.message; - is_flash_ptr = entry.is_flash_ptr; break; } } } - // Log with appropriate format based on pointer type ESP_LOGE(TAG, " %s is marked FAILED: %s", LOG_STR_ARG(this->get_component_log_str()), - error_msg ? (is_flash_ptr ? LOG_STR_ARG((const LogString *) error_msg) : error_msg) - : LOG_STR_LITERAL("unspecified")); + error_msg ? LOG_STR_ARG(error_msg) : LOG_STR_LITERAL("unspecified")); } } @@ -390,23 +381,13 @@ void Component::status_set_warning(const LogString *message) { message ? LOG_STR_ARG(message) : LOG_STR_LITERAL("unspecified")); } void Component::status_set_error() { this->status_set_error((const LogString *) nullptr); } -void Component::status_set_error(const char *message) { - if (!this->set_status_flag_(STATUS_LED_ERROR)) - return; - ESP_LOGE(TAG, "%s set Error flag: %s", LOG_STR_ARG(this->get_component_log_str()), - message ? message : LOG_STR_LITERAL("unspecified")); - if (message != nullptr) { - store_component_error_message(this, message, false); - } -} void Component::status_set_error(const LogString *message) { if (!this->set_status_flag_(STATUS_LED_ERROR)) return; ESP_LOGE(TAG, "%s set Error flag: %s", LOG_STR_ARG(this->get_component_log_str()), message ? LOG_STR_ARG(message) : LOG_STR_LITERAL("unspecified")); if (message != nullptr) { - // Store the LogString pointer directly (safe because LogString is always in flash/static memory) - store_component_error_message(this, LOG_STR_ARG(message), true); + store_component_error_message(this, message); } } void Component::status_clear_warning_slow_path_() { diff --git a/esphome/core/component.h b/esphome/core/component.h index 5baf795ca6..ff10f1a8f1 100644 --- a/esphome/core/component.h +++ b/esphome/core/component.h @@ -220,18 +220,6 @@ class Component { */ void mark_failed(); - // Remove before 2026.6.0 - ESPDEPRECATED("Use mark_failed(LOG_STR(\"static string literal\")) instead. Do NOT use .c_str() from temporary " - "strings. Will stop working in 2026.6.0", - "2025.12.0") - void mark_failed(const char *message) { -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wdeprecated-declarations" - this->status_set_error(message); -#pragma GCC diagnostic pop - this->mark_failed(); - } - void mark_failed(const LogString *message) { this->status_set_error(message); this->mark_failed(); @@ -296,11 +284,6 @@ class Component { void status_set_warning(const LogString *message); void status_set_error(); // Set error flag without message - // Remove before 2026.6.0 - ESPDEPRECATED("Use status_set_error(LOG_STR(\"static string literal\")) instead. Do NOT use .c_str() from temporary " - "strings. Will stop working in 2026.6.0", - "2025.12.0") - void status_set_error(const char *message); void status_set_error(const LogString *message); void status_clear_warning() { diff --git a/esphome/core/config.py b/esphome/core/config.py index 5a98b94781..6125c4ecc9 100644 --- a/esphome/core/config.py +++ b/esphome/core/config.py @@ -13,6 +13,7 @@ from esphome.const import ( CONF_AREA, CONF_AREA_ID, CONF_AREAS, + CONF_BUILD_FLAGS, CONF_BUILD_PATH, CONF_COMMENT, CONF_COMPILE_PROCESS_LIMIT, @@ -288,6 +289,7 @@ CONFIG_SCHEMA = cv.All( cv.string_strict: cv.Any([cv.string], cv.string), } ), + cv.Optional(CONF_BUILD_FLAGS, default=[]): cv.ensure_list(cv.string_strict), cv.Optional(CONF_ENVIRONMENT_VARIABLES, default={}): cv.Schema( { cv.string_strict: cv.string, @@ -510,6 +512,12 @@ async def _add_platformio_options(pio_options): cg.add_platformio_option(key, val) +@coroutine_with_priority(CoroPriority.FINAL) +async def _add_build_flags(flags: list[str]) -> None: + for flag in flags: + cg.add_build_flag(flag) + + @coroutine_with_priority(CoroPriority.FINAL) async def _add_environment_variables(env_vars: dict[str, str]) -> None: # Set environment variables for the build process @@ -705,6 +713,9 @@ async def to_code(config: ConfigType) -> None: if config[CONF_PLATFORMIO_OPTIONS]: CORE.add_job(_add_platformio_options, config[CONF_PLATFORMIO_OPTIONS]) + if config[CONF_BUILD_FLAGS]: + CORE.add_job(_add_build_flags, config[CONF_BUILD_FLAGS]) + if config[CONF_ENVIRONMENT_VARIABLES]: CORE.add_job(_add_environment_variables, config[CONF_ENVIRONMENT_VARIABLES]) diff --git a/esphome/core/defines.h b/esphome/core/defines.h index ee8e89de8b..0229bc14fa 100644 --- a/esphome/core/defines.h +++ b/esphome/core/defines.h @@ -71,6 +71,7 @@ #define USE_GRAPH #define USE_GRAPHICAL_DISPLAY_MENU #define USE_HOMEASSISTANT_TIME +#define USE_HOMEASSISTANT_TIMEZONE #define USE_HTTP_REQUEST_OTA_WATCHDOG_TIMEOUT 8000 // NOLINT #define USE_I2S_AUDIO_SPDIF_MODE #define USE_IMAGE @@ -401,6 +402,7 @@ #define USE_LOGGER_USB_CDC #define USE_SOCKET_IMPL_LWIP_TCP #define USE_RP2040_BLE +#define USE_RP2040_VARIANT_RP2040 #define USE_SPI #ifndef USE_ETHERNET #define USE_ETHERNET diff --git a/esphome/cpp_generator.py b/esphome/cpp_generator.py index 49abdb40ca..18ce76f767 100644 --- a/esphome/cpp_generator.py +++ b/esphome/cpp_generator.py @@ -918,7 +918,7 @@ class MockObj(Expression): def __getattr__(self, attr: str) -> "MockObj": # prevent python dunder methods being replaced by mock objects if attr.startswith("__"): - raise AttributeError() + raise AttributeError next_op = "." if attr.startswith("P") and self.op not in ["::", ""]: attr = attr[1:] @@ -1102,43 +1102,45 @@ class MockObj(Expression): op = BinOpExpression(other, "|", self) return MockObj(op) - def __iadd__(self, other: SafeExpType) -> "MockObj": + # MockObj operator overloads build a new C++ expression rather than mutating self, + # so the PYI034 "augmented assignment returns self" assumption does not apply. + def __iadd__(self, other: SafeExpType) -> "MockObj": # noqa: PYI034 op = BinOpExpression(self, "+=", other) return MockObj(op) - def __isub__(self, other: SafeExpType) -> "MockObj": + def __isub__(self, other: SafeExpType) -> "MockObj": # noqa: PYI034 op = BinOpExpression(self, "-=", other) return MockObj(op) - def __imul__(self, other: SafeExpType) -> "MockObj": + def __imul__(self, other: SafeExpType) -> "MockObj": # noqa: PYI034 op = BinOpExpression(self, "*=", other) return MockObj(op) - def __itruediv__(self, other: SafeExpType) -> "MockObj": + def __itruediv__(self, other: SafeExpType) -> "MockObj": # noqa: PYI034 op = BinOpExpression(self, "/=", other) return MockObj(op) - def __imod__(self, other: SafeExpType) -> "MockObj": + def __imod__(self, other: SafeExpType) -> "MockObj": # noqa: PYI034 op = BinOpExpression(self, "%=", other) return MockObj(op) - def __ilshift__(self, other: SafeExpType) -> "MockObj": + def __ilshift__(self, other: SafeExpType) -> "MockObj": # noqa: PYI034 op = BinOpExpression(self, "<<=", other) return MockObj(op) - def __irshift__(self, other: SafeExpType) -> "MockObj": + def __irshift__(self, other: SafeExpType) -> "MockObj": # noqa: PYI034 op = BinOpExpression(self, ">>=", other) return MockObj(op) - def __iand__(self, other: SafeExpType) -> "MockObj": + def __iand__(self, other: SafeExpType) -> "MockObj": # noqa: PYI034 op = BinOpExpression(self, "&=", other) return MockObj(op) - def __ixor__(self, other: SafeExpType) -> "MockObj": + def __ixor__(self, other: SafeExpType) -> "MockObj": # noqa: PYI034 op = BinOpExpression(self, "^=", other) return MockObj(op) - def __ior__(self, other: SafeExpType) -> "MockObj": + def __ior__(self, other: SafeExpType) -> "MockObj": # noqa: PYI034 op = BinOpExpression(self, "|=", other) return MockObj(op) diff --git a/esphome/dashboard/dashboard.py b/esphome/dashboard/dashboard.py index 81c10763e7..7fc21f8a44 100644 --- a/esphome/dashboard/dashboard.py +++ b/esphome/dashboard/dashboard.py @@ -6,6 +6,7 @@ from concurrent.futures import ThreadPoolExecutor import contextlib import logging import os +from pathlib import Path import socket import threading from time import monotonic @@ -149,4 +150,4 @@ async def async_start(args) -> None: await dashboard.async_run() finally: if sock: - os.remove(sock) + Path(sock).unlink() diff --git a/esphome/dashboard/status/mdns.py b/esphome/dashboard/status/mdns.py index 881340ab24..9da9bb8f01 100644 --- a/esphome/dashboard/status/mdns.py +++ b/esphome/dashboard/status/mdns.py @@ -115,7 +115,7 @@ class MDNSStatus: results = await asyncio.gather( *(self.aiozc.async_resolve_host(name) for name in poll_names) ) - for name, address_list in zip(poll_names, results): + for name, address_list in zip(poll_names, results, strict=True): result = bool(address_list) host_mdns_state[name] = result for entry in poll_names[name]: diff --git a/esphome/dashboard/status/ping.py b/esphome/dashboard/status/ping.py index b4f106d21a..eb69fbb9b3 100644 --- a/esphome/dashboard/status/ping.py +++ b/esphome/dashboard/status/ping.py @@ -83,7 +83,7 @@ class PingStatus: return_exceptions=True, ) - for entry, result in zip(ping_group, dns_results): + for entry, result in zip(ping_group, dns_results, strict=True): if isinstance(result, Exception): # Only update state if its unknown or from ping # so we don't mark it as offline if we have a state @@ -106,7 +106,7 @@ class PingStatus: return_exceptions=True, ) - for entry_addresses, result in zip(entry_addresses, results): + for entry_address, result in zip(entry_addresses, results, strict=True): if isinstance(result, Exception): ping_result = False elif isinstance(result, BaseException): @@ -114,7 +114,7 @@ class PingStatus: else: host: Host = result ping_result = host.is_alive - entry: DashboardEntry = entry_addresses[0] + entry: DashboardEntry = entry_address[0] # If we can reach it via ping, we always set it # online, however if we can't reach it via ping # we only set it to offline if the state is unknown diff --git a/esphome/dashboard/web_server.py b/esphome/dashboard/web_server.py index 916e937a53..97d6639c1f 100644 --- a/esphome/dashboard/web_server.py +++ b/esphome/dashboard/web_server.py @@ -1030,7 +1030,7 @@ class DownloadListRequestHandler(BaseHandler): try: module = importlib.import_module(f"esphome.components.{platform}") - get_download_types = getattr(module, "get_download_types") + get_download_types = module.get_download_types except AttributeError as exc: raise ValueError(f"Unknown platform {platform}") from exc downloads = get_download_types(storage_json) @@ -1040,7 +1040,7 @@ class DownloadListRequestHandler(BaseHandler): class DownloadBinaryRequestHandler(BaseHandler): def _load_file(self, path: str, compressed: bool) -> bytes: """Load a file from disk and compress it if requested.""" - with open(path, "rb") as f: + with Path(path).open("rb") as f: data = f.read() if compressed: return gzip.compress(data, 9) @@ -1146,7 +1146,7 @@ class MainRequestHandler(BaseHandler): begin = bool(self.get_argument("begin", False)) if settings.using_password: # Simply accessing the xsrf_token sets the cookie for us - self.xsrf_token # pylint: disable=pointless-statement + self.xsrf_token # pylint: disable=pointless-statement # noqa: B018 else: self.clear_cookie("_xsrf") @@ -1292,7 +1292,7 @@ class EditRequestHandler(BaseHandler): def _read_file(self, filename: str, configuration: str) -> bytes | None: """Read a file and return the content as bytes.""" try: - with open(file=filename, encoding="utf-8") as f: + with Path(filename).open(encoding="utf-8") as f: return f.read() except FileNotFoundError: if configuration in const.SECRETS_FILES: @@ -1493,7 +1493,7 @@ def get_base_frontend_path() -> Path: static_path += "/" # This path can be relative, so resolve against the root or else templates don't work - path = Path(os.getcwd()) / static_path / "esphome_dashboard" + path = Path.cwd() / static_path / "esphome_dashboard" return path.resolve() @@ -1519,7 +1519,10 @@ def get_static_file_url(name: str) -> str: return f"{base}?hash={hash_}" -def make_app(debug=get_bool_env(ENV_DEV)) -> tornado.web.Application: +def make_app(debug: bool | None = None) -> tornado.web.Application: + if debug is None: + debug = get_bool_env(ENV_DEV) + def log_function(handler: tornado.web.RequestHandler) -> None: if handler.get_status() < 400: log_method = access_log.info diff --git a/esphome/espidf/component.py b/esphome/espidf/component.py index a452a3f34a..050002d9e2 100644 --- a/esphome/espidf/component.py +++ b/esphome/espidf/component.py @@ -55,7 +55,7 @@ ESPHOME_DATA_EXTRA_CMAKE_KEY = "EXTRA_CMAKE" class Source: def download(self, dir_suffix: str, force: bool = False) -> Path: - raise NotImplementedError() + raise NotImplementedError class URLSource(Source): @@ -317,24 +317,26 @@ def _collect_filtered_files(src_dir: PathType, src_filters: list[str]) -> list[s if pattern.endswith("/"): pattern = pattern.rstrip("/") + "/**" - full_pattern = os.path.join(glob.escape(str(src_dir)), pattern) + # glob.escape has no pathlib equivalent and the matcher works on raw + # path strings, so PTH118/PTH207 don't apply here. + full_pattern = os.path.join(glob.escape(str(src_dir)), pattern) # noqa: PTH118 matched = [] - for item in glob.glob(full_pattern, recursive=True): - if not os.path.isdir(item): + for item in glob.glob(full_pattern, recursive=True): # noqa: PTH207 + if not Path(item).is_dir(): matched.append(item) else: # PlatformIO quirk: a directory matched with "*" should include all its # nested files and subdirectories, not just the directory itself. for root, _, files in os.walk(item): - matched.extend([os.path.join(root, f) for f in files]) + matched.extend([str(Path(root) / f) for f in files]) if sign == "+": selected.update(matched) elif sign == "-": selected.difference_update(matched) - return [r for r in selected if os.path.isfile(r)] + return [r for r in selected if Path(r).is_file()] def _convert_library_to_component(library: Library) -> IDFComponent: @@ -486,7 +488,7 @@ def generate_cmakelists_txt(component: IDFComponent) -> str: # Only keep sources build_src_files = [os.path.relpath(p, component.path) for p in build_src_files] build_src_files = [ - f for f in build_src_files if os.path.splitext(f)[1] in SRC_FILE_EXTENSIONS + f for f in build_src_files if Path(f).suffix in SRC_FILE_EXTENSIONS ] # Handle build flags @@ -610,11 +612,17 @@ def _check_library_data(data: dict): """ Check if a library data is compatible with the ESP-IDF framework. + A platform mismatch (e.g. an AVR-only library on ESP32) raises + ``InvalidIDFComponent`` so the caller skips the library. A framework + mismatch only logs a warning — PIO manifests often understate the + frameworks they actually compile under, and IDF (unlike PIO's + ``lib_compat_mode``) has no opt-out, so we include the library anyway. + Args: - component: IDFComponent object being processed + data: PIO library manifest dict being processed. Raises: - ValueError: If library has unsupported platforms or frameworks + InvalidIDFComponent: If the library does not support the ESP32 platform. """ platforms = data.get("platforms", "*") if isinstance(platforms, str): @@ -632,12 +640,21 @@ def _check_library_data(data: dict): frameworks = [a.strip() for a in frameworks.split(",")] frameworks = _ensure_list(frameworks) - # Check if library supports ESP-IDF framework + # Check if library declares the active framework. PIO library manifests + # often list only "arduino" even when the library actually compiles fine + # under ESP-IDF, and IDF (unlike PIO with `lib_compat_mode`) has no way to + # opt out of the check. Warn instead of failing so the user isn't forced to + # fork the library to fix the manifest. framework = "arduino" if CORE.using_arduino else "espidf" valid_framework = "*" in frameworks or framework in frameworks if not valid_framework: - raise InvalidIDFComponent(f"Unsupported library frameworks: {frameworks}") + _LOGGER.warning( + "Library %s declares frameworks %s that do not include '%s'; including anyway", + data.get("name", ""), + frameworks, + framework, + ) def _process_dependencies(component: IDFComponent): @@ -725,7 +742,7 @@ def _parse_library_json(library_json_path: PathType): Returns: dict: Parsed JSON content as a Python dictionary. """ - with open(library_json_path, encoding="utf8") as fp: + with Path(library_json_path).open(encoding="utf8") as fp: return json.load(fp) @@ -739,7 +756,7 @@ def _parse_library_properties(library_properties_path: PathType): Returns: dict[str, str]: Mapping of parsed property keys to values. """ - with open(library_properties_path, encoding="utf8") as fp: + with Path(library_properties_path).open(encoding="utf8") as fp: data = {} for line in fp.read().splitlines(): line = line.strip() diff --git a/esphome/espidf/extra_script.py b/esphome/espidf/extra_script.py index 2f22f23c10..bead63ca21 100644 --- a/esphome/espidf/extra_script.py +++ b/esphome/espidf/extra_script.py @@ -108,7 +108,7 @@ def run_extra_script( """ env = _FakeSConsEnv(board_mcu=idf_target, pio_env=f"esphome_{idf_target}") code = compile(script_path.read_text(), str(script_path), "exec") - old_cwd = os.getcwd() + old_cwd = Path.cwd() try: os.chdir(library_dir) exec( # noqa: S102 pylint: disable=exec-used diff --git a/esphome/espidf/framework.py b/esphome/espidf/framework.py index 079c97cc98..331c2f84b0 100644 --- a/esphome/espidf/framework.py +++ b/esphome/espidf/framework.py @@ -8,6 +8,7 @@ import logging import os from pathlib import Path import platform +import re import shutil import subprocess import sys @@ -39,7 +40,7 @@ def _str_to_lst_of_str(a: str | list[str]) -> list[str]: """ if isinstance(a, list): return a - return list(f.strip() for f in a.split(";") if f.strip()) + return [f.strip() for f in a.split(";") if f.strip()] ESPHOME_STAMP_FILE = ".esphome.stamp.json" @@ -137,7 +138,7 @@ def rmdir(directory: PathType, msg: str | None = None): Raises: RuntimeError: If directory removal fails """ - if os.path.isdir(directory): + if Path(directory).is_dir(): try: if msg: _LOGGER.debug(msg) @@ -191,7 +192,7 @@ def _check_stamp(file: PathType, data: dict[str, str]) -> bool: return False try: - with open(file, encoding="utf-8") as f: + with Path(file).open(encoding="utf-8") as f: return json.load(f) == data except (json.JSONDecodeError, OSError): return False @@ -205,7 +206,7 @@ def _write_stamp(file: PathType, data: dict[str, str]): file: Path to the stamp file to write data: Dictionary containing data to write """ - with open(file, "w", encoding="utf8") as fp: + with Path(file).open("w", encoding="utf8") as fp: json.dump(data, fp) @@ -470,8 +471,12 @@ def _tar_extract_all( import stat import tarfile + # Tar extraction safety: os.path.realpath / commonpath / normpath have no + # pathlib equivalents and Path.resolve() would follow symlinks unsafely. + # Use os.path for the security-sensitive parts; the simple checks move to + # Path. extract_dir = os.fspath(extract_dir) - abs_dest = os.path.abspath(extract_dir) + abs_dest = os.path.abspath(extract_dir) # noqa: PTH100 with tarfile.open(fileobj=data, mode="r") as tar_ref: all_members = tar_ref.getmembers() @@ -490,8 +495,8 @@ def _tar_extract_all( name = name.lstrip("/" + os.sep) # 2. Reject absolute paths (incl. Windows drive) - if os.path.isabs(name) or ( - os.name == "nt" and ":" in name.split(os.sep)[0] + if Path(name).is_absolute() or ( + os.name == "nt" and ":" in name.split(os.sep)[0] # noqa: PTH206 ): continue @@ -505,7 +510,7 @@ def _tar_extract_all( name = norm[len(strip_prefix) :] # 4. Compute final path - target_path = os.path.realpath(os.path.join(abs_dest, name)) + target_path = os.path.realpath(os.path.join(abs_dest, name)) # noqa: PTH118 if os.path.commonpath([abs_dest, target_path]) != abs_dest: continue @@ -514,18 +519,20 @@ def _tar_extract_all( linkname = member.linkname # Reject absolute link targets - if os.path.isabs(linkname): + if Path(linkname).is_absolute(): continue # Strip leading slashes linkname = os.path.normpath(linkname) if member.issym(): - link_target = os.path.join( - abs_dest, os.path.dirname(name), linkname + link_target = os.path.join( # noqa: PTH118 + abs_dest, + os.path.dirname(name), # noqa: PTH120 + linkname, ) else: - link_target = os.path.join(abs_dest, linkname) + link_target = os.path.join(abs_dest, linkname) # noqa: PTH118 link_target = os.path.realpath(link_target) if os.path.commonpath([abs_dest, link_target]) != abs_dest: @@ -597,7 +604,9 @@ def _zip_extract_all( """ import zipfile - extract_dir = os.path.abspath(extract_dir) + # See note in archive_extract_all_tar: os.path is used intentionally for + # the security-sensitive abspath/commonpath checks below. + extract_dir = os.path.abspath(extract_dir) # noqa: PTH100 with zipfile.ZipFile(data, "r") as zip_ref: all_members = zip_ref.infolist() @@ -617,8 +626,8 @@ def _zip_extract_all( name = member.filename.lstrip("/\\") # 2. Reject absolute paths / Windows drives - if os.path.isabs(name) or ( - os.name == "nt" and ":" in name.split(os.sep)[0] + if Path(name).is_absolute() or ( + os.name == "nt" and ":" in name.split(os.sep)[0] # noqa: PTH206 ): continue @@ -632,7 +641,7 @@ def _zip_extract_all( name = norm[len(strip_prefix) :] # 4. Compute safe target path - target_path = os.path.abspath(os.path.join(extract_dir, name)) + target_path = os.path.abspath(os.path.join(extract_dir, name)) # noqa: PTH100, PTH118 if os.path.commonpath([extract_dir, target_path]) != extract_dir: raise ValueError(f"Unsafe path detected: {member.filename}") @@ -679,7 +688,7 @@ def archive_extract_all( with ExitStack() as stack: archive_ref: io.BufferedIOBase if isinstance(archive, (str, os.PathLike)): - archive_ref = stack.enter_context(open(archive, "rb")) + archive_ref = stack.enter_context(Path(archive).open("rb")) elif isinstance(archive, (io.BufferedReader, io.BufferedRandom)): archive_ref = archive elif isinstance(archive, io.RawIOBase): @@ -726,7 +735,7 @@ def download_from_mirrors( # 1. Open target file for writing if path given with ExitStack() as stack: if isinstance(target, (str, os.PathLike)): - f = stack.enter_context(open(target, "wb")) + f = stack.enter_context(Path(target).open("wb")) elif isinstance(target, (io.RawIOBase, io.IOBase)): f = target else: @@ -784,6 +793,77 @@ def download_from_mirrors( return None +_GITHUB_SHORTHAND_RE = re.compile( + r"^github://([a-zA-Z0-9\-]+)/([a-zA-Z0-9\-\._]+?)(?:@([a-zA-Z0-9\-_.\./]+))?$" +) +_GITHUB_HTTPS_RE = re.compile( + r"^(https://github\.com/[a-zA-Z0-9\-]+/[a-zA-Z0-9\-\._]+?\.git)(?:@([a-zA-Z0-9\-_.\./]+))?$" +) + + +def _parse_git_source(source_url: str) -> tuple[str, str | None] | None: + """Return ``(url, ref)`` for ``github://owner/repo[@ref]`` or + ``https://github.com/owner/repo.git[@ref]``, else ``None``.""" + if m := _GITHUB_SHORTHAND_RE.match(source_url): + owner, repo, ref = m.group(1), m.group(2), m.group(3) + # Tolerate a trailing ".git" on the shorthand repo so the + # github://owner/repo.git form doesn't silently become repo.git.git. + repo = repo.removesuffix(".git") + return f"https://github.com/{owner}/{repo}.git", ref + if m := _GITHUB_HTTPS_RE.match(source_url): + return m.group(1), m.group(2) + return None + + +def _clone_idf_with_submodules( + framework_path: Path, git_url: str, ref: str | None +) -> None: + """Shallow-clone ESP-IDF with submodules into ``framework_path``. + + GitHub's archive zip strips submodules, so vendored components + (mbedtls, openthread, esptool, ...) come down empty and CMake fails. + + Uses clone + ``fetch FETCH_HEAD`` + ``reset --hard`` instead of + ``--branch``: ``--branch`` only accepts branch or tag names, but a + user can also point at a commit SHA. The fetch-then-reset pattern + handles branches, tags, and SHAs uniformly (mirrors the approach in + ``esphome.git.clone_or_update``). + """ + from esphome.git import run_git_command + + _LOGGER.info("Cloning ESP-IDF from %s%s", git_url, f"@{ref}" if ref else "") + run_git_command(["git", "clone", "--depth=1", "--", git_url, str(framework_path)]) + if ref: + run_git_command( + ["git", "fetch", "--depth=1", "--", "origin", ref], + git_dir=framework_path, + ) + run_git_command( + ["git", "reset", "--hard", "FETCH_HEAD"], + git_dir=framework_path, + ) + run_git_command( + [ + "git", + "submodule", + "update", + "--init", + "--recursive", + "--depth=1", + ], + git_dir=framework_path, + ) + + # Sanity-check the resulting tree. run_git_command only raises when + # stderr is non-empty, so a clone that silently produces no working + # tree would otherwise be marked extracted and stuck until + # ``esphome clean``. + if not (framework_path / "tools" / "idf_tools.py").is_file(): + raise RuntimeError( + f"Clone of {git_url} produced no usable ESP-IDF tree at {framework_path}" + ) + + def _write_idf_version_txt(framework_path: Path, version: str) -> None: """Write /version.txt if missing. @@ -845,7 +925,7 @@ def _patch_tools_json_for_linux_arm64(framework_path: Path) -> None: return try: - with open(tools_json, encoding="utf-8") as f: + with tools_json.open(encoding="utf-8") as f: data = json.load(f) except (json.JSONDecodeError, OSError) as e: _LOGGER.warning( @@ -939,27 +1019,34 @@ def _check_esphome_idf_framework_install( if install: rmdir(framework_path, msg=f"Clean up ESP-IDF {version} framework") - # Download in temporary file - with tempfile.NamedTemporaryFile() as tmp: - _LOGGER.info("Downloading ESP-IDF %s framework ...", version) + git_source = _parse_git_source(source_url) if source_url else None + if git_source is not None: + git_url, ref = git_source + _clone_idf_with_submodules(framework_path, git_url, ref) + else: + # Download in temporary file + with tempfile.NamedTemporaryFile() as tmp: + _LOGGER.info("Downloading ESP-IDF %s framework ...", version) - # Create substitutions for the URLs - substitutions = {"VERSION": version} - try: - ver = Version.parse(version) - substitutions["MAJOR"] = str(ver.major) - substitutions["MINOR"] = str(ver.minor) - substitutions["PATCH"] = str(ver.patch) - substitutions["EXTRA"] = ver.extra - except ValueError: - pass + # Create substitutions for the URLs + substitutions = {"VERSION": version} + try: + ver = Version.parse(version) + substitutions["MAJOR"] = str(ver.major) + substitutions["MINOR"] = str(ver.minor) + substitutions["PATCH"] = str(ver.patch) + substitutions["EXTRA"] = ver.extra + except ValueError: + pass - mirrors = [source_url] if source_url else ESPHOME_IDF_FRAMEWORK_MIRRORS - download_from_mirrors(mirrors, substitutions, tmp.file) + mirrors = [source_url] if source_url else ESPHOME_IDF_FRAMEWORK_MIRRORS + download_from_mirrors(mirrors, substitutions, tmp.file) - _LOGGER.info("Extracting ESP-IDF %s framework ...", version) - archive_extract_all(tmp.file, framework_path, progress_header="Extracting") - extracted_marker.touch() + _LOGGER.info("Extracting ESP-IDF %s framework ...", version) + archive_extract_all( + tmp.file, framework_path, progress_header="Extracting" + ) + extracted_marker.touch() # Idempotent post-extract patch: written every invocation so a build # dir extracted before this fix gets the file too, without forcing a diff --git a/esphome/espidf/get_idf_tool_paths.py b/esphome/espidf/get_idf_tool_paths.py index 2e8859631d..7d99e629b1 100644 --- a/esphome/espidf/get_idf_tool_paths.py +++ b/esphome/espidf/get_idf_tool_paths.py @@ -10,6 +10,7 @@ not installed. import json import os +from pathlib import Path import sys from types import SimpleNamespace @@ -25,7 +26,7 @@ from idf_tools import ( g.idf_path = sys.argv[1] g.idf_tools_path = os.environ.get("IDF_TOOLS_PATH") -g.tools_json = os.path.join(g.idf_path, TOOLS_FILE) +g.tools_json = str(Path(g.idf_path) / TOOLS_FILE) tools_info = filter_tools_info(IDFEnv.get_idf_env(), load_tools_info()) args = SimpleNamespace(prefer_system=False) diff --git a/esphome/espidf/runner.py b/esphome/espidf/runner.py index da3f77cdd3..7c568db7be 100644 --- a/esphome/espidf/runner.py +++ b/esphome/espidf/runner.py @@ -91,6 +91,7 @@ def main() -> int: # ---- end sys.path fix-up ----------------------------------------------- import os + from pathlib import Path import re import runpy @@ -164,6 +165,12 @@ def main() -> int: self._line_buffer = "" def __getattr__(self, name: str): + # Hide ``buffer`` so consumers that use either + # ``getattr(stream, 'buffer', None)`` or + # ``hasattr(stream, 'buffer')`` see this as a text-only stream + # and skip writing raw bytes (which would bypass the filter). + if name == "buffer": + raise AttributeError(name) return getattr(self._stream, name) def isatty(self) -> bool: @@ -223,7 +230,7 @@ def main() -> int: # runpy.run_path does not do this automatically, but idf.py relies # on it to import its sibling modules (python_version_checker, # idf_py_actions, ...). - script_dir = os.path.dirname(os.path.abspath(script_path)) + script_dir = str(Path(script_path).resolve().parent) if script_dir not in sys.path: sys.path.insert(0, script_dir) diff --git a/esphome/espidf/toolchain.py b/esphome/espidf/toolchain.py index ef28575caa..752f582e74 100644 --- a/esphome/espidf/toolchain.py +++ b/esphome/espidf/toolchain.py @@ -241,20 +241,21 @@ def has_outdated_files(): dependency_lock_path = CORE.relative_build_path("dependencies.lock") build_ninja_path = CORE.relative_build_path("build/build.ninja") - if not os.path.isdir(build_config_path) or not os.listdir(build_config_path): + if not build_config_path.is_dir() or not any(build_config_path.iterdir()): return True - if not os.path.isfile(cmakecache_txt_path): + if not cmakecache_txt_path.is_file(): return True - if not os.path.isfile(build_ninja_path): + if not build_ninja_path.is_file(): return True - if os.path.isfile(dependency_lock_path) and os.path.getmtime( - dependency_lock_path - ) > os.path.getmtime(build_ninja_path): + if ( + dependency_lock_path.is_file() + and dependency_lock_path.stat().st_mtime > build_ninja_path.stat().st_mtime + ): return True - cmakecache_txt_mtime = os.path.getmtime(cmakecache_txt_path) + cmakecache_txt_mtime = cmakecache_txt_path.stat().st_mtime return any( - os.path.getmtime(f) > cmakecache_txt_mtime + f.stat().st_mtime > cmakecache_txt_mtime for f in [sdkconfig_internal_path, idf_component_yml_path] if f.exists() ) @@ -452,7 +453,7 @@ def create_factory_bin() -> bool: return False try: - with open(flasher_args_path, encoding="utf-8") as f: + with flasher_args_path.open(encoding="utf-8") as f: flash_data = json.load(f) except (json.JSONDecodeError, OSError) as e: _LOGGER.error("Failed to read flasher_args.json: %s", e) diff --git a/esphome/espota2.py b/esphome/espota2.py index 701a125bcd..266702c142 100644 --- a/esphome/espota2.py +++ b/esphome/espota2.py @@ -517,7 +517,7 @@ def run_ota_impl_( continue _LOGGER.info("Connected to %s", sa[0]) - with open(filename, "rb") as file_handle: + with Path(filename).open("rb") as file_handle: try: perform_ota(sock, password, file_handle, filename, ota_type) except OTAError as err: diff --git a/esphome/external_files.py b/esphome/external_files.py index dfabc54f47..4e73c8dc21 100644 --- a/esphome/external_files.py +++ b/esphome/external_files.py @@ -7,6 +7,7 @@ from datetime import UTC, datetime import logging import os from pathlib import Path +import time import requests @@ -141,9 +142,11 @@ def has_remote_file_changed( def is_file_recent(file_path: Path, refresh: TimePeriodSeconds) -> bool: if file_path.exists(): - creation_time = file_path.stat().st_ctime - current_time = datetime.now().timestamp() - return current_time - creation_time <= refresh.total_seconds + # st_mtime, not st_ctime: ctime is inode-change time on POSIX + # (bumped by chmod/chown/rename) so a metadata touch would make + # the file look fresh. + modification_time = file_path.stat().st_mtime + return time.time() - modification_time <= refresh.total_seconds return False diff --git a/esphome/git.py b/esphome/git.py index 0106f24845..744ce35ef6 100644 --- a/esphome/git.py +++ b/esphome/git.py @@ -1,11 +1,12 @@ from collections.abc import Callable from dataclasses import dataclass -from datetime import datetime import hashlib import logging from pathlib import Path import re import subprocess +import sys +import time import urllib.parse import esphome.config_validation as cv @@ -72,8 +73,9 @@ def run_git_command(cmd: list[str], git_dir: Path | None = None) -> str: ) except FileNotFoundError as err: raise GitNotInstalledError( - "git is not installed but required for external_components.\n" - "Please see https://git-scm.com/book/en/v2/Getting-Started-Installing-Git for installing git" + "git is not installed. See " + "https://git-scm.com/book/en/v2/Getting-Started-Installing-Git " + "for installation instructions." ) from err if ret.returncode != 0 and ret.stderr: @@ -93,6 +95,92 @@ def _compute_destination_path(key: str, domain: str) -> Path: return base_dir / h.hexdigest()[:8] +def resolve_symlink_stub(repo_dir: Path, file_path: Path) -> Path | None: + """Return the symlink target if ``file_path`` is a Windows-checked-out symlink stub. + + On Windows, when ``core.symlinks=false`` (the default unless the user has + SeCreateSymbolicLinkPrivilege — i.e. Developer Mode or running elevated), + git materializes files with tree mode ``120000`` as plain text files + whose content is the literal symlink target path. Opening such a file + yields the target path string instead of the target's content. + + If ``file_path`` is one of those stubs, return the resolved target Path + inside ``repo_dir``. Otherwise return ``None`` and the caller should use + ``file_path`` as-is. + + Designed to be called *only* when normal access has already produced an + unexpected result (e.g. YAML parsed as a top-level scalar), so the + per-file ``git ls-files`` subprocess cost is paid only on the failure + path. Returns ``None`` on any error or check failure — it's purely a + best-effort recovery, never raises. + """ + # On non-Windows, git creates real symlinks; ordinary file access already + # transparently follows them. + if sys.platform != "win32": + return None + if file_path.is_symlink(): + return None + if not file_path.is_file(): + return None + + try: + rel = file_path.relative_to(repo_dir) + except ValueError: + return None + + try: + # ``git ls-files -s `` prints " \t" + # for that single entry, or empty if untracked. + out = run_git_command( + ["git", "ls-files", "-s", "--", rel.as_posix()], + git_dir=repo_dir, + ) + except GitException: + return None + + parts = out.split() + if not parts or parts[0] != "120000": + return None + + # Stubs are short ASCII relative paths. Decode defensively, and only + # strip the trailing newline git's checkout may append — preserving any + # whitespace that could be part of a valid target name. + try: + raw = file_path.read_bytes() + except OSError: + return None + try: + target_str = raw.decode("utf-8").rstrip("\r\n") + except UnicodeDecodeError: + return None + + # ``Path()`` and ``Path.resolve()`` can raise on malformed inputs (e.g. + # embedded NUL bytes from a hostile symlink blob, paths too long for the + # OS, or temporary I/O errors). Catch broadly — this helper is purely a + # best-effort recovery and must never raise. + try: + target_path = (file_path.parent / target_str).resolve() + repo_root_resolved = repo_dir.resolve() + except (OSError, ValueError, RuntimeError): + return None + + # ``Path.resolve()`` follows ``..``; re-verify containment afterwards. + try: + target_path.relative_to(repo_root_resolved) + except ValueError: + _LOGGER.warning( + "Refusing to follow symlink %s -> %s (escapes repository)", + file_path, + target_str, + ) + return None + + if not target_path.is_file(): + return None + + return target_path + + def clone_or_update( *, url: str, @@ -159,11 +247,11 @@ def clone_or_update( return repo_dir, None file_timestamp = Path(repo_dir / ".git" / "FETCH_HEAD") - # On first clone, FETCH_HEAD does not exists + # On first clone, FETCH_HEAD does not exist if not file_timestamp.exists(): file_timestamp = Path(repo_dir / ".git" / "HEAD") - age = datetime.now() - datetime.fromtimestamp(file_timestamp.stat().st_mtime) - if refresh is None or age.total_seconds() > refresh.total_seconds: + age_seconds = time.time() - file_timestamp.stat().st_mtime + if refresh is None or age_seconds > refresh.total_seconds: # Try to update the repository, recovering from broken state if needed old_sha: str | None = None try: diff --git a/esphome/helpers.py b/esphome/helpers.py index d7ddb5c416..733474c9c9 100644 --- a/esphome/helpers.py +++ b/esphome/helpers.py @@ -385,7 +385,7 @@ def rmtree(path: Path | str) -> None: def _onerror(func, path, exc_info): if os.access(path, os.W_OK): raise exc_info[1].with_traceback(exc_info[2]) - os.chmod(path, stat.S_IWUSR | stat.S_IRUSR) + Path(path).chmod(stat.S_IWUSR | stat.S_IRUSR) func(path) # ``onerror`` is deprecated in 3.12 in favour of ``onexc`` (different @@ -512,7 +512,7 @@ def copy_file_if_changed(src: Path, dst: Path) -> bool: # -> delete file (it would be overwritten anyway), and try again # if that fails, use normal error handler with suppress(OSError): - os.unlink(dst) + Path(dst).unlink() shutil.copyfile(src, dst) return True diff --git a/esphome/idf_component.yml b/esphome/idf_component.yml index 6bc166ff44..5af25fc351 100644 --- a/esphome/idf_component.yml +++ b/esphome/idf_component.yml @@ -36,7 +36,7 @@ dependencies: rules: - if: "target in [esp32h2, esp32p4]" espressif/esp_hosted: - version: 2.12.7 + version: 2.12.8 rules: - if: "target in [esp32h2, esp32p4]" zorxx/multipart-parser: diff --git a/esphome/loader.py b/esphome/loader.py index d50554f8c9..8823d82fc1 100644 --- a/esphome/loader.py +++ b/esphome/loader.py @@ -12,7 +12,6 @@ from types import ModuleType from typing import TYPE_CHECKING, Any from esphome.const import SOURCE_FILE_EXTENSIONS -from esphome.core import CORE from esphome.types import ConfigType if TYPE_CHECKING: @@ -206,18 +205,6 @@ def install_meta_finder( sys.meta_path.insert(0, ComponentMetaFinder(components_path, allowed_components)) -def install_custom_components_meta_finder(): - # Remove before 2026.6.0 - custom_components_dir = (Path(CORE.config_dir) / "custom_components").resolve() - if custom_components_dir.is_dir() and any(custom_components_dir.iterdir()): - _LOGGER.warning( - "The 'custom_components' folder is deprecated and will be removed in 2026.6.0. " - "Please use 'external_components' instead. " - "See https://esphome.io/components/external_components.html for more information." - ) - install_meta_finder(custom_components_dir) - - def _lookup_module(domain: str, exception: bool) -> ComponentManifest | None: if domain in _COMPONENT_CACHE: return _COMPONENT_CACHE[domain] @@ -239,12 +226,12 @@ def _lookup_module(domain: str, exception: bool) -> ComponentManifest | None: "Unable to import component %s: %s", domain, str(e), exc_info=False ) else: - _LOGGER.error("Unable to import component %s:", domain, exc_info=True) + _LOGGER.exception("Unable to import component %s:", domain) return None except Exception: # pylint: disable=broad-except if exception: raise - _LOGGER.error("Unable to load component %s:", domain, exc_info=True) + _LOGGER.exception("Unable to load component %s:", domain) return None manif = ComponentManifest(module) diff --git a/esphome/log.py b/esphome/log.py index bfd1875b55..b120c930d0 100644 --- a/esphome/log.py +++ b/esphome/log.py @@ -28,10 +28,12 @@ class AnsiFore(Enum): class AnsiStyle(Enum): + # BOLD/BRIGHT and THIN/DIM are intentional ANSI synonyms; Enum treats the + # second name in each pair as an alias of the first. BRIGHT = "\033[1m" - BOLD = "\033[1m" + BOLD = "\033[1m" # noqa: PIE796 DIM = "\033[2m" - THIN = "\033[2m" + THIN = "\033[2m" # noqa: PIE796 NORMAL = "\033[22m" RESET_ALL = "\033[0m" diff --git a/esphome/mqtt.py b/esphome/mqtt.py index ccacbaea54..c6a7a7558b 100644 --- a/esphome/mqtt.py +++ b/esphome/mqtt.py @@ -2,7 +2,7 @@ import contextlib from datetime import datetime import json import logging -import os +from pathlib import Path import ssl import tempfile import time @@ -120,8 +120,8 @@ def prepare( key_file.close() context.load_cert_chain(cert_file.name, key_file.name) finally: - os.unlink(cert_file.name) - os.unlink(key_file.name) + Path(cert_file.name).unlink() + Path(key_file.name).unlink() client.tls_set_context(context) try: @@ -139,7 +139,7 @@ def show_discover(config, username=None, password=None, client_id=None): _LOGGER.info("Starting log output from %s", topic) def on_message(client, userdata, msg): - time_ = datetime.now().time().strftime("[%H:%M:%S]") + time_ = datetime.now().astimezone().time().strftime("[%H:%M:%S]") payload = msg.payload.decode(errors="backslashreplace") if len(payload) > 0: message = time_ + " " + payload @@ -159,7 +159,7 @@ def get_esphome_device_ip( username: str | None = None, password: str | None = None, client_id: str | None = None, - timeout: int | float = 25, + timeout: float = 25, ) -> list[str]: if CONF_MQTT not in config: raise EsphomeError( @@ -184,7 +184,7 @@ def get_esphome_device_ip( def on_message(client, userdata, msg): nonlocal dev_ip - time_ = datetime.now().time().strftime("[%H:%M:%S]") + time_ = datetime.now().astimezone().time().strftime("[%H:%M:%S]") payload = msg.payload.decode(errors="backslashreplace") if len(payload) > 0: message = time_ + " " + payload @@ -253,7 +253,7 @@ def show_logs(config, topic=None, username=None, password=None, client_id=None): _LOGGER.info("Starting log output from %s", topic) def on_message(client, userdata, msg): - time_ = datetime.now().time().strftime("[%H:%M:%S]") + time_ = datetime.now().astimezone().time().strftime("[%H:%M:%S]") payload = msg.payload.decode(errors="backslashreplace") message = time_ + payload safe_print(message) diff --git a/esphome/pins.py b/esphome/pins.py index bdaa0e28ab..d6393508ab 100644 --- a/esphome/pins.py +++ b/esphome/pins.py @@ -272,9 +272,10 @@ def check_strapping_pin(conf, strapping_pin_list: set[int], logger: Logger): num = conf[CONF_NUMBER] if num in strapping_pin_list and not conf.get(CONF_IGNORE_STRAPPING_WARNING): logger.warning( - f"GPIO{num} is a strapping PIN and should only be used for I/O with care.\n" + "GPIO%s is a strapping PIN and should only be used for I/O with care.\n" "Attaching external pullup/down resistors to strapping pins can cause unexpected failures.\n" "See https://esphome.io/guides/faq/#why-am-i-getting-a-warning-about-strapping-pins", + num, ) # mitigate undisciplined use of strapping: if num not in strapping_pin_list and conf.get(CONF_IGNORE_STRAPPING_WARNING): @@ -313,9 +314,7 @@ def gpio_base_schema( :return: A schema for the pin """ mode_default = len(modes) == 1 - mode_dict = dict( - map(lambda m: (cv.Optional(m, default=mode_default), cv.boolean), modes) - ) + mode_dict = {cv.Optional(m, default=mode_default): cv.boolean for m in modes} def _number_validator(value): if isinstance(value, str) and value.upper().startswith("GPIOX"): diff --git a/esphome/platformio/toolchain.py b/esphome/platformio/toolchain.py index 073e134ac4..c81420e6ca 100644 --- a/esphome/platformio/toolchain.py +++ b/esphome/platformio/toolchain.py @@ -96,7 +96,7 @@ def _run_idedata(config): try: return json.loads(match.group()) except ValueError: - _LOGGER.error("Could not parse idedata", exc_info=True) + _LOGGER.exception("Could not parse idedata") _LOGGER.error("Stdout: %s", stdout) raise diff --git a/esphome/storage_json.py b/esphome/storage_json.py index 7f8885ba5f..04f5881465 100644 --- a/esphome/storage_json.py +++ b/esphome/storage_json.py @@ -338,7 +338,10 @@ class EsphomeStorageJSON: @property def last_update_check(self) -> datetime | None: try: - return datetime.strptime(self.last_update_check_str, "%Y-%m-%dT%H:%M:%S") + # Stored format is naive ISO without %z; preserved for backward compat. + return datetime.strptime( # noqa: DTZ007 + self.last_update_check_str, "%Y-%m-%dT%H:%M:%S" + ) except Exception: # pylint: disable=broad-except return None diff --git a/esphome/upload_targets.py b/esphome/upload_targets.py index 302ecf7301..d9d9713fc1 100644 --- a/esphome/upload_targets.py +++ b/esphome/upload_targets.py @@ -57,7 +57,7 @@ def get_port_type(port: str) -> PortType: """ if port == "BOOTSEL": return PortType.BOOTSEL - if port.startswith("/") or port.startswith("COM"): + if port.startswith(("/", "COM")): return PortType.SERIAL if port == "MQTT": return PortType.MQTT diff --git a/esphome/web_server_ota.py b/esphome/web_server_ota.py index 7c31c1b123..8d0fdeecff 100644 --- a/esphome/web_server_ota.py +++ b/esphome/web_server_ota.py @@ -126,7 +126,7 @@ def _try_upload( _LOGGER.info("Connecting to %s port %s...", ip, port) try: - with open(filename, "rb") as fh: + with filename.open("rb") as fh: streamer = _MultipartStreamer(fh, file_size, filename.name) try: response = requests.post( diff --git a/esphome/writer.py b/esphome/writer.py index ad3877465d..ef7cbf5ac4 100644 --- a/esphome/writer.py +++ b/esphome/writer.py @@ -200,8 +200,8 @@ ESPHome automatically populates the build directory, and any changes to this directory will be removed the next time esphome is run. -For modifying esphome's core files, please use a development esphome install, -the custom_components folder or the external_components feature. +For modifying esphome's core files, please use a development esphome install +or the external_components feature. """ @@ -358,7 +358,7 @@ def copy_src_tree(): platform = "esphome.components." + CORE.target_platform try: module = importlib.import_module(platform) - copy_files = getattr(module, "copy_files") + copy_files = module.copy_files copy_files() except AttributeError: pass diff --git a/esphome/yaml_util.py b/esphome/yaml_util.py index 9a36ad089c..bfe1fb0136 100644 --- a/esphome/yaml_util.py +++ b/esphome/yaml_util.py @@ -52,6 +52,16 @@ _load_listeners: list[Callable[[Path], None]] = [] DocumentPath = list[str | int] +class SensitiveStr(str): + """Marker subclass for validated strings that should be masked in + user-visible YAML output. ``cv.sensitive`` wraps validated values in this + type so ``dump()`` can render them with ANSI conceal codes without + needing a post-process regex. + """ + + __slots__ = () + + @contextmanager def track_yaml_loads() -> Generator[list[Path]]: """Context manager that records every file loaded by the YAML loader. @@ -763,7 +773,7 @@ def parse_yaml(file_name: Path, file_handle: TextIOWrapper, yaml_loader=None) -> def _load_yaml_internal_with_type( - loader_type: type[ESPHomeLoader] | type[ESPHomePurePythonLoader], + loader_type: type[ESPHomeLoader | ESPHomePurePythonLoader], fname: Path, content: TextIOWrapper, yaml_loader: Callable[[Path], dict[str, Any]], @@ -808,11 +818,18 @@ def dump(dict_, show_secrets=False, sort_keys=False): if show_secrets: _SECRET_VALUES.clear() _SECRET_CACHE.clear() + + # Per-call subclass so the redaction flag doesn't leak across calls. + # (``_SECRET_VALUES`` / ``_SECRET_CACHE`` remain module globals; YAML + # processing is single-threaded today, so this isolates only the flag.) + class _Dumper(ESPHomeDumper): + _redact_sensitive = not show_secrets + return yaml.dump( dict_, default_flow_style=False, allow_unicode=True, - Dumper=ESPHomeDumper, + Dumper=_Dumper, sort_keys=sort_keys, ) @@ -958,6 +975,10 @@ def format_path(path: DocumentPath, current_obj: Any) -> str: class ESPHomeDumper(yaml.SafeDumper): + # Default for the base class; per-call subclass in ``dump()`` overrides. + # When True, ``represent_sensitive`` wraps values in ANSI conceal codes. + _redact_sensitive: bool = False + def represent_mapping(self, tag, mapping, flow_style=None): value = [] node = yaml.MappingNode(tag, value, flow_style=flow_style) @@ -992,6 +1013,20 @@ class ESPHomeDumper(yaml.SafeDumper): return self.represent_secret(value) return self.represent_scalar(tag="tag:yaml.org,2002:str", value=str(value)) + def represent_sensitive(self, value: SensitiveStr) -> yaml.ScalarNode: + # Only the redact-and-not-a-secret branch is unique to sensitive + # values; otherwise let ``represent_stringify`` handle ``!secret`` + # precedence and the plain-str fallthrough. Conceal sequence is + # emitted as literal ``\033`` text (not actual ESC bytes) so the + # output matches the prior regex format and device-builder's + # ``\033[8m...\033[28m`` parser keeps working. + if self._redact_sensitive and not is_secret(value): + return self.represent_scalar( + tag="tag:yaml.org,2002:str", + value=f"\\033[8m{value}\\033[28m", + ) + return self.represent_stringify(value) + # pylint: disable=arguments-renamed def represent_bool(self, value): return self.represent_scalar( @@ -1063,6 +1098,8 @@ ESPHomeDumper.add_multi_representer( ) ESPHomeDumper.add_multi_representer(bool, ESPHomeDumper.represent_bool) ESPHomeDumper.add_multi_representer(str, ESPHomeDumper.represent_stringify) +# MRO-walked dispatch; SensitiveStr's own entry wins over the str one. +ESPHomeDumper.add_multi_representer(SensitiveStr, ESPHomeDumper.represent_sensitive) ESPHomeDumper.add_multi_representer(int, ESPHomeDumper.represent_int) ESPHomeDumper.add_multi_representer(float, ESPHomeDumper.represent_float) ESPHomeDumper.add_multi_representer(_BaseAddress, ESPHomeDumper.represent_stringify) diff --git a/esphome/zeroconf.py b/esphome/zeroconf.py index 5d922ea911..a4f4f46097 100644 --- a/esphome/zeroconf.py +++ b/esphome/zeroconf.py @@ -249,7 +249,7 @@ async def async_resolve_hosts( ), return_exceptions=True, ) - for host, result in zip(pending, results): + for host, result in zip(pending, results, strict=True): if isinstance(result, BaseException): _LOGGER.debug("Failed to resolve %s: %s", host, result) diff --git a/pyproject.toml b/pyproject.toml index d16bf2b625..d2f30ea3d7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -111,16 +111,36 @@ exclude = ['generated'] [tool.ruff.lint] select = [ + "B", # flake8-bugbear + "C4", # flake8-comprehensions + "DTZ", # flake8-datetimez "E", # pycodestyle + "EXE", # flake8-executable "F", # pyflakes/autoflake + "FA", # flake8-future-annotations "FLY", # flynt: convert string formatting to f-strings "FURB", # refurb + "G", # flake8-logging-format "I", # isort + "ICN", # flake8-import-conventions + "ISC", # flake8-implicit-str-concat + "LOG", # flake8-logging + "NPY", # numpy-specific rules "PERF", # performance + "PGH", # pygrep-hooks + "PIE", # flake8-pie "PL", # pylint + "PTH", # flake8-use-pathlib + "PYI", # flake8-pyi + "Q", # flake8-quotes + "RSE", # flake8-raise "SIM", # flake8-simplify + "SLOT", # flake8-slots "RET", # flake8-ret + "T10", # flake8-debugger "UP", # pyupgrade + "W", # pycodestyle warnings + "YTT", # flake8-2020 ] ignore = [ diff --git a/requirements.txt b/requirements.txt index 178e05497f..14dddbb1aa 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,7 +12,7 @@ platformio==6.1.19 esptool==5.2.0 click==8.3.3 esphome-dashboard==20260425.0 -aioesphomeapi==45.0.4 +aioesphomeapi==45.3.1 zeroconf==0.149.16 puremagic==1.30 ruamel.yaml==0.19.1 # dashboard_import diff --git a/requirements_test.txt b/requirements_test.txt index 102a9cae6e..aad1da0807 100644 --- a/requirements_test.txt +++ b/requirements_test.txt @@ -8,7 +8,7 @@ pre-commit pytest==9.0.3 pytest-cov==7.1.0 pytest-mock==3.15.1 -pytest-asyncio==1.3.0 +pytest-asyncio==1.4.0 pytest-xdist==3.8.0 asyncmock==0.4.2 hypothesis==6.92.1 diff --git a/script/api_protobuf/api_protobuf.py b/script/api_protobuf/api_protobuf.py index 5c4a75c64a..04d6d159b9 100755 --- a/script/api_protobuf/api_protobuf.py +++ b/script/api_protobuf/api_protobuf.py @@ -84,12 +84,7 @@ def indent_list(text: str, padding: str = " ") -> list[str]: """Indent each line of the given text with the specified padding.""" lines = [] for line in text.splitlines(): - if ( - line == "" - or line.startswith("#ifdef") - or line.startswith("#if ") - or line.startswith("#endif") - ): + if line == "" or line.startswith(("#ifdef", "#if ", "#endif")): p = "" else: p = padding @@ -1283,11 +1278,11 @@ class PackedBufferTypeInfo(TypeInfo): """Dump shows buffer info but not decoded values.""" return ( f'out.append(2, \' \').append_p(ESPHOME_PSTR("{self.name}")).append(": ");\n' - + 'out.append_p(ESPHOME_PSTR("packed buffer ["));\n' - + f"append_uint(out, this->{self.field_name}_count_);\n" - + 'out.append_p(ESPHOME_PSTR(" values, "));\n' - + f"append_uint(out, this->{self.field_name}_length_);\n" - + 'out.append_p(ESPHOME_PSTR(" bytes]\\n"));' + 'out.append_p(ESPHOME_PSTR("packed buffer ["));\n' + f"append_uint(out, this->{self.field_name}_count_);\n" + 'out.append_p(ESPHOME_PSTR(" values, "));\n' + f"append_uint(out, this->{self.field_name}_length_);\n" + 'out.append_p(ESPHOME_PSTR(" bytes]\\n"));' ) def dump(self, name: str) -> str: @@ -3163,7 +3158,7 @@ def main() -> None: defines_content += "\n" defines_content += "\nnamespace esphome::api {} // namespace esphome::api\n" - with open(root / "api_pb2_defines.h", "w", encoding="utf-8") as f: + with (root / "api_pb2_defines.h").open("w", encoding="utf-8") as f: f.write(defines_content) content = FILE_HEADER @@ -3448,13 +3443,13 @@ static void dump_bytes_field(DumpBuffer &out, const char *field_name, const uint #endif // HAS_PROTO_MESSAGE_DUMP """ - with open(root / "api_pb2.h", "w", encoding="utf-8") as f: + with (root / "api_pb2.h").open("w", encoding="utf-8") as f: f.write(content) - with open(root / "api_pb2.cpp", "w", encoding="utf-8") as f: + with (root / "api_pb2.cpp").open("w", encoding="utf-8") as f: f.write(cpp) - with open(root / "api_pb2_dump.cpp", "w", encoding="utf-8") as f: + with (root / "api_pb2_dump.cpp").open("w", encoding="utf-8") as f: f.write(dump_cpp) hpp = FILE_HEADER @@ -3551,7 +3546,7 @@ static const char *const TAG = "api.service"; if id_ is not None and not mt.options.deprecated: id_to_msg_name[id_] = mt.name - for id_, (_, _, case_label) in cases: + for id_, (_, _, _case_label) in cases: msg_name = id_to_msg_name.get(id_, "") if msg_name in message_auth_map: needs_auth = message_auth_map[msg_name] @@ -3614,7 +3609,7 @@ static const char *const TAG = "api.service"; # Dispatch switch out += " switch (msg_type) {\n" - for i, (case, ifdef, case_label) in cases: + for _i, (case, ifdef, case_label) in cases: if ifdef is not None: out += _make_ifdef_line(ifdef) + "\n" @@ -3641,10 +3636,10 @@ static const char *const TAG = "api.service"; } // namespace esphome::api """ - with open(root / "api_pb2_service.h", "w", encoding="utf-8") as f: + with (root / "api_pb2_service.h").open("w", encoding="utf-8") as f: f.write(hpp) - with open(root / "api_pb2_service.cpp", "w", encoding="utf-8") as f: + with (root / "api_pb2_service.cpp").open("w", encoding="utf-8") as f: f.write(cpp) prot_file.unlink() diff --git a/script/build_helpers.py b/script/build_helpers.py index fa722aa099..52f7ee317e 100644 --- a/script/build_helpers.py +++ b/script/build_helpers.py @@ -195,7 +195,7 @@ def load_component_yaml_configs(components: list[str], tests_dir: Path) -> dict: yaml_path = tests_dir / component / BENCHMARK_YAML_FILENAME if not yaml_path.is_file(): continue - with open(yaml_path) as f: + with yaml_path.open() as f: component_config = yaml.safe_load(f) if component_config and isinstance(component_config, dict): for key, value in component_config.items(): diff --git a/script/build_language_schema.py b/script/build_language_schema.py index 921ee9d3d7..6e4000e06e 100755 --- a/script/build_language_schema.py +++ b/script/build_language_schema.py @@ -39,7 +39,11 @@ parser.add_argument( ) parser.add_argument("--check", action="store_true", help="Check only for CI") -args = parser.parse_args() +# Module-level ``Namespace`` so helper functions can reference ``args`` +# without threading it through every call. ``main()`` fills it via +# ``parser.parse_args(namespace=args)``; tests import this module without +# invoking ``main()`` and rely on the defaults below. +args = argparse.Namespace(output_path=".", check=False) DUMP_RAW = False DUMP_UNKNOWN = False @@ -850,6 +854,12 @@ def convert(schema, config_var, path): convert(ext, config_var, f"{path}/ext{idx}") return + if isinstance(schema, cv.SensitiveValidator): + config_var["sensitive"] = True + config_var["sensitive_source"] = "explicit" + convert(schema.inner, config_var, f"{path}/sensitive") + return + if isinstance(schema, cv.All): i = 0 for inner in schema.validators: @@ -972,7 +982,7 @@ def convert(schema, config_var, path): } elif schema_type == "use_id": if inspect.ismodule(data): - m_attr_obj = getattr(data, "CONFIG_SCHEMA") + m_attr_obj = data.CONFIG_SCHEMA use_schema = known_schemas.get(repr(m_attr_obj)) if use_schema: [output_module, output_name] = use_schema[0][1].split(".") @@ -1125,6 +1135,25 @@ def convert_keys(converted, schema, path): # Do value convert(v, result, path + f"/{str(k)}") + + # Heuristic fallback when the field's validator wasn't explicitly + # wrapped in ``cv.sensitive``. Only applies to string-typed leaves so + # we don't mark unrelated nested schemas. ``sensitive_source`` lets + # consumers distinguish explicit markers from heuristic matches. Pull + # the field name from ``k.schema`` (voluptuous's stored key) rather + # than ``str(k)`` so we don't depend on the marker's ``__str__`` + # representation. + if ( + "sensitive" not in result + and result.get(S_TYPE) == "string" + and isinstance(k, (cv.Required, cv.Optional, cv.Inclusive, cv.Exclusive)) + and isinstance(k.schema, str) + ): + key_lower = k.schema.lower() + if any(frag in key_lower for frag in cv.SENSITIVE_KEY_FRAGMENTS): + result["sensitive"] = True + result["sensitive_source"] = "heuristic" + if "schema" not in converted: converted[S_TYPE] = "schema" converted["schema"] = {S_CONFIG_VARS: {}} @@ -1142,4 +1171,10 @@ def convert_keys(converted, schema, path): config_vars["string"] = config_vars.pop(key) -build_schema() +def main() -> None: + parser.parse_args(namespace=args) + build_schema() + + +if __name__ == "__main__": + main() diff --git a/script/bump-version.py b/script/bump-version.py index ed927cb991..e09fc87c60 100755 --- a/script/bump-version.py +++ b/script/bump-version.py @@ -2,6 +2,7 @@ import argparse from dataclasses import dataclass +from pathlib import Path import re import sys @@ -39,12 +40,12 @@ class Version: def sub(path, pattern, repl, expected_count=1): - with open(path, encoding="utf-8") as fh: + with Path(path).open(encoding="utf-8") as fh: content = fh.read() content, count = re.subn(pattern, repl, content, flags=re.MULTILINE) if expected_count is not None: assert count == expected_count, f"Pattern {pattern} replacement failed!" - with open(path, "w", encoding="utf-8") as fh: + with Path(path).open("w", encoding="utf-8") as fh: fh.write(content) diff --git a/script/ci-custom.py b/script/ci-custom.py index 56ca0d0355..78ff6cf781 100755 --- a/script/ci-custom.py +++ b/script/ci-custom.py @@ -14,7 +14,7 @@ import time import colorama from helpers import filter_changed, git_ls_files, print_error_for_file, styled -sys.path.append(os.path.dirname(__file__)) +sys.path.append(str(Path(__file__).parent)) def find_all(a_str, sub): @@ -341,9 +341,9 @@ def lint_const_ordered(fname, content): matching = [ (i + 1, line) for i, line in enumerate(lines) if line.startswith(start) ] - ordered = list(sorted(matching, key=lambda x: x[1].replace("_", " "))) - ordered = [(mi, ol) for (mi, _), (_, ol) in zip(matching, ordered)] - for (mi, mline), (_, ol) in zip(matching, ordered): + ordered = sorted(matching, key=lambda x: x[1].replace("_", " ")) + ordered = [(mi, ol) for (mi, _), (_, ol) in zip(matching, ordered, strict=True)] + for (mi, mline), (_, ol) in zip(matching, ordered, strict=True): if mline == ol: continue target = next(i for i, line in ordered if line == mline) @@ -562,7 +562,7 @@ def lint_constants_usage(): # Maximum allowed CONF_ constants in esphome/const.py. # This file is frozen — new constants go in esphome/components/const/__init__.py. # Decrease this number when constants are moved out of const.py. -CONST_PY_MAX_CONF = 1012 +CONST_PY_MAX_CONF = 1013 @lint_content_check(include=["esphome/const.py"]) @@ -693,19 +693,6 @@ def lint_esphome_h(fname, line, col, content): ) -@lint_content_find_check( - "CORE.using_esp_idf", - include=py_include, - exclude=["esphome/core/__init__.py", "script/ci-custom.py"], -) -def lint_using_esp_idf_deprecated(fname, line, col, content): - return ( - f"{highlight('CORE.using_esp_idf')} is deprecated and will change behavior in 2026.6. " - "ESP32 Arduino builds on top of ESP-IDF, so ESP-IDF features are available in both frameworks. " - f"Please use {highlight('CORE.is_esp32')} and/or {highlight('CORE.using_arduino')} instead." - ) - - @lint_content_check(include=["*.h"], exclude=["esphome/core/entity_types.h"]) def lint_pragma_once(fname, content): if "#pragma once" not in content: diff --git a/script/ci_add_metadata_to_json.py b/script/ci_add_metadata_to_json.py index 687b5131c0..e884e9a64c 100755 --- a/script/ci_add_metadata_to_json.py +++ b/script/ci_add_metadata_to_json.py @@ -44,7 +44,7 @@ def main() -> int: return 1 try: - with open(json_path, encoding="utf-8") as f: + with Path(json_path).open(encoding="utf-8") as f: data = json.load(f) except (json.JSONDecodeError, OSError) as e: print(f"Error loading JSON: {e}", file=sys.stderr) @@ -74,7 +74,7 @@ def main() -> int: # Write back try: - with open(json_path, "w", encoding="utf-8") as f: + with Path(json_path).open("w", encoding="utf-8") as f: json.dump(data, f, indent=2) print(f"Added metadata to {args.json_file}", file=sys.stderr) except OSError as e: diff --git a/script/ci_helpers.py b/script/ci_helpers.py old mode 100755 new mode 100644 index 48b0e4bbfe..a51a857ada --- a/script/ci_helpers.py +++ b/script/ci_helpers.py @@ -3,6 +3,7 @@ from __future__ import annotations import os +from pathlib import Path def write_github_output(outputs: dict[str, str | int]) -> None: @@ -16,7 +17,7 @@ def write_github_output(outputs: dict[str, str | int]) -> None: """ github_output = os.environ.get("GITHUB_OUTPUT") if github_output: - with open(github_output, "a", encoding="utf-8") as f: + with Path(github_output).open("a", encoding="utf-8") as f: f.writelines(f"{key}={value}\n" for key, value in outputs.items()) else: for key, value in outputs.items(): diff --git a/script/ci_memory_impact_comment.py b/script/ci_memory_impact_comment.py index 01316da27f..0908b99595 100755 --- a/script/ci_memory_impact_comment.py +++ b/script/ci_memory_impact_comment.py @@ -91,7 +91,7 @@ def load_analysis_json(json_path: str) -> dict | None: return None try: - with open(json_file, encoding="utf-8") as f: + with Path(json_file).open(encoding="utf-8") as f: return json.load(f) except (json.JSONDecodeError, OSError) as e: print(f"Failed to load analysis JSON: {e}", file=sys.stderr) diff --git a/script/ci_memory_impact_extract.py b/script/ci_memory_impact_extract.py index 2aa7394b11..feacc2b1af 100755 --- a/script/ci_memory_impact_extract.py +++ b/script/ci_memory_impact_extract.py @@ -127,7 +127,7 @@ def run_detailed_analysis(build_dir: str) -> dict | None: if not idedata_path.exists(): continue try: - with open(idedata_path, encoding="utf-8") as f: + with idedata_path.open(encoding="utf-8") as f: raw_data = json.load(f) idedata = IDEData(raw_data) print(f"Loaded idedata from: {idedata_path}", file=sys.stderr) @@ -264,7 +264,7 @@ def main() -> int: output_path = Path(args.output_json) output_path.parent.mkdir(parents=True, exist_ok=True) - with open(output_path, "w", encoding="utf-8") as f: + with output_path.open("w", encoding="utf-8") as f: json.dump(output_data, f, indent=2) print(f"Saved analysis to {args.output_json}", file=sys.stderr) diff --git a/script/clang-format b/script/clang-format index 028d752c55..df45798a30 100755 --- a/script/clang-format +++ b/script/clang-format @@ -2,6 +2,7 @@ import argparse import os +from pathlib import Path import queue import re import subprocess @@ -70,7 +71,7 @@ def main(): ) args = parser.parse_args() - cwd = os.getcwd() + cwd = Path.cwd() files = [ os.path.relpath(path, cwd) for path in git_ls_files(["*.cpp", "*.h", "*.tcc"]) ] diff --git a/script/clang-tidy b/script/clang-tidy index 1c413ffa23..56c0a9db71 100755 --- a/script/clang-tidy +++ b/script/clang-tidy @@ -2,6 +2,7 @@ import argparse import os +from pathlib import Path import queue import re import shutil @@ -32,7 +33,7 @@ def clang_options(idedata): cmd = [] # extract target architecture from triplet in g++ filename - triplet = os.path.basename(idedata["cxx_path"])[:-4] + triplet = Path(idedata["cxx_path"]).name[:-4] if triplet.startswith("xtensa-"): # clang doesn't support Xtensa (yet?), so compile in 32-bit mode and pretend we're the Xtensa compiler cmd.append("-m32") @@ -153,8 +154,8 @@ def run_tidy(executable, args, options, tmpdir, path_queue, lock, failed_files): if sys.stdout.isatty(): invocation.append("--use-color") - invocation.append(f"--header-filter={os.path.abspath(basepath)}/.*") - invocation.append(os.path.abspath(path)) + invocation.append(f"--header-filter={Path(basepath).resolve()}/.*") + invocation.append(str(Path(path).resolve())) invocation.append("--") invocation.extend(options) @@ -229,7 +230,7 @@ def main(): ) args = parser.parse_args() - cwd = os.getcwd() + cwd = Path.cwd() files = [os.path.relpath(path, cwd) for path in git_ls_files(["*.cpp"])] # Exclude benchmark files — they require google benchmark headers not # available in the ESP32 toolchain and use different naming conventions. diff --git a/script/clang_tidy_hash.py b/script/clang_tidy_hash.py index d0d8438437..f478535567 100755 --- a/script/clang_tidy_hash.py +++ b/script/clang_tidy_hash.py @@ -16,7 +16,7 @@ sys.path.insert(0, str(script_dir)) def read_file_lines(path: Path) -> list[str]: """Read lines from a file.""" - with open(path) as f: + with path.open() as f: return f.readlines() @@ -65,7 +65,7 @@ def get_clang_tidy_version_from_requirements(repo_root: Path | None = None) -> s def read_file_bytes(path: Path) -> bytes: """Read bytes from a file.""" - with open(path, "rb") as f: + with path.open("rb") as f: return f.read() @@ -120,7 +120,7 @@ def read_stored_hash(repo_root: Path | None = None) -> str | None: def write_file_content(path: Path, content: str) -> None: """Write content to a file.""" - with open(path, "w") as f: + with path.open("w") as f: f.write(content) diff --git a/script/determine-jobs.py b/script/determine-jobs.py index ef2175eb79..d91936952e 100755 --- a/script/determine-jobs.py +++ b/script/determine-jobs.py @@ -306,7 +306,7 @@ def _is_clang_tidy_full_scan() -> bool: """ try: result = subprocess.run( - [os.path.join(root_path, "script", "clang_tidy_hash.py"), "--check"], + [str(Path(root_path) / "script" / "clang_tidy_hash.py"), "--check"], capture_output=True, check=False, ) @@ -483,9 +483,7 @@ def should_run_device_builder(branch: str | None = None) -> bool: True if the device-builder downstream tests should run, False otherwise. """ target_branch = get_target_branch() - if target_branch and ( - target_branch.startswith("release") or target_branch.startswith("beta") - ): + if target_branch and (target_branch.startswith(("release", "beta"))): return False for file in changed_files(branch): @@ -955,9 +953,7 @@ def detect_memory_impact_config( # all components at once would produce nonsensical memory impact results. # Memory impact analysis is most useful for focused PRs targeting dev. target_branch = get_target_branch() - if target_branch and ( - target_branch.startswith("release") or target_branch.startswith("beta") - ): + if target_branch and (target_branch.startswith(("release", "beta"))): print( f"Memory impact: Skipping analysis for target branch {target_branch} " f"(would try to build all components at once, giving nonsensical results)", @@ -1047,7 +1043,7 @@ def detect_memory_impact_config( # Find common platforms supported by ALL components # This ensures we can build all components together in a merged config common_platforms = set(MEMORY_IMPACT_PLATFORM_PREFERENCE) - for component, platforms in component_platforms_map.items(): + for platforms in component_platforms_map.values(): common_platforms &= platforms # Select the most preferred platform from the common set @@ -1311,7 +1307,7 @@ def main() -> None: # (no isolation, all components are groupable) target_branch = get_target_branch() is_release_branch = target_branch and ( - target_branch.startswith("release") or target_branch.startswith("beta") + target_branch.startswith(("release", "beta")) ) if is_release_branch: diff --git a/script/extract_automations.py b/script/extract_automations.py index 4e650ce25f..3cdfb5d32c 100755 --- a/script/extract_automations.py +++ b/script/extract_automations.py @@ -12,9 +12,9 @@ if __name__ == "__main__": components = get_components_with_dependencies(files, True) dump = { - "actions": sorted(list(ACTION_REGISTRY.keys())), - "conditions": sorted(list(CONDITION_REGISTRY.keys())), - "pin_providers": sorted(list(PIN_SCHEMA_REGISTRY.keys())), + "actions": sorted(ACTION_REGISTRY.keys()), + "conditions": sorted(CONDITION_REGISTRY.keys()), + "pin_providers": sorted(PIN_SCHEMA_REGISTRY.keys()), } print(json.dumps(dump, indent=2)) diff --git a/script/helpers.py b/script/helpers.py index 0716b12470..7b7001b518 100644 --- a/script/helpers.py +++ b/script/helpers.py @@ -17,10 +17,10 @@ from typing import Any import colorama -root_path = os.path.abspath(os.path.normpath(os.path.join(__file__, "..", ".."))) -basepath = os.path.join(root_path, "esphome") -temp_folder = os.path.join(root_path, ".temp") -temp_header_file = os.path.join(temp_folder, "all-include.cpp") +root_path = str(Path(__file__).resolve().parent.parent) +basepath = str(Path(root_path) / "esphome") +temp_folder = str(Path(root_path) / ".temp") +temp_header_file = str(Path(temp_folder) / "all-include.cpp") # C++ file extensions used for clang-tidy and clang-format checks CPP_FILE_EXTENSIONS = (".cpp", ".h", ".hpp", ".cc", ".cxx", ".c", ".tcc") @@ -103,9 +103,7 @@ def get_component_from_path(file_path: str) -> str | None: Returns: Component name if path is in components or tests directory, None otherwise """ - if file_path.startswith(ESPHOME_COMPONENTS_PATH) or file_path.startswith( - ESPHOME_TESTS_COMPONENTS_PATH - ): + if file_path.startswith((ESPHOME_COMPONENTS_PATH, ESPHOME_TESTS_COMPONENTS_PATH)): parts = file_path.split("/") if len(parts) >= 3 and parts[2]: # Verify that parts[2] is actually a component directory, not a file @@ -160,7 +158,7 @@ def is_validate_only_file(test_file: Path) -> bool: ``esphome config`` only and skipped during compile. """ name = test_file.name - return name.startswith("validate.") or name.startswith("validate-") + return name.startswith(("validate.", "validate-")) @dataclass(frozen=True) @@ -345,8 +343,8 @@ def _get_github_event_data() -> dict | None: Parsed event data dictionary, or None if not available """ github_event_path = os.environ.get("GITHUB_EVENT_PATH") - if github_event_path and os.path.exists(github_event_path): - with open(github_event_path) as f: + if github_event_path and Path(github_event_path).exists(): + with Path(github_event_path).open() as f: return json.load(f) return None @@ -470,7 +468,8 @@ def _get_changed_files_from_command(command: list[str]) -> list[str]: raise Exception(f"Command failed: {' '.join(command)}\nstderr: {proc.stderr}") changed_files = splitlines_no_ends(proc.stdout) - changed_files = [os.path.relpath(f, os.getcwd()) for f in changed_files if f] + cwd = Path.cwd() + changed_files = [os.path.relpath(f, cwd) for f in changed_files if f] # noqa: PTH109 changed_files.sort() return changed_files @@ -505,7 +504,7 @@ def get_changed_components() -> list[str] | None: return None # Use list-components.py to get changed components - script_path = os.path.join(root_path, "script", "list-components.py") + script_path = str(Path(root_path) / "script" / "list-components.py") cmd = [script_path, "--changed"] try: @@ -625,7 +624,7 @@ def filter_changed(files: list[str]) -> list[str]: def filter_grep(files: list[str], value: list[str]) -> list[str]: matched = [] for file in files: - with open(file, encoding="utf-8") as handle: + with Path(file).open(encoding="utf-8") as handle: contents = handle.read() if any(v in contents for v in value): matched.append(file) diff --git a/script/lint-python b/script/lint-python index 18281c711e..e4b3314d2a 100755 --- a/script/lint-python +++ b/script/lint-python @@ -2,6 +2,7 @@ import argparse import os +from pathlib import Path import re import sys @@ -66,11 +67,12 @@ def main(): args = parser.parse_args() files = [] + cwd = Path.cwd() for path in git_ls_files(): filetypes = (".py",) - ext = os.path.splitext(path)[1] + ext = Path(path).suffix if ext in filetypes and path.startswith("esphome"): - path = os.path.relpath(path, os.getcwd()) + path = os.path.relpath(path, cwd) files.append(path) # Match against re file_name_re = re.compile("|".join(args.files)) diff --git a/script/split_components_for_ci.py b/script/split_components_for_ci.py index 0d10246bb4..7f06f50f48 100755 --- a/script/split_components_for_ci.py +++ b/script/split_components_for_ci.py @@ -295,7 +295,7 @@ def main() -> int: # Sort groups by signature for readability groupable_groups = [] isolated_groups = [] - for (platform, signature), group_comps in sorted(signature_groups.items()): + for (_platform, signature), group_comps in sorted(signature_groups.items()): if signature.startswith(ISOLATED_SIGNATURE_PREFIX): isolated_groups.append((signature, group_comps)) else: diff --git a/script/sync-device_class.py b/script/sync-device_class.py index 121c89b8f9..660142195a 100755 --- a/script/sync-device_class.py +++ b/script/sync-device_class.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 +from pathlib import Path import re # pylint: disable=import-error @@ -34,10 +35,10 @@ DOMAINS = { def sub(path, pattern, repl): - with open(path, encoding="utf-8") as handle: + with Path(path).open(encoding="utf-8") as handle: content = handle.read() content = re.sub(pattern, repl, content, flags=re.MULTILINE) - with open(path, "w", encoding="utf-8") as handle: + with Path(path).open("w", encoding="utf-8") as handle: handle.write(content) diff --git a/script/test_build_components.py b/script/test_build_components.py index 43b71004eb..767b55c94b 100755 --- a/script/test_build_components.py +++ b/script/test_build_components.py @@ -297,7 +297,7 @@ def write_github_summary( test_results: List of all test results """ summary_content = format_github_summary(test_results, toolchain) - with open(os.environ["GITHUB_STEP_SUMMARY"], "a", encoding="utf-8") as f: + with Path(os.environ["GITHUB_STEP_SUMMARY"]).open("a", encoding="utf-8") as f: f.write(summary_content) @@ -890,7 +890,7 @@ def run_grouped_component_tests( print("=" * 80 + "\n") # Execute grouped tests - for (platform, signature), components in grouped_components.items(): + for (platform, _signature), components in grouped_components.items(): # Only group if we have multiple components with same signature if len(components) <= 1: continue @@ -1055,7 +1055,7 @@ def test_components( # Create empty test files for each platform (or filtered platform) reference_tests: list[Path] = [] - for platform_name, base_file in platform_bases.items(): + for platform_name in platform_bases: if platform_filter and not platform_name.startswith(platform_filter): continue # Create an empty test file named to match the platform diff --git a/tests/component_tests/display/test_display_metadata.py b/tests/component_tests/display/test_display_metadata.py index e569754494..ef3f12cb73 100644 --- a/tests/component_tests/display/test_display_metadata.py +++ b/tests/component_tests/display/test_display_metadata.py @@ -2,6 +2,8 @@ from unittest.mock import patch +import pytest + from esphome.components.display import ( DisplayMetaData, add_metadata, @@ -74,8 +76,5 @@ def test_add_metadata_overwrites_existing(): def test_metadata_is_frozen(): """Test that DisplayMetaData instances are immutable (frozen dataclass).""" meta = DisplayMetaData(320, 240, True, False) - try: + with pytest.raises(AttributeError): meta.width = 640 - assert False, "Expected FrozenInstanceError" - except AttributeError: - pass diff --git a/tests/component_tests/lvgl/test_automation_schema_lazy.py b/tests/component_tests/lvgl/test_automation_schema_lazy.py new file mode 100644 index 0000000000..46430824f6 --- /dev/null +++ b/tests/component_tests/lvgl/test_automation_schema_lazy.py @@ -0,0 +1,71 @@ +"""Tests for lvgl automation_schema lazy validate_automation build.""" + +from __future__ import annotations + +from unittest.mock import patch + +import esphome.components.lvgl # noqa: F401 +from esphome.components.lvgl import schemas as lvgl_schemas +from esphome.components.lvgl.schemas import ( + WIDGET_TYPES, + _lazy_validate_automation, + automation_schema, +) +from esphome.components.lvgl.widgets import WidgetType +from esphome.config_validation import GenerateID, declare_id +from esphome.const import CONF_TRIGGER_ID +from esphome.core.config import StartupTrigger + + +def _widget_type(name: str = "obj") -> WidgetType: + wt = WIDGET_TYPES.get(name) + assert wt is not None, f"widget type {name!r} not registered" + return wt + + +def _trigger_extra_schema() -> dict: + return {GenerateID(CONF_TRIGGER_ID): declare_id(StartupTrigger)} + + +def test_lazy_validator_defers_build_until_first_call() -> None: + with patch( + "esphome.components.lvgl.schemas.validate_automation", + wraps=lvgl_schemas.validate_automation, + ) as va_mock: + validator = _lazy_validate_automation(_trigger_extra_schema()) + assert va_mock.call_count == 0 + validator({"then": []}) + assert va_mock.call_count == 1 + validator({"then": []}) + assert va_mock.call_count == 1 + + +def test_eager_build_when_schema_extraction_enabled() -> None: + with ( + patch("esphome.components.lvgl.schemas.EnableSchemaExtraction", True), + patch( + "esphome.components.lvgl.schemas.validate_automation", + wraps=lvgl_schemas.validate_automation, + ) as va_mock, + ): + _lazy_validate_automation(_trigger_extra_schema()) + assert va_mock.call_count == 1 + + +def test_lazy_and_eager_produce_equivalent_validation() -> None: + extra = _trigger_extra_schema() + with patch("esphome.components.lvgl.schemas.EnableSchemaExtraction", True): + eager = _lazy_validate_automation(extra) + lazy = _lazy_validate_automation(_trigger_extra_schema()) + sample = {"then": []} + assert lazy(sample) == eager(sample) + + +def test_automation_schema_uses_lazy_validators() -> None: + wt = _widget_type("obj") + with patch( + "esphome.components.lvgl.schemas.validate_automation", + wraps=lvgl_schemas.validate_automation, + ) as va_mock: + automation_schema(wt.w_type) + assert va_mock.call_count == 0 diff --git a/tests/component_tests/lvgl/test_obj_schema_cache.py b/tests/component_tests/lvgl/test_obj_schema_cache.py new file mode 100644 index 0000000000..860ee211dd --- /dev/null +++ b/tests/component_tests/lvgl/test_obj_schema_cache.py @@ -0,0 +1,67 @@ +"""Tests for obj_schema() memoization.""" + +from __future__ import annotations + +from collections.abc import Generator + +import pytest + +import esphome.components.lvgl # noqa: F401 +from esphome.components.lvgl import schemas as lvgl_schemas +from esphome.components.lvgl.schemas import WIDGET_TYPES, obj_schema + + +@pytest.fixture(autouse=True) +def _clear_obj_schema_cache() -> Generator[None]: + cache = getattr(lvgl_schemas, "_OBJ_SCHEMA_CACHE", None) + if cache is not None: + cache.clear() + yield + if cache is not None: + cache.clear() + + +def _widget_type(name: str = "obj"): + wt = WIDGET_TYPES.get(name) + assert wt is not None, f"widget type {name!r} not registered" + return wt + + +def test_same_widget_type_returns_same_schema() -> None: + wt = _widget_type("obj") + assert obj_schema(wt) is obj_schema(wt) + + +def test_different_widget_types_return_different_schemas() -> None: + assert obj_schema(_widget_type("obj")) is not obj_schema(_widget_type("label")) + + +def test_cache_is_populated_after_first_call() -> None: + wt = _widget_type("obj") + assert id(wt) not in lvgl_schemas._OBJ_SCHEMA_CACHE + obj_schema(wt) + assert id(wt) in lvgl_schemas._OBJ_SCHEMA_CACHE + + +def test_cached_schema_produces_equivalent_output() -> None: + wt = _widget_type("obj") + cached_result = obj_schema(wt)({}) + lvgl_schemas._OBJ_SCHEMA_CACHE.clear() + fresh_result = obj_schema(wt)({}) + assert cached_result == fresh_result + + +def test_id_recycling_is_caught_by_identity_guard() -> None: + wt = _widget_type("obj") + real_schema = obj_schema(wt) + + cached_widget_type, _ = lvgl_schemas._OBJ_SCHEMA_CACHE[id(wt)] + sentinel_schema = object() + lvgl_schemas._OBJ_SCHEMA_CACHE[id(wt)] = (cached_widget_type, sentinel_schema) + assert obj_schema(wt) is sentinel_schema + + other = _widget_type("label") + lvgl_schemas._OBJ_SCHEMA_CACHE[id(wt)] = (other, sentinel_schema) + rebuilt = obj_schema(wt) + assert rebuilt is not sentinel_schema + assert rebuilt is not real_schema diff --git a/tests/component_tests/lvgl/test_schema_dict_helpers.py b/tests/component_tests/lvgl/test_schema_dict_helpers.py new file mode 100644 index 0000000000..16714f54d7 --- /dev/null +++ b/tests/component_tests/lvgl/test_schema_dict_helpers.py @@ -0,0 +1,236 @@ +"""Tests for part_dict / obj_dict / part_schema / obj_schema mapping contracts. + +These guard the dict-merge refactor: the dict helpers must keep returning the +same logical mapping as the chained-extend version produced, and the +corresponding Schema(...) wrappers must accept and reject the same configs. +""" + +from __future__ import annotations + +from collections.abc import Generator + +import pytest +import voluptuous as vol + +from esphome import config_validation as cv +import esphome.components.lvgl +from esphome.components.lvgl import ( + _theme_schema, + defines as df, + schemas as lvgl_schemas, +) +from esphome.components.lvgl.schemas import ( + ALIGN_TO_SCHEMA, + FLAG_SCHEMA, + FULL_STYLE_SCHEMA, + STATE_SCHEMA, + STYLE_SCHEMA, + WIDGET_TYPES, + automation_schema, + obj_dict, + obj_schema, + part_dict, + part_schema, +) +from esphome.components.lvgl.types import LvType +from esphome.components.lvgl.widgets import WidgetType + + +@pytest.fixture(autouse=True) +def _clear_obj_dict_cache() -> Generator[None]: + cache = getattr(lvgl_schemas, "_OBJ_DICT_CACHE", None) + if cache is not None: + cache.clear() + # The lazily-built theme schema is cached on _build_theme_schema; clear it + # too so each test starts from a clean slate. + build_theme = getattr(esphome.components.lvgl, "_build_theme_schema", None) + if build_theme is not None and hasattr(build_theme, "cache_clear"): + build_theme.cache_clear() + yield + if cache is not None: + cache.clear() + if build_theme is not None and hasattr(build_theme, "cache_clear"): + build_theme.cache_clear() + + +def _marker_names(mapping) -> set[str]: + """Return the underlying string names of every voluptuous Marker key.""" + names: set[str] = set() + for key in mapping: + if isinstance(key, vol.Marker): + schema = key.schema + if isinstance(schema, str): + names.add(schema) + return names + + +def _widget_type(name: str = "obj"): + wt = WIDGET_TYPES.get(name) + assert wt is not None, f"widget type {name!r} not registered" + return wt + + +def test_part_dict_includes_state_flag_and_part_keys() -> None: + parts = ("indicator", "knob") + keys = _marker_names(part_dict(parts)) + + assert {"indicator", "knob"} <= keys + assert _marker_names(STATE_SCHEMA.schema) <= keys + assert _marker_names(FLAG_SCHEMA.schema) <= keys + + +def test_obj_dict_extends_part_dict_with_align_automation_state_group() -> None: + wt = _widget_type("obj") + part_keys = _marker_names(part_dict(wt.parts)) + obj_keys = _marker_names(obj_dict(wt)) + + assert part_keys <= obj_keys + assert _marker_names(ALIGN_TO_SCHEMA) <= obj_keys + assert _marker_names(automation_schema(wt.w_type)) <= obj_keys + assert {"state", "group"} <= obj_keys + + +def test_obj_dict_is_memoized_by_widget_type() -> None: + wt = _widget_type("obj") + first = obj_dict(wt) + second = obj_dict(wt) + assert first is second + # Different widget type, different dict. + assert obj_dict(_widget_type("label")) is not first + + +def test_part_schema_round_trips_known_state_and_part_settings() -> None: + schema = part_schema(("indicator",)) + out = schema( + { + "bg_color": 0x112233, + "checked": {"bg_color": 0x445566}, + "indicator": {"bg_color": 0x778899}, + } + ) + assert out["bg_color"] == 0x112233 + assert out["checked"]["bg_color"] == 0x445566 + assert out["indicator"]["bg_color"] == 0x778899 + + +def test_part_schema_rejects_unknown_part() -> None: + schema = part_schema(("indicator",)) + with pytest.raises(vol.Invalid): + schema({"definitely_not_a_part": {}}) + + +@pytest.mark.parametrize("name", sorted(WIDGET_TYPES)) +def test_obj_schema_accepts_empty_config_for_every_widget_type(name: str) -> None: + obj_schema(_widget_type(name))({}) + + +def test_obj_schema_accepts_align_to_and_state_group() -> None: + schema = obj_schema(_widget_type("obj")) + out = schema( + { + df.CONF_ALIGN_TO: { + "id": "some_other_widget", + df.CONF_ALIGN: "TOP_LEFT", + }, + "state": {"checked": True}, + } + ) + assert out[df.CONF_ALIGN_TO][df.CONF_ALIGN] == "LV_ALIGN_TOP_LEFT" + assert out["state"]["checked"] is True + + +def test_obj_schema_rejects_unknown_top_level_key() -> None: + with pytest.raises(vol.Invalid): + obj_schema(_widget_type("obj"))({"definitely_not_a_real_key": 1}) + + +def test_part_schema_returns_cv_schema_for_extend_callers() -> None: + schema = part_schema(("indicator",)) + extended = schema.extend({cv.Optional("extra_key"): cv.string}) + out = extended({"extra_key": "value", "bg_color": 0xAABBCC}) + assert out["extra_key"] == "value" + assert out["bg_color"] == 0xAABBCC + + +def test_obj_schema_returns_cv_schema_for_extend_callers() -> None: + schema = obj_schema(_widget_type("obj")) + extended = schema.extend({cv.Optional("extra_key"): cv.string}) + extended({"extra_key": "value"}) + + +@pytest.mark.parametrize( + "schema", + [STATE_SCHEMA, FLAG_SCHEMA, STYLE_SCHEMA, FULL_STYLE_SCHEMA], +) +def test_spread_sources_carry_no_extra_schemas(schema: cv.Schema) -> None: + # part_dict / obj_dict reach into .schema and rebuild via cv.Schema(...), + # which silently drops _extra_schemas and any non-default extra/required. + # Lock the invariant so a future add_extra() on these sources fails CI + # instead of quietly removing validation from part/obj/theme schemas. + assert not schema._extra_schemas + assert schema.extra is vol.PREVENT_EXTRA + assert schema.required is False + + +def test_theme_schema_merges_obj_dict_and_full_style_props() -> None: + # _theme_schema is the riskiest merge: obj_dict(w) and FULL_STYLE_SCHEMA.schema + # share many STYLE_SCHEMA marker instances. Exercise the merged schema + # end-to-end with one key from each side (a STATE_SCHEMA part from obj_dict + # and a FULL_STYLE-only property) to lock the behaviour against future + # regressions in either source. + out = _theme_schema( + { + df.CONF_DARK_MODE: True, + "obj": { + "bg_color": 0x112233, + "checked": {"bg_color": 0x445566}, + df.CONF_PAD_ROW: 4, + df.CONF_GRID_CELL_X_ALIGN: "CENTER", + }, + } + ) + assert out[df.CONF_DARK_MODE] is True + obj_out = out["obj"] + assert obj_out["bg_color"] == 0x112233 + assert obj_out["checked"]["bg_color"] == 0x445566 + assert obj_out[df.CONF_PAD_ROW] == 4 + assert obj_out[df.CONF_GRID_CELL_X_ALIGN] == "LV_GRID_ALIGN_CENTER" + + +def test_theme_schema_self_heals_when_a_widget_type_is_registered_later() -> None: + # _build_theme_schema is functools.cached on a snapshot of WIDGET_TYPES. + # any_widget_schema explicitly supports external components registering + # widgets lazily, and the device builder revalidates in-process, so a + # widget registered after first use must invalidate the cached snapshot. + _theme_schema({df.CONF_DARK_MODE: True}) # populate the cache + + name = "test_self_heal_widget" + assert name not in WIDGET_TYPES + # is_mock=True skips registration side-effects; insert into WIDGET_TYPES + # manually so the next theme call sees the new entry. + WIDGET_TYPES[name] = WidgetType(name, LvType("test_fake_t"), (), is_mock=True) + try: + out = _theme_schema({df.CONF_DARK_MODE: False, name: {"bg_color": 0x010203}}) + assert out[name]["bg_color"] == 0x010203 + finally: + WIDGET_TYPES.pop(name, None) + + +@pytest.mark.parametrize( + "schema", + [STATE_SCHEMA, FLAG_SCHEMA, STYLE_SCHEMA, FULL_STYLE_SCHEMA], +) +def test_spread_sources_have_no_top_level_marker_defaults(schema: cv.Schema) -> None: + # _theme_schema merges obj_dict(w) with FULL_STYLE_SCHEMA.schema; on a key + # collision, dict-spread keeps the first source's marker (and its default) + # but the last source's value, whereas .extend() would take both from the + # later source. The two are equivalent today because the overlapping + # markers are the same instances (both derive from STYLE_SCHEMA) and none + # carry a top-level default. Lock that so a future divergent default would + # fail CI rather than silently drift the merged validation. + offenders = [ + marker.schema + for marker in schema.schema + if isinstance(marker, vol.Optional) and marker.default is not vol.UNDEFINED + ] + assert not offenders, f"top-level Optional with default: {offenders}" diff --git a/tests/component_tests/packages/test_packages.py b/tests/component_tests/packages/test_packages.py index 8c809c5e91..66f946a5bd 100644 --- a/tests/component_tests/packages/test_packages.py +++ b/tests/component_tests/packages/test_packages.py @@ -510,15 +510,9 @@ def test_package_merge_by_missing_id() -> None: ], } - error_raised = False - try: + with pytest.raises(cv.Invalid) as exc_info: packages_pass(config) - assert False, "Expected validation error for missing ID" - except cv.Invalid as err: - error_raised = True - assert err.path == [CONF_SENSOR, 2] - - assert error_raised + assert exc_info.value.path == [CONF_SENSOR, 2] def test_package_list_remove_by_id() -> None: diff --git a/tests/component_tests/time/__init__.py b/tests/component_tests/time/__init__.py new file mode 100644 index 0000000000..dc24f4e532 --- /dev/null +++ b/tests/component_tests/time/__init__.py @@ -0,0 +1 @@ +"""Tests for the time component.""" diff --git a/tests/component_tests/time/test_init.py b/tests/component_tests/time/test_init.py new file mode 100644 index 0000000000..44469cfe28 --- /dev/null +++ b/tests/component_tests/time/test_init.py @@ -0,0 +1,369 @@ +"""Tests for time component – ha-timezone branch changes. + +Covers: +- detect_tz() platform guard (returns None for unsupported platforms) +- detect_tz() result caching (avoids duplicate log messages) +- detect_tz() error paths (tzlocal None, tzdata missing) +- validate_tz() accepts/rejects POSIX timezone strings and IANA keys +- TIME_SCHEMA: timezone is now truly optional (was SplitDefault) +- homeassistant/time: USE_HOMEASSISTANT_TIMEZONE define emitted iff + CONF_TIMEZONE is absent from the config +""" + +from __future__ import annotations + +from unittest import mock + +import pytest + +from esphome.components.time import DOMAIN, TIME_SCHEMA, detect_tz, validate_tz +import esphome.config_validation as cv +from esphome.const import ( + CONF_ID, + CONF_TIMEZONE, + KEY_CORE, + KEY_TARGET_FRAMEWORK, + KEY_TARGET_PLATFORM, + Platform, + PlatformFramework, +) +from esphome.core import CORE, EsphomeError +from tests.component_tests.types import SetCoreConfigCallable + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +# A minimal TZif v2/v3 file that encodes "EST5EDT" as the footer line. +# The binary content is not validated at this level – what matters is that +# _extract_tz_string() picks up the last-but-one newline-terminated line. +_FAKE_TZFILE = b"\x00" * 44 + b"TZif2\x00" * 1 + b"\n" + b"EST5EDT,M3.2.0,M11.1.0\n" + + +def _set_platform(platform: Platform) -> None: + """Set CORE.data so that CORE.target_platform returns *platform*.""" + CORE.data[KEY_CORE] = { + KEY_TARGET_PLATFORM: platform, + KEY_TARGET_FRAMEWORK: "arduino", + } + + +# --------------------------------------------------------------------------- +# detect_tz – platform guard +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "platform_framework", + [ + PlatformFramework.NRF52_ZEPHYR, + ], +) +def test_detect_tz_returns_none_for_unsupported_platform( + platform_framework: PlatformFramework, + set_core_config: SetCoreConfigCallable, +) -> None: + """detect_tz() must return None for platforms that do not support TZ auto-detection.""" + set_core_config(platform_framework) + result = detect_tz() + assert result is None + + +@pytest.mark.parametrize( + "platform_framework", + [ + PlatformFramework.ESP32_IDF, + PlatformFramework.ESP32_ARDUINO, + PlatformFramework.ESP8266_ARDUINO, + PlatformFramework.RP2040_ARDUINO, + PlatformFramework.BK72XX_ARDUINO, + PlatformFramework.RTL87XX_ARDUINO, + PlatformFramework.LN882X_ARDUINO, + PlatformFramework.HOST_NATIVE, + ], +) +def test_detect_tz_calls_tzlocal_for_supported_platform( + platform_framework: PlatformFramework, + set_core_config: SetCoreConfigCallable, +) -> None: + """detect_tz() must call tzlocal for every supported platform.""" + set_core_config(platform_framework) + with ( + mock.patch( + "esphome.components.time.tzlocal.get_localzone_name", + return_value="America/New_York", + ), + mock.patch( + "esphome.components.time._load_tzdata", + return_value=_FAKE_TZFILE, + ), + ): + result = detect_tz() + assert result is not None + assert isinstance(result, str) + assert len(result) > 0 + + +# --------------------------------------------------------------------------- +# detect_tz – caching +# --------------------------------------------------------------------------- + + +def test_detect_tz_caches_result( + set_core_config: SetCoreConfigCallable, + caplog: pytest.LogCaptureFixture, +) -> None: + """detect_tz() must cache the TZ string after the first call so that + subsequent invocations (e.g. when multiple time platforms are configured) + skip tzlocal and avoid duplicate INFO messages.""" + set_core_config(PlatformFramework.ESP32_IDF) + + with ( + mock.patch( + "esphome.components.time.tzlocal.get_localzone_name", + return_value="America/New_York", + ) as mock_tz, + mock.patch( + "esphome.components.time._load_tzdata", + return_value=_FAKE_TZFILE, + ) as mock_load, + ): + first = detect_tz() + second = detect_tz() + + assert first == second + # tzlocal and _load_tzdata must be called exactly once despite two detect_tz() calls + mock_tz.assert_called_once() + mock_load.assert_called_once() + + +def test_detect_tz_cache_stored_in_core_data( + set_core_config: SetCoreConfigCallable, +) -> None: + """The cached TZ string should be stored under CORE.data[DOMAIN][CONF_TIMEZONE].""" + set_core_config(PlatformFramework.ESP32_IDF) + + with ( + mock.patch( + "esphome.components.time.tzlocal.get_localzone_name", + return_value="Europe/London", + ), + mock.patch( + "esphome.components.time._load_tzdata", + return_value=_FAKE_TZFILE, + ), + ): + result = detect_tz() + + assert CORE.data.get(DOMAIN, {}).get(CONF_TIMEZONE) == result + + +def test_detect_tz_returns_pre_seeded_cache( + set_core_config: SetCoreConfigCallable, +) -> None: + """If CORE.data already has a cached TZ string, detect_tz() must return it + without calling tzlocal at all.""" + set_core_config(PlatformFramework.ESP32_IDF) + CORE.data[DOMAIN] = {CONF_TIMEZONE: "CET-1CEST,M3.5.0,M10.5.0/3"} + + with mock.patch("esphome.components.time.tzlocal.get_localzone_name") as mock_tz: + result = detect_tz() + + assert result == "CET-1CEST,M3.5.0,M10.5.0/3" + mock_tz.assert_not_called() + + +# --------------------------------------------------------------------------- +# detect_tz – error paths +# --------------------------------------------------------------------------- + + +def test_detect_tz_raises_when_tzlocal_returns_none( + set_core_config: SetCoreConfigCallable, +) -> None: + """detect_tz() must raise EsphomeError when the local timezone cannot be determined.""" + set_core_config(PlatformFramework.ESP32_IDF) + + with ( + mock.patch( + "esphome.components.time.tzlocal.get_localzone_name", + return_value=None, + ), + pytest.raises(EsphomeError, match="Could not automatically determine timezone"), + ): + detect_tz() + + +def test_detect_tz_raises_when_tzdata_not_found( + set_core_config: SetCoreConfigCallable, +) -> None: + """detect_tz() must raise EsphomeError when tzdata has no entry for the IANA key.""" + set_core_config(PlatformFramework.ESP32_IDF) + + with ( + mock.patch( + "esphome.components.time.tzlocal.get_localzone_name", + return_value="Antarctica/Troll", + ), + mock.patch( + "esphome.components.time._load_tzdata", + return_value=None, + ), + pytest.raises(EsphomeError, match="Could not automatically determine timezone"), + ): + detect_tz() + + +# --------------------------------------------------------------------------- +# validate_tz +# --------------------------------------------------------------------------- + + +def test_validate_tz_accepts_valid_posix_string() -> None: + """validate_tz() must accept a syntactically valid POSIX TZ string.""" + result = validate_tz("UTC0") + assert result == "UTC0" + + +def test_validate_tz_accepts_posix_string_with_dst() -> None: + """validate_tz() must accept a full POSIX TZ string with DST rules.""" + tz = "EST5EDT,M3.2.0,M11.1.0" + result = validate_tz(tz) + assert result == tz + + +def test_validate_tz_accepts_iana_key_and_converts() -> None: + """validate_tz() must accept an IANA timezone key and return the POSIX string.""" + with mock.patch( + "esphome.components.time._load_tzdata", + return_value=_FAKE_TZFILE, + ): + result = validate_tz("America/New_York") + + # Should have been converted from IANA to POSIX via _extract_tz_string + assert result == "EST5EDT,M3.2.0,M11.1.0" + + +def test_validate_tz_rejects_invalid_posix_string() -> None: + """validate_tz() must raise cv.Invalid for a malformed POSIX TZ string.""" + with pytest.raises(cv.Invalid, match="Invalid POSIX timezone string"): + validate_tz("NOTAVALIDTZ!!!") + + +def test_validate_tz_accepts_empty_string() -> None: + """An empty string is accepted by validate_tz() and signals 'disable timezone'.""" + result = validate_tz("") + assert result == "" + + +# --------------------------------------------------------------------------- +# TIME_SCHEMA – timezone is now cv.Optional (no SplitDefault) +# --------------------------------------------------------------------------- + + +def test_time_schema_timezone_is_optional( + set_core_config: SetCoreConfigCallable, +) -> None: + """TIME_SCHEMA must accept a config with no timezone key on a supported platform.""" + set_core_config(PlatformFramework.ESP32_IDF) + # Should not raise + config = TIME_SCHEMA({}) + assert CONF_TIMEZONE not in config + + +def test_time_schema_explicit_timezone_accepted( + set_core_config: SetCoreConfigCallable, +) -> None: + """TIME_SCHEMA must accept an explicit valid POSIX timezone on Arduino/IDF.""" + set_core_config(PlatformFramework.ESP32_IDF) + config = TIME_SCHEMA({CONF_TIMEZONE: "UTC0"}) + assert config[CONF_TIMEZONE] == "UTC0" + + +def test_time_schema_explicit_empty_timezone_accepted( + set_core_config: SetCoreConfigCallable, +) -> None: + """An empty timezone string (timezone-disable sentinel) must pass TIME_SCHEMA.""" + set_core_config(PlatformFramework.ESP32_IDF) + config = TIME_SCHEMA({CONF_TIMEZONE: ""}) + assert config[CONF_TIMEZONE] == "" + + +def test_time_schema_timezone_rejected_on_zephyr( + set_core_config: SetCoreConfigCallable, +) -> None: + """TIME_SCHEMA must reject a timezone value on Zephyr with the framework error. + + The platform check (cv.only_with_framework) must run BEFORE validate_tz so + that users receive an actionable "unsupported framework" message rather than a + confusing TZ-parsing error. + """ + set_core_config(PlatformFramework.NRF52_ZEPHYR) + with pytest.raises(cv.Invalid, match="only available with framework"): + TIME_SCHEMA({CONF_TIMEZONE: "UTC0"}) + + +def test_time_schema_invalid_tz_on_zephyr_gives_framework_error( + set_core_config: SetCoreConfigCallable, +) -> None: + """Even a syntactically invalid TZ string must produce the framework error on Zephyr. + + This specifically tests that cv.only_with_framework is evaluated before + validate_tz: if the order were reversed, an invalid POSIX string would + generate a misleading TZ-parsing error instead. + """ + set_core_config(PlatformFramework.NRF52_ZEPHYR) + with pytest.raises(cv.Invalid, match="only available with framework"): + TIME_SCHEMA({CONF_TIMEZONE: "NOTAVALIDTZ!!!"}) + + +# --------------------------------------------------------------------------- +# homeassistant/time: USE_HOMEASSISTANT_TIMEZONE define +# --------------------------------------------------------------------------- + + +@pytest.fixture +def mock_ha_cg(): + """Mock codegen functions used by homeassistant/time to_code.""" + with ( + mock.patch( + "esphome.components.homeassistant.time.cg.new_Pvariable", + return_value=mock.MagicMock(), + ), + mock.patch( + "esphome.components.homeassistant.time.cg.add_define", + ) as mock_add_define, + mock.patch( + "esphome.components.homeassistant.time.cg.register_component", + new_callable=mock.AsyncMock, + ), + mock.patch( + "esphome.components.homeassistant.time.time_.register_time", + new_callable=mock.AsyncMock, + ), + ): + yield mock_add_define + + +@pytest.mark.asyncio +async def test_ha_time_defines_ha_timezone_when_no_explicit_tz(mock_ha_cg) -> None: + """When CONF_TIMEZONE is absent from the config, to_code() must call + cg.add_define('USE_HOMEASSISTANT_TIMEZONE').""" + from esphome.components.homeassistant.time import to_code + + await to_code({CONF_ID: mock.MagicMock()}) + + mock_ha_cg.assert_any_call("USE_HOMEASSISTANT_TIMEZONE") + + +@pytest.mark.asyncio +async def test_ha_time_no_ha_timezone_define_when_explicit_tz(mock_ha_cg) -> None: + """When CONF_TIMEZONE is present in the config, to_code() must NOT call + cg.add_define('USE_HOMEASSISTANT_TIMEZONE').""" + from esphome.components.homeassistant.time import to_code + + await to_code({CONF_ID: mock.MagicMock(), CONF_TIMEZONE: "UTC0"}) + + define_calls = [call.args[0] for call in mock_ha_cg.call_args_list] + assert "USE_HOMEASSISTANT_TIME" in define_calls + assert "USE_HOMEASSISTANT_TIMEZONE" not in define_calls diff --git a/tests/components/api/common-base.yaml b/tests/components/api/common-base.yaml index 504c52a57b..ca86445777 100644 --- a/tests/components/api/common-base.yaml +++ b/tests/components/api/common-base.yaml @@ -120,12 +120,12 @@ api: lambda: 'return condition;' then: - logger.log: - format: "Condition true, value: %d" - args: ['value'] + format: "Condition true, value: %ld" + args: ['(long) value'] else: - logger.log: - format: "Condition false, value: %d" - args: ['value'] + format: "Condition false, value: %ld" + args: ['(long) value'] - logger.log: "After if/else" # Test nested IfAction (multiple ContinuationAction instances) - action: test_nested_if @@ -171,8 +171,8 @@ api: count: !lambda 'return count;' then: - logger.log: - format: "Repeat iteration: %d" - args: ['iteration'] + format: "Repeat iteration: %lu" + args: ['(unsigned long) iteration'] - logger.log: "After repeat" # Test combined continuations (if + while + repeat) - action: test_combined_continuations @@ -193,8 +193,8 @@ api: lambda: 'return id(api_continuation_test_counter) > 0;' then: - logger.log: - format: "Combined: repeat=%d, while=%d" - args: ['iteration', 'id(api_continuation_test_counter)'] + format: "Combined: repeat=%lu, while=%d" + args: ['(unsigned long) iteration', 'id(api_continuation_test_counter)'] - lambda: 'id(api_continuation_test_counter)--;' else: - logger.log: "Skipped loops" @@ -208,8 +208,8 @@ api: - api.respond: success: true - logger.log: - format: "Status response sent (call_id=%d)" - args: [call_id] + format: "Status response sent (call_id=%lu)" + args: ['(unsigned long) call_id'] - action: test_respond_status_error variables: @@ -229,8 +229,8 @@ api: value: float then: - logger.log: - format: "Optional response (call_id=%d, return_response=%d)" - args: [call_id, return_response] + format: "Optional response (call_id=%lu, return_response=%lu)" + args: ['(unsigned long) call_id', '(unsigned long) return_response'] - api.respond: data: !lambda |- root["sensor"] = sensor_name; @@ -264,8 +264,8 @@ api: input: string then: - logger.log: - format: "Only response (call_id=%d)" - args: [call_id] + format: "Only response (call_id=%lu)" + args: ['(unsigned long) call_id'] - api.respond: data: !lambda |- root["input"] = input; diff --git a/tests/components/audio_file/validate.esp32-idf.yaml b/tests/components/audio_file/validate.esp32-idf.yaml new file mode 100644 index 0000000000..085f853c8e --- /dev/null +++ b/tests/components/audio_file/validate.esp32-idf.yaml @@ -0,0 +1,11 @@ +audio_file: + - id: test_audio + file: + type: local + path: $component_dir/test.wav + +media_source: + - platform: audio_file + id: audio_file_source + # task_stack_in_psram: false must validate without a psram: component + task_stack_in_psram: false diff --git a/tests/components/bluetooth_proxy/test.esp32-c6-idf.yaml b/tests/components/bluetooth_proxy/test.esp32-c6-idf.yaml index 6c27bd35d0..df5b0123b5 100644 --- a/tests/components/bluetooth_proxy/test.esp32-c6-idf.yaml +++ b/tests/components/bluetooth_proxy/test.esp32-c6-idf.yaml @@ -1,6 +1,8 @@ <<: !include common.yaml esp32_ble_tracker: + +esp32_ble: max_connections: 9 bluetooth_proxy: diff --git a/tests/components/esp32_hosted/common.yaml b/tests/components/esp32_hosted/common.yaml index ab029e5064..332fe5b070 100644 --- a/tests/components/esp32_hosted/common.yaml +++ b/tests/components/esp32_hosted/common.yaml @@ -3,6 +3,7 @@ esp32_hosted: slot: 1 active_high: true reset_pin: GPIO15 + use_psram: true cmd_pin: GPIO13 clk_pin: GPIO12 d0_pin: GPIO11 diff --git a/tests/components/esphome/common.yaml b/tests/components/esphome/common.yaml index db75b08b38..93f82824e6 100644 --- a/tests/components/esphome/common.yaml +++ b/tests/components/esphome/common.yaml @@ -2,6 +2,8 @@ esphome: debug_scheduler: true platformio_options: board_build.flash_mode: dio + build_flags: + - "-DESPHOME_TEST_BUILD_FLAG" environment_variables: TEST_ENV_VAR: "test_value" BUILD_NUMBER: "12345" diff --git a/tests/components/lvgl/lvgl-package.yaml b/tests/components/lvgl/lvgl-package.yaml index 0f4b961297..7af058e6b8 100644 --- a/tests/components/lvgl/lvgl-package.yaml +++ b/tests/components/lvgl/lvgl-package.yaml @@ -1313,6 +1313,7 @@ lvgl: width: 6 start_value: 0 end_value: 360 + rounded: true - id: page3 layout: Horizontal pad_all: 6px diff --git a/tests/components/micro_wake_word/common.yaml b/tests/components/micro_wake_word/common.yaml index c051c8dd57..cd060c176e 100644 --- a/tests/components/micro_wake_word/common.yaml +++ b/tests/components/micro_wake_word/common.yaml @@ -1,3 +1,6 @@ +psram: + mode: quad + i2s_audio: i2s_lrclk_pin: GPIO18 i2s_bclk_pin: GPIO19 @@ -12,6 +15,7 @@ microphone: micro_wake_word: microphone: echo_microphone + task_stack_in_psram: true on_wake_word_detected: - logger.log: "Wake word detected" - micro_wake_word.stop: diff --git a/tests/components/mitsubishi_cn105/climate/mitsubishi_cn105_climate_tests.cpp b/tests/components/mitsubishi_cn105/climate/mitsubishi_cn105_climate_tests.cpp new file mode 100644 index 0000000000..36e0fc90b4 --- /dev/null +++ b/tests/components/mitsubishi_cn105/climate/mitsubishi_cn105_climate_tests.cpp @@ -0,0 +1,165 @@ +#include "../common.h" + +namespace esphome::mitsubishi_cn105::testing { + +TEST(MitsubishiCN105ClimateTests, SupportedSwingModeOffLeavesTraitsEmpty) { + TestableMitsubishiCN105Climate sut; + + sut.set_supported_swing_mode(climate::CLIMATE_SWING_OFF); + + EXPECT_FALSE(sut.traits().get_supports_swing_modes()); +} + +TEST(MitsubishiCN105ClimateTests, SupportedSwingModeVerticalExposesOffAndVertical) { + TestableMitsubishiCN105Climate sut; + + sut.set_supported_swing_mode(climate::CLIMATE_SWING_VERTICAL); + + EXPECT_TRUE(sut.traits().supports_swing_mode(climate::CLIMATE_SWING_OFF)); + EXPECT_TRUE(sut.traits().supports_swing_mode(climate::CLIMATE_SWING_VERTICAL)); + EXPECT_FALSE(sut.traits().supports_swing_mode(climate::CLIMATE_SWING_HORIZONTAL)); + EXPECT_FALSE(sut.traits().supports_swing_mode(climate::CLIMATE_SWING_BOTH)); +} + +TEST(MitsubishiCN105ClimateTests, SupportedSwingModeHorizontalExposesOffAndHorizontal) { + TestableMitsubishiCN105Climate sut; + + sut.set_supported_swing_mode(climate::CLIMATE_SWING_HORIZONTAL); + + EXPECT_TRUE(sut.traits().supports_swing_mode(climate::CLIMATE_SWING_OFF)); + EXPECT_FALSE(sut.traits().supports_swing_mode(climate::CLIMATE_SWING_VERTICAL)); + EXPECT_TRUE(sut.traits().supports_swing_mode(climate::CLIMATE_SWING_HORIZONTAL)); + EXPECT_FALSE(sut.traits().supports_swing_mode(climate::CLIMATE_SWING_BOTH)); +} + +TEST(MitsubishiCN105ClimateTests, SupportedSwingModeBothExposesAllExpectedModes) { + TestableMitsubishiCN105Climate sut; + + sut.set_supported_swing_mode(climate::CLIMATE_SWING_BOTH); + + EXPECT_TRUE(sut.traits().supports_swing_mode(climate::CLIMATE_SWING_OFF)); + EXPECT_TRUE(sut.traits().supports_swing_mode(climate::CLIMATE_SWING_VERTICAL)); + EXPECT_TRUE(sut.traits().supports_swing_mode(climate::CLIMATE_SWING_HORIZONTAL)); + EXPECT_TRUE(sut.traits().supports_swing_mode(climate::CLIMATE_SWING_BOTH)); +} + +TEST(MitsubishiCN105ClimateTests, ApplyValuesMapsVerticalSwingWhenSupported) { + TestableMitsubishiCN105Climate sut; + + sut.set_supported_swing_mode(climate::CLIMATE_SWING_VERTICAL); + + sut.status().vane_mode = MitsubishiCN105::VaneMode::SWING; + sut.status().wide_vane_mode = MitsubishiCN105::WideVaneMode::CENTER; + + sut.apply_values_(); + + EXPECT_EQ(sut.swing_mode, climate::CLIMATE_SWING_VERTICAL); +} + +TEST(MitsubishiCN105ClimateTests, ApplyValuesMapsHorizontalSwingWhenSupported) { + TestableMitsubishiCN105Climate sut; + + sut.set_supported_swing_mode(climate::CLIMATE_SWING_HORIZONTAL); + + sut.status().vane_mode = MitsubishiCN105::VaneMode::AUTO; + sut.status().wide_vane_mode = MitsubishiCN105::WideVaneMode::SWING; + + sut.apply_values_(); + + EXPECT_EQ(sut.swing_mode, climate::CLIMATE_SWING_HORIZONTAL); +} + +TEST(MitsubishiCN105ClimateTests, ApplyValuesMapsBothSwingWhenSupported) { + TestableMitsubishiCN105Climate sut; + + sut.set_supported_swing_mode(climate::CLIMATE_SWING_BOTH); + + sut.status().vane_mode = MitsubishiCN105::VaneMode::SWING; + sut.status().wide_vane_mode = MitsubishiCN105::WideVaneMode::SWING; + + sut.apply_values_(); + + EXPECT_EQ(sut.swing_mode, climate::CLIMATE_SWING_BOTH); +} + +TEST(MitsubishiCN105ClimateTests, ApplyValuesMapsSwingOffWhenNoSwingActive) { + TestableMitsubishiCN105Climate sut; + + sut.set_supported_swing_mode(climate::CLIMATE_SWING_BOTH); + + sut.status().vane_mode = MitsubishiCN105::VaneMode::POSITION_3; + sut.status().wide_vane_mode = MitsubishiCN105::WideVaneMode::CENTER; + + sut.apply_values_(); + + EXPECT_EQ(sut.swing_mode, climate::CLIMATE_SWING_OFF); +} + +TEST(MitsubishiCN105ClimateTests, ApplyValuesRemembersLastNonSwingPositions) { + TestableMitsubishiCN105Climate sut; + + sut.set_supported_swing_mode(climate::CLIMATE_SWING_BOTH); + + sut.status().vane_mode = MitsubishiCN105::VaneMode::POSITION_4; + sut.status().wide_vane_mode = MitsubishiCN105::WideVaneMode::RIGHT; + + sut.apply_values_(); + + EXPECT_EQ(sut.last_non_swing_vane_mode_, MitsubishiCN105::VaneMode::POSITION_4); + EXPECT_EQ(sut.last_non_swing_wide_vane_mode_, MitsubishiCN105::WideVaneMode::RIGHT); + + sut.status().vane_mode = MitsubishiCN105::VaneMode::SWING; + sut.status().wide_vane_mode = MitsubishiCN105::WideVaneMode::SWING; + + sut.apply_values_(); + + EXPECT_EQ(sut.last_non_swing_vane_mode_, MitsubishiCN105::VaneMode::POSITION_4); + EXPECT_EQ(sut.last_non_swing_wide_vane_mode_, MitsubishiCN105::WideVaneMode::RIGHT); + EXPECT_EQ(sut.swing_mode, climate::CLIMATE_SWING_BOTH); +} + +TEST(MitsubishiCN105ClimateTests, ApplyValuesDoesNotOverwriteRememberedPositionWithUnknownValues) { + TestableMitsubishiCN105Climate sut; + + sut.set_supported_swing_mode(climate::CLIMATE_SWING_BOTH); + + sut.last_non_swing_vane_mode_ = MitsubishiCN105::VaneMode::POSITION_2; + sut.last_non_swing_wide_vane_mode_ = MitsubishiCN105::WideVaneMode::LEFT; + + sut.status().vane_mode = MitsubishiCN105::VaneMode::UNKNOWN; + sut.status().wide_vane_mode = MitsubishiCN105::WideVaneMode::UNKNOWN; + + sut.apply_values_(); + + EXPECT_EQ(sut.last_non_swing_vane_mode_, MitsubishiCN105::VaneMode::POSITION_2); + EXPECT_EQ(sut.last_non_swing_wide_vane_mode_, MitsubishiCN105::WideVaneMode::LEFT); + EXPECT_EQ(sut.swing_mode, climate::CLIMATE_SWING_OFF); +} + +TEST(MitsubishiCN105ClimateTests, ApplyValuesIgnoresUnsupportedVerticalSwingState) { + TestableMitsubishiCN105Climate sut; + + sut.set_supported_swing_mode(climate::CLIMATE_SWING_HORIZONTAL); + + sut.status().vane_mode = MitsubishiCN105::VaneMode::SWING; + sut.status().wide_vane_mode = MitsubishiCN105::WideVaneMode::CENTER; + + sut.apply_values_(); + + EXPECT_EQ(sut.swing_mode, climate::CLIMATE_SWING_OFF); +} + +TEST(MitsubishiCN105ClimateTests, ApplyValuesIgnoresUnsupportedHorizontalSwingState) { + TestableMitsubishiCN105Climate sut; + + sut.set_supported_swing_mode(climate::CLIMATE_SWING_VERTICAL); + + sut.status().vane_mode = MitsubishiCN105::VaneMode::AUTO; + sut.status().wide_vane_mode = MitsubishiCN105::WideVaneMode::SWING; + + sut.apply_values_(); + + EXPECT_EQ(sut.swing_mode, climate::CLIMATE_SWING_OFF); +} + +} // namespace esphome::mitsubishi_cn105::testing diff --git a/tests/components/mitsubishi_cn105/common.h b/tests/components/mitsubishi_cn105/common.h index 59b6203732..798f7283f6 100644 --- a/tests/components/mitsubishi_cn105/common.h +++ b/tests/components/mitsubishi_cn105/common.h @@ -8,6 +8,7 @@ #include #include "esphome/components/uart/uart_component.h" #include "esphome/components/mitsubishi_cn105/mitsubishi_cn105.h" +#include "esphome/components/mitsubishi_cn105/mitsubishi_cn105_climate.h" namespace esphome::mitsubishi_cn105::testing { @@ -44,6 +45,7 @@ class TestableMitsubishiCN105 : public MitsubishiCN105 { using MitsubishiCN105::State; using MitsubishiCN105::UpdateFlag; using MitsubishiCN105::state_; + using MitsubishiCN105::status_; using MitsubishiCN105::operation_start_ms_; using MitsubishiCN105::use_temperature_encoding_b_; using MitsubishiCN105::set_wide_vane_high_bit_; @@ -58,4 +60,13 @@ class TestableMitsubishiCN105 : public MitsubishiCN105 { void set_current_time(uint32_t ms) { test_loop_time_ms = ms; } }; +class TestableMitsubishiCN105Climate : public MitsubishiCN105Climate { + public: + using MitsubishiCN105Climate::apply_values_; + using MitsubishiCN105Climate::last_non_swing_vane_mode_; + using MitsubishiCN105Climate::last_non_swing_wide_vane_mode_; + + MitsubishiCN105::Status &status() { return static_cast(this->hp_).status_; } +}; + } // namespace esphome::mitsubishi_cn105::testing diff --git a/tests/components/mitsubishi_cn105/common.yaml b/tests/components/mitsubishi_cn105/common.yaml index 4b64f51261..5b9c3aaaf6 100644 --- a/tests/components/mitsubishi_cn105/common.yaml +++ b/tests/components/mitsubishi_cn105/common.yaml @@ -3,6 +3,9 @@ climate: id: ac name: "AC Test" uart_id: uart_bus + update_interval: 30s + current_temperature_min_interval: 120s + supported_swing_modes: BOTH esphome: on_boot: diff --git a/tests/components/mixer/common.yaml b/tests/components/mixer/common.yaml index e171b9499c..ef613b82bc 100644 --- a/tests/components/mixer/common.yaml +++ b/tests/components/mixer/common.yaml @@ -16,8 +16,12 @@ speaker: id: speaker_id dac_type: external i2s_dout_pin: ${dout_pin} + bits_per_sample: 32bit + channel: stereo - platform: mixer output_speaker: speaker_id + bits_per_sample: 32 + num_channels: 2 source_speakers: - id: source_speaker_1_id - id: source_speaker_2_id diff --git a/tests/components/router/common.yaml b/tests/components/router/common.yaml new file mode 100644 index 0000000000..f1239de3cb --- /dev/null +++ b/tests/components/router/common.yaml @@ -0,0 +1,40 @@ +esphome: + on_boot: + then: + - router.speaker.switch_output: + id: router_id + target_speaker: speaker_b_id + # id omitted: auto-resolved since there's a single router instance + - router.speaker.switch_output: + target_speaker: !lambda return id(speaker_a_id); + +i2s_audio: + i2s_lrclk_pin: ${a_lrclk_pin} + i2s_bclk_pin: ${a_bclk_pin} + +speaker: + - platform: i2s_audio + id: speaker_a_id + dac_type: external + i2s_dout_pin: ${a_dout_pin} + sample_rate: 48000 + bits_per_sample: 16bit + channel: stereo + - platform: i2s_audio + id: speaker_b_id + dac_type: external + i2s_dout_pin: ${b_dout_pin} + spdif_mode: true + use_apll: true + sample_rate: 48000 + bits_per_sample: 16bit + channel: stereo + i2s_mode: primary + - platform: router + id: router_id + output_speakers: + - speaker_a_id + - speaker_b_id + sample_rate: 48000 + bits_per_sample: 16 + num_channels: 2 diff --git a/tests/components/router/test.esp32-idf.yaml b/tests/components/router/test.esp32-idf.yaml new file mode 100644 index 0000000000..241a9a8903 --- /dev/null +++ b/tests/components/router/test.esp32-idf.yaml @@ -0,0 +1,7 @@ +substitutions: + a_lrclk_pin: GPIO4 + a_bclk_pin: GPIO5 + a_dout_pin: GPIO14 + b_dout_pin: GPIO19 + +<<: !include common.yaml diff --git a/tests/components/rp2040/test.rp2040-ard.yaml b/tests/components/rp2040/test.rp2040-ard.yaml index 1eb315a3b4..09531f914e 100644 --- a/tests/components/rp2040/test.rp2040-ard.yaml +++ b/tests/components/rp2040/test.rp2040-ard.yaml @@ -1,4 +1,5 @@ rp2040: + variant: rp2040 enable_full_printf: false logger: diff --git a/tests/components/rp2040/test.rp2040-pico2-ard.yaml b/tests/components/rp2040/test.rp2040-pico2-ard.yaml new file mode 100644 index 0000000000..c9d795840d --- /dev/null +++ b/tests/components/rp2040/test.rp2040-pico2-ard.yaml @@ -0,0 +1,6 @@ +rp2040: + variant: rp2350 + enable_full_printf: false + +logger: + level: VERBOSE diff --git a/tests/dashboard/test_web_server_paths.py b/tests/dashboard/test_web_server_paths.py index b596ebb581..efeafbf3b5 100644 --- a/tests/dashboard/test_web_server_paths.py +++ b/tests/dashboard/test_web_server_paths.py @@ -34,9 +34,7 @@ def test_get_base_frontend_path_dev_mode() -> None: # The function uses Path.resolve() which resolves symlinks # The actual function adds "/" to the path, so we simulate that test_path_with_slash = test_path if test_path.endswith("/") else test_path + "/" - expected = ( - Path(os.getcwd()) / test_path_with_slash / "esphome_dashboard" - ).resolve() + expected = (Path.cwd() / test_path_with_slash / "esphome_dashboard").resolve() assert result == expected @@ -62,9 +60,7 @@ def test_get_base_frontend_path_dev_mode_relative_path() -> None: # The function uses Path.resolve() which resolves symlinks # The actual function adds "/" to the path, so we simulate that test_path_with_slash = test_path if test_path.endswith("/") else test_path + "/" - expected = ( - Path(os.getcwd()) / test_path_with_slash / "esphome_dashboard" - ).resolve() + expected = (Path.cwd() / test_path_with_slash / "esphome_dashboard").resolve() assert result == expected assert result.is_absolute() @@ -157,7 +153,7 @@ def test_load_file_path(tmp_path: Path) -> None: test_file = tmp_path / "test.txt" test_file.write_bytes(b"test content") - with open(test_file, "rb") as f: + with test_file.open("rb") as f: content = f.read() assert content == b"test content" diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index fb025ce427..a9c9e0686f 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -79,7 +79,7 @@ def shared_platformio_cache() -> Generator[Path]: lock_file = Path.home() / ".esphome-integration-tests-init.lock" # Always acquire the lock to ensure cache is ready before proceeding - with open(lock_file, "w") as lock_fd: + with lock_file.open("w") as lock_fd: fcntl.flock(lock_fd.fileno(), fcntl.LOCK_EX) # Check if the native platform is installed (the actual indicator of a populated cache) @@ -407,8 +407,10 @@ async def wait_and_connect_api_client( # Wait for connection with timeout try: await asyncio.wait_for(connected_future, timeout=timeout) - except TimeoutError: - raise TimeoutError(f"Failed to connect to API after {timeout} seconds") + except TimeoutError as err: + raise TimeoutError( + f"Failed to connect to API after {timeout} seconds" + ) from err if return_disconnect_event: yield client, disconnect_event diff --git a/tests/integration/test_gpio_expander_cache.py b/tests/integration/test_gpio_expander_cache.py index e5f0f2818f..1d36ca3446 100644 --- a/tests/integration/test_gpio_expander_cache.py +++ b/tests/integration/test_gpio_expander_cache.py @@ -43,7 +43,7 @@ async def test_gpio_expander_cache( # ensure logs are in the expected order log_order = [ (digital_read_hw_pattern, 0), - [(digital_read_cache_pattern, i) for i in range(0, 8)], + [(digital_read_cache_pattern, i) for i in range(8)], (digital_read_hw_pattern, 8), [(digital_read_cache_pattern, i) for i in range(8, 16)], (digital_read_hw_pattern, 16), @@ -68,7 +68,7 @@ async def test_gpio_expander_cache( # uint16_t component tests (single bank of 16 pins) (uint16_read_hw_pattern, 0), # First pin triggers hw read [ - (uint16_read_cache_pattern, i) for i in range(0, 16) + (uint16_read_cache_pattern, i) for i in range(16) ], # All 16 pins return via cache # After cache reset (uint16_read_hw_pattern, 5), # First read after reset triggers hw diff --git a/tests/script/test_build_language_schema.py b/tests/script/test_build_language_schema.py index 59b8c7484b..dd1d88e74c 100644 --- a/tests/script/test_build_language_schema.py +++ b/tests/script/test_build_language_schema.py @@ -3,8 +3,11 @@ from __future__ import annotations import ast +import importlib.util from pathlib import Path +from esphome import config_validation as cv + SCRIPT_PATH = ( Path(__file__).resolve().parent.parent.parent / "script" @@ -12,10 +15,16 @@ SCRIPT_PATH = ( ) +def _load_script_module(): + spec = importlib.util.spec_from_file_location("build_language_schema", SCRIPT_PATH) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + def _extract_sort_obj(): - # build_language_schema.py runs argparse, loads every component, and - # calls build_schema() at import time, so a plain import isn't viable - # in a unit test. Pull just the pure helper out via AST instead. + # ``sort_obj`` is pure and self-contained; pulling it via AST avoids + # exercising the module-level component-loading state for these tests. tree = ast.parse(SCRIPT_PATH.read_text()) for node in tree.body: if isinstance(node, ast.FunctionDef) and node.name == "sort_obj": @@ -27,6 +36,7 @@ def _extract_sort_obj(): sort_obj = _extract_sort_obj() +_bls = _load_script_module() def test_sort_obj_sorts_dict_keys() -> None: @@ -96,3 +106,56 @@ def test_sort_obj_passes_through_scalars() -> None: assert sort_obj(42) == 42 assert sort_obj(None) is None assert sort_obj(True) is True + + +def test_convert_emits_explicit_sensitive_marker() -> None: + config_var: dict = {} + _bls.convert(cv.sensitive(cv.string), config_var, "/test") + + assert config_var["sensitive"] is True + assert config_var["sensitive_source"] == "explicit" + assert config_var["type"] == "string" + + +def test_convert_keys_emits_heuristic_sensitive_marker() -> None: + converted: dict = {} + _bls.convert_keys(converted, {cv.Optional("password"): cv.string}, "/root") + + entry = converted["schema"]["config_vars"]["password"] + assert entry["sensitive"] is True + assert entry["sensitive_source"] == "heuristic" + assert entry["type"] == "string" + + +def test_convert_keys_explicit_beats_heuristic() -> None: + # Key name matches a fragment but the validator is explicitly wrapped; + # the explicit branch should win and emit ``sensitive_source: explicit``. + converted: dict = {} + _bls.convert_keys( + converted, {cv.Optional("password"): cv.sensitive(cv.string)}, "/root" + ) + + entry = converted["schema"]["config_vars"]["password"] + assert entry["sensitive"] is True + assert entry["sensitive_source"] == "explicit" + + +def test_convert_keys_no_heuristic_for_non_string_leaves() -> None: + # Even though the key contains a fragment, a non-string leaf must not + # be flagged. Prevents false positives on unrelated fields whose name + # happens to embed a substring like "token". + converted: dict = {} + _bls.convert_keys(converted, {cv.Optional("password"): cv.boolean}, "/root") + + entry = converted["schema"]["config_vars"]["password"] + assert "sensitive" not in entry + assert "sensitive_source" not in entry + + +def test_convert_keys_no_marker_for_non_sensitive_field() -> None: + converted: dict = {} + _bls.convert_keys(converted, {cv.Optional("hostname"): cv.string}, "/root") + + entry = converted["schema"]["config_vars"]["hostname"] + assert "sensitive" not in entry + assert "sensitive_source" not in entry diff --git a/tests/script/test_check_import_time.py b/tests/script/test_check_import_time.py index 223c58002c..528ca0701c 100644 --- a/tests/script/test_check_import_time.py +++ b/tests/script/test_check_import_time.py @@ -4,7 +4,6 @@ from __future__ import annotations import importlib.util import json -import os from pathlib import Path import sys from unittest.mock import patch @@ -13,12 +12,10 @@ import pytest # Load the script-under-test as `check_import_time` (it's a hyphenated path # inside `script/` that mirrors the existing `determine_jobs` pattern). -script_dir = os.path.abspath( - os.path.join(os.path.dirname(__file__), "..", "..", "script") -) +script_dir = str((Path(__file__).parent / ".." / ".." / "script").resolve()) sys.path.insert(0, script_dir) spec = importlib.util.spec_from_file_location( - "check_import_time", os.path.join(script_dir, "check_import_time.py") + "check_import_time", str(Path(script_dir) / "check_import_time.py") ) check_import_time = importlib.util.module_from_spec(spec) spec.loader.exec_module(check_import_time) diff --git a/tests/script/test_determine_jobs.py b/tests/script/test_determine_jobs.py index 7bb9fe2543..ac3c6424bf 100644 --- a/tests/script/test_determine_jobs.py +++ b/tests/script/test_determine_jobs.py @@ -3,7 +3,6 @@ from collections.abc import Generator import importlib.util import json -import os from pathlib import Path import sys from unittest.mock import Mock, call, patch @@ -11,9 +10,7 @@ from unittest.mock import Mock, call, patch import pytest # Add the script directory to Python path so we can import the module -script_dir = os.path.abspath( - os.path.join(os.path.dirname(__file__), "..", "..", "script") -) +script_dir = str((Path(__file__).parent / ".." / ".." / "script").resolve()) sys.path.insert(0, script_dir) # Import helpers module for patching @@ -22,7 +19,7 @@ import helpers # noqa: E402 import script.helpers # noqa: E402 spec = importlib.util.spec_from_file_location( - "determine_jobs", os.path.join(script_dir, "determine-jobs.py") + "determine_jobs", str(Path(script_dir) / "determine-jobs.py") ) determine_jobs = importlib.util.module_from_spec(spec) spec.loader.exec_module(determine_jobs) diff --git a/tests/script/test_helpers.py b/tests/script/test_helpers.py index 10f258aa83..82ff5e1411 100644 --- a/tests/script/test_helpers.py +++ b/tests/script/test_helpers.py @@ -12,9 +12,7 @@ import pytest from pytest import MonkeyPatch # Add the script directory to Python path so we can import helpers -sys.path.insert( - 0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "script")) -) +sys.path.insert(0, str((Path(__file__).parent / ".." / ".." / "script").resolve())) import helpers # noqa: E402 diff --git a/tests/script/test_test_helpers.py b/tests/script/test_test_helpers.py index 3149712563..a8100252da 100644 --- a/tests/script/test_test_helpers.py +++ b/tests/script/test_test_helpers.py @@ -1,6 +1,5 @@ """Unit tests for script/build_helpers.py manifest override and build helpers.""" -import os from pathlib import Path import sys import textwrap @@ -9,9 +8,7 @@ from unittest.mock import MagicMock, patch import pytest # Add the script directory to Python path so we can import build_helpers -sys.path.insert( - 0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "script")) -) +sys.path.insert(0, str((Path(__file__).parent / ".." / ".." / "script").resolve())) import build_helpers # noqa: E402 diff --git a/tests/test_build_components/build_components_base.esp32-c6-idf.yaml b/tests/test_build_components/build_components_base.esp32-c6-idf.yaml index 9dbc465ca2..4105481dc5 100644 --- a/tests/test_build_components/build_components_base.esp32-c6-idf.yaml +++ b/tests/test_build_components/build_components_base.esp32-c6-idf.yaml @@ -4,6 +4,7 @@ esphome: esp32: board: esp32-c6-devkitc-1 + flash_size: 8MB framework: type: esp-idf diff --git a/tests/unit_tests/components/test_rp2040.py b/tests/unit_tests/components/test_rp2040.py index 25a9ade567..8e726933ed 100644 --- a/tests/unit_tests/components/test_rp2040.py +++ b/tests/unit_tests/components/test_rp2040.py @@ -1,6 +1,11 @@ -"""Tests for RP2040 component public helpers.""" +"""Tests for RP2040 component public helpers and variant detection.""" -from esphome.components.rp2040 import board_id_has_wifi +import pytest + +from esphome.components.rp2040 import _detect_variant, board_id_has_wifi +from esphome.components.rp2040.const import VARIANT_RP2040, VARIANT_RP2350 +import esphome.config_validation as cv +from esphome.const import CONF_BOARD, CONF_VARIANT def test_board_id_has_wifi_for_known_wifi_board() -> None: @@ -27,3 +32,61 @@ def test_board_id_has_wifi_for_unknown_board_returns_true() -> None: "no CYW43" guard at compile time. """ assert board_id_has_wifi("not-a-real-board-id") is True + + +def test_detect_variant_derives_variant_from_board() -> None: + """Board alone resolves to the matching variant.""" + result = _detect_variant({CONF_BOARD: "rpipicow"}) + assert result[CONF_BOARD] == "rpipicow" + assert result[CONF_VARIANT] == VARIANT_RP2040 + + +def test_detect_variant_derives_variant_from_rp2350_board() -> None: + """An RP2350 board resolves to ``RP2350``.""" + result = _detect_variant({CONF_BOARD: "rpipico2"}) + assert result[CONF_BOARD] == "rpipico2" + assert result[CONF_VARIANT] == VARIANT_RP2350 + + +def test_detect_variant_only_picks_default_board_rp2040() -> None: + """Variant alone picks Pico W as the canonical RP2040 board.""" + result = _detect_variant({CONF_VARIANT: VARIANT_RP2040}) + assert result[CONF_BOARD] == "rpipicow" + assert result[CONF_VARIANT] == VARIANT_RP2040 + + +def test_detect_variant_only_picks_default_board_rp2350() -> None: + """Variant alone picks Pico 2 W as the canonical RP2350 board.""" + result = _detect_variant({CONF_VARIANT: VARIANT_RP2350}) + assert result[CONF_BOARD] == "rpipico2w" + assert result[CONF_VARIANT] == VARIANT_RP2350 + + +def test_detect_variant_matching_explicit_variant_passes() -> None: + """Specifying both a board and the matching variant is allowed.""" + result = _detect_variant({CONF_BOARD: "rpipico2", CONF_VARIANT: VARIANT_RP2350}) + assert result[CONF_BOARD] == "rpipico2" + assert result[CONF_VARIANT] == VARIANT_RP2350 + + +def test_detect_variant_mismatched_variant_raises() -> None: + """Board/variant mismatch must be rejected and name the offending board.""" + with pytest.raises( + cv.Invalid, match=r"does not match the selected board 'rpipicow'" + ): + _detect_variant({CONF_BOARD: "rpipicow", CONF_VARIANT: VARIANT_RP2350}) + + +def test_detect_variant_unknown_board_without_variant_raises() -> None: + """Unknown board with no variant tells the user how to recover.""" + with pytest.raises(cv.Invalid, match="please specify the chip variant"): + _detect_variant({CONF_BOARD: "not-a-real-board"}) + + +def test_detect_variant_unknown_board_with_variant_passes() -> None: + """Unknown board + explicit variant is accepted (with a warning).""" + result = _detect_variant( + {CONF_BOARD: "not-a-real-board", CONF_VARIANT: VARIANT_RP2040} + ) + assert result[CONF_BOARD] == "not-a-real-board" + assert result[CONF_VARIANT] == VARIANT_RP2040 diff --git a/tests/unit_tests/components/test_time.py b/tests/unit_tests/components/test_time.py index 6325bfbe75..5ae9d787d6 100644 --- a/tests/unit_tests/components/test_time.py +++ b/tests/unit_tests/components/test_time.py @@ -70,11 +70,11 @@ def test_numeric_offset_slash() -> None: def test_star() -> None: - assert _parse_cron_part("*", 0, 59, {}) == set(range(0, 60)) + assert _parse_cron_part("*", 0, 59, {}) == set(range(60)) def test_question() -> None: - assert _parse_cron_part("?", 0, 59, {}) == set(range(0, 60)) + assert _parse_cron_part("?", 0, 59, {}) == set(range(60)) def test_range() -> None: diff --git a/tests/unit_tests/core/test_config.py b/tests/unit_tests/core/test_config.py index 4ce862315d..b5b35b5172 100644 --- a/tests/unit_tests/core/test_config.py +++ b/tests/unit_tests/core/test_config.py @@ -486,7 +486,7 @@ def test_preload_core_config_basic(setup_core: Path) -> None: assert CONF_BUILD_PATH in config[CONF_ESPHOME] # Verify default build path is "build/" build_path = config[CONF_ESPHOME][CONF_BUILD_PATH] - assert build_path.endswith(os.path.join("build", "test_device")) + assert build_path.endswith(str(Path("build") / "test_device")) def test_preload_core_config_with_build_path(setup_core: Path) -> None: @@ -523,7 +523,7 @@ def test_preload_core_config_env_build_path(setup_core: Path) -> None: assert "test_device" in config[CONF_ESPHOME][CONF_BUILD_PATH] # Verify it uses the env var path with device name appended build_path = config[CONF_ESPHOME][CONF_BUILD_PATH] - expected_path = os.path.join("/env/build", "test_device") + expected_path = str(Path("/env/build") / "test_device") assert build_path == expected_path or build_path == expected_path.replace( "/", os.sep ) @@ -739,7 +739,7 @@ async def test_add_includes_with_single_file( """Test add_includes copies a single header file to build directory.""" CORE.config_path = tmp_path / "config.yaml" CORE.build_path = tmp_path / "build" - os.makedirs(CORE.build_path, exist_ok=True) + CORE.build_path.mkdir(parents=True, exist_ok=True) # Create include file include_file = tmp_path / "my_header.h" @@ -769,7 +769,7 @@ async def test_add_includes_with_directory_unix( """Test add_includes copies all files from a directory on Unix.""" CORE.config_path = tmp_path / "config.yaml" CORE.build_path = tmp_path / "build" - os.makedirs(CORE.build_path, exist_ok=True) + CORE.build_path.mkdir(parents=True, exist_ok=True) # Create include directory with files include_dir = tmp_path / "includes" @@ -814,7 +814,7 @@ async def test_add_includes_with_directory_windows( """Test add_includes copies all files from a directory on Windows.""" CORE.config_path = tmp_path / "config.yaml" CORE.build_path = tmp_path / "build" - os.makedirs(CORE.build_path, exist_ok=True) + CORE.build_path.mkdir(parents=True, exist_ok=True) # Create include directory with files include_dir = tmp_path / "includes" @@ -856,7 +856,7 @@ async def test_add_includes_with_multiple_sources( """Test add_includes with multiple files and directories.""" CORE.config_path = tmp_path / "config.yaml" CORE.build_path = tmp_path / "build" - os.makedirs(CORE.build_path, exist_ok=True) + CORE.build_path.mkdir(parents=True, exist_ok=True) # Create various include sources single_file = tmp_path / "single.h" @@ -884,7 +884,7 @@ async def test_add_includes_empty_directory( """Test add_includes with an empty directory doesn't fail.""" CORE.config_path = tmp_path / "config.yaml" CORE.build_path = tmp_path / "build" - os.makedirs(CORE.build_path, exist_ok=True) + CORE.build_path.mkdir(parents=True, exist_ok=True) # Create empty directory empty_dir = tmp_path / "empty" @@ -906,7 +906,7 @@ async def test_add_includes_preserves_directory_structure_unix( """Test that add_includes preserves relative directory structure on Unix.""" CORE.config_path = tmp_path / "config.yaml" CORE.build_path = tmp_path / "build" - os.makedirs(CORE.build_path, exist_ok=True) + CORE.build_path.mkdir(parents=True, exist_ok=True) # Create nested directory structure lib_dir = tmp_path / "lib" @@ -940,7 +940,7 @@ async def test_add_includes_preserves_directory_structure_windows( """Test that add_includes preserves relative directory structure on Windows.""" CORE.config_path = tmp_path / "config.yaml" CORE.build_path = tmp_path / "build" - os.makedirs(CORE.build_path, exist_ok=True) + CORE.build_path.mkdir(parents=True, exist_ok=True) # Create nested directory structure lib_dir = tmp_path / "lib" @@ -973,7 +973,7 @@ async def test_add_includes_overwrites_existing_files( """Test that add_includes overwrites existing files in build directory.""" CORE.config_path = tmp_path / "config.yaml" CORE.build_path = tmp_path / "build" - os.makedirs(CORE.build_path, exist_ok=True) + CORE.build_path.mkdir(parents=True, exist_ok=True) # Create include file include_file = tmp_path / "header.h" diff --git a/tests/unit_tests/test_config_normalization.py b/tests/unit_tests/test_config_normalization.py index d70f3c24e0..4ec17b3c7c 100644 --- a/tests/unit_tests/test_config_normalization.py +++ b/tests/unit_tests/test_config_normalization.py @@ -67,7 +67,7 @@ def test_iter_component_configs_with_multi_conf(mock_get_component: Mock) -> Non configs = list(config.iter_component_configs(test_config)) assert len(configs) == 2 - for domain, component, conf in configs: + for domain, _component, conf in configs: assert domain == "switch" assert "name" in conf diff --git a/tests/unit_tests/test_config_validation.py b/tests/unit_tests/test_config_validation.py index fd6c0e95f2..74d9a5047a 100644 --- a/tests/unit_tests/test_config_validation.py +++ b/tests/unit_tests/test_config_validation.py @@ -27,6 +27,7 @@ from esphome.const import ( SCHEDULER_DONT_RUN, ) from esphome.core import CORE, HexInt, Lambda +from esphome.yaml_util import SensitiveStr def test_check_not_templatable__invalid(): @@ -127,6 +128,85 @@ def test_string_string__invalid(value): config_validation.string_strict(value) +def test_sensitive__default_delegates_to_string() -> None: + validator = config_validation.sensitive() + + assert isinstance(validator, config_validation.SensitiveValidator) + assert validator.inner is config_validation.string + assert validator("hunter2") == "hunter2" + assert validator(42) == "42" + + +def test_sensitive__custom_inner_delegates_validation() -> None: + validator = config_validation.sensitive(config_validation.string_strict) + + assert validator.inner is config_validation.string_strict + assert validator("abc") == "abc" + with pytest.raises(Invalid, match="Must be string, got"): + validator(123) + + +def test_sensitive__wraps_string_result_in_sensitive_str() -> None: + validator = config_validation.sensitive() + result = validator("hunter2") + + assert isinstance(result, SensitiveStr) + assert isinstance(result, str) + assert result == "hunter2" + + +def test_sensitive__does_not_double_tag_already_sensitive() -> None: + # If the inner validator already returns a SensitiveStr (e.g., nested + # cv.sensitive wrappers), re-tagging is a no-op rather than a new + # SensitiveStr around the same value. + pre_tagged = SensitiveStr("hunter2") + + def inner(_value): + return pre_tagged + + validator = config_validation.sensitive(inner) + result = validator("anything") + + assert result is pre_tagged + + +def test_sensitive__non_string_result_passes_through() -> None: + # If an inner validator returns something other than a string (e.g., a + # Lambda template), the sensitive wrapper must not coerce it. + sentinel = object() + + def inner(_value): + return sentinel + + validator = config_validation.sensitive(inner) + assert validator("anything") is sentinel + + +def test_sensitive__is_detectable_via_isinstance() -> None: + validator = config_validation.sensitive() + + assert isinstance(validator, config_validation.SensitiveValidator) + + +def test_sensitive__repr_mirrors_inner() -> None: + # The schema dump dedups on ``repr(schema)``; mirroring the inner + # validator's repr keeps two ``cv.sensitive(cv.string)`` wrappers + # interchangeable for that purpose and avoids leaking the wrapper as + # noise in voluptuous error messages. + assert repr(config_validation.sensitive(config_validation.string)) == repr( + config_validation.string + ) + assert repr(config_validation.sensitive(config_validation.string)) == repr( + config_validation.sensitive(config_validation.string) + ) + + +def test_sensitive_key_fragments__covers_common_terms() -> None: + assert isinstance(config_validation.SENSITIVE_KEY_FRAGMENTS, frozenset) + for term in ("password", "passcode", "secret", "token", "api_key", "apikey", "psk"): + assert term in config_validation.SENSITIVE_KEY_FRAGMENTS + + @given( builds( lambda v: "mdi:" + v, diff --git a/tests/unit_tests/test_espidf_component.py b/tests/unit_tests/test_espidf_component.py index 7d6c861ffd..4f0a71053d 100644 --- a/tests/unit_tests/test_espidf_component.py +++ b/tests/unit_tests/test_espidf_component.py @@ -261,9 +261,14 @@ def test_check_library_data_invalid_platform(esp32_idf_core): _check_library_data({"platforms": ["other"], "frameworks": "*"}) -def test_check_library_data_invalid_framework(esp32_idf_core): - with pytest.raises(InvalidIDFComponent): - _check_library_data({"platforms": "*", "frameworks": ["other"]}) +def test_check_library_data_invalid_framework( + esp32_idf_core: None, caplog: pytest.LogCaptureFixture +) -> None: + # Framework mismatch is a warning, not a hard skip: the library is still + # included so that PIO manifests that only list "arduino" (but actually + # compile under IDF) can be used without forking them. + _check_library_data({"name": "lib", "platforms": "*", "frameworks": ["other"]}) + assert "do not include 'espidf'" in caplog.text def test_extra_script_captures_libpath_libs_and_defines(tmp_path): @@ -288,7 +293,7 @@ def test_extra_script_captures_libpath_libs_and_defines(tmp_path): result = run_extra_script(script, library_dir=tmp_path, idf_target="esp32") - assert result.libpath == [os.path.join("src", "esp32")] + assert result.libpath == [str(Path("src") / "esp32")] assert result.libs == ["algobsec"] assert ("BAR", "1") in result.cppdefines assert "FOO" in result.cppdefines diff --git a/tests/unit_tests/test_espidf_framework.py b/tests/unit_tests/test_espidf_framework.py new file mode 100644 index 0000000000..9f4e4fcca8 --- /dev/null +++ b/tests/unit_tests/test_espidf_framework.py @@ -0,0 +1,156 @@ +"""Tests for esphome.espidf.framework helpers.""" + +# pylint: disable=protected-access + +from pathlib import Path +from unittest.mock import patch + +import pytest + +from esphome.espidf.framework import _clone_idf_with_submodules, _parse_git_source + + +@pytest.mark.parametrize( + ("source", "expected"), + [ + # github:// shorthand + ( + "github://espressif/esp-idf", + ("https://github.com/espressif/esp-idf.git", None), + ), + ( + "github://espressif/esp-idf@master", + ("https://github.com/espressif/esp-idf.git", "master"), + ), + ( + "github://espressif/esp-idf@release/v6.0", + ("https://github.com/espressif/esp-idf.git", "release/v6.0"), + ), + # explicit https://github.com/...git URL + ( + "https://github.com/espressif/esp-idf.git", + ("https://github.com/espressif/esp-idf.git", None), + ), + ( + "https://github.com/espressif/esp-idf.git@master", + ("https://github.com/espressif/esp-idf.git", "master"), + ), + ( + "https://github.com/espressif/esp-idf.git@v6.0.1", + ("https://github.com/espressif/esp-idf.git", "v6.0.1"), + ), + # Tolerate a trailing ".git" on the shorthand so the user doesn't + # silently end up with a doubled "...esp-idf.git.git" URL. + ( + "github://espressif/esp-idf.git", + ("https://github.com/espressif/esp-idf.git", None), + ), + ( + "github://espressif/esp-idf.git@master", + ("https://github.com/espressif/esp-idf.git", "master"), + ), + ], +) +def test_parse_git_source_recognized( + source: str, expected: tuple[str, str | None] +) -> None: + assert _parse_git_source(source) == expected + + +@pytest.mark.parametrize( + "source", + [ + # archive URLs fall through to the existing download path + "https://github.com/espressif/esp-idf/archive/refs/heads/master.zip", + "https://dl.espressif.com/dl/esp-idf/v6.0.1/esp-idf-v6.0.1.zip", + "https://github.com/esphome-libs/esp-idf/releases/download/v5.5.4/esp-idf-v5.5.4.tar.xz", + # SSH and other git protocols are intentionally rejected — match + # external_components, which only recognizes github:// + structured + # dicts for these. + "git@github.com:espressif/esp-idf.git", + "ssh://git@github.com/espressif/esp-idf.git", + "git://github.com/espressif/esp-idf.git", + # non-GitHub .git URLs are intentionally rejected for the same reason + "https://gitlab.com/foo/bar.git", + "https://github.example.com/foo/bar.git", + ], +) +def test_parse_git_source_rejected(source: str) -> None: + assert _parse_git_source(source) is None + + +def _make_idf_tree(framework_path: Path) -> None: + """Create the minimum tree _clone_idf_with_submodules sanity-checks for.""" + (framework_path / "tools").mkdir(parents=True) + (framework_path / "tools" / "idf_tools.py").write_text("# stub\n") + + +def test_clone_idf_with_submodules_without_ref(tmp_path: Path) -> None: + framework_path = tmp_path / "idf" + framework_path.mkdir() + _make_idf_tree(framework_path) + + with patch("esphome.git.run_git_command", return_value="") as run_git_command_mock: + _clone_idf_with_submodules( + framework_path, "https://github.com/espressif/esp-idf.git", None + ) + + # No ref -> just clone + submodule update, no fetch/reset. + calls = [c.args[0] for c in run_git_command_mock.call_args_list] + assert calls[0] == [ + "git", + "clone", + "--depth=1", + "--", + "https://github.com/espressif/esp-idf.git", + str(framework_path), + ] + assert calls[-1][:5] == ["git", "submodule", "update", "--init", "--recursive"] + assert not any(c[1] == "fetch" for c in calls) + assert not any(c[1] == "reset" for c in calls) + + +def test_clone_idf_with_submodules_with_ref(tmp_path: Path) -> None: + framework_path = tmp_path / "idf" + framework_path.mkdir() + _make_idf_tree(framework_path) + + with patch("esphome.git.run_git_command", return_value="") as run_git_command_mock: + _clone_idf_with_submodules( + framework_path, + "https://github.com/espressif/esp-idf.git", + "master", + ) + + calls = [c.args[0] for c in run_git_command_mock.call_args_list] + # clone, fetch ref, reset hard, submodule update + assert calls[0][:2] == ["git", "clone"] + assert calls[1] == [ + "git", + "fetch", + "--depth=1", + "--", + "origin", + "master", + ] + assert calls[2] == ["git", "reset", "--hard", "FETCH_HEAD"] + assert calls[3][:5] == ["git", "submodule", "update", "--init", "--recursive"] + + +def test_clone_idf_with_submodules_raises_when_tree_missing( + tmp_path: Path, +) -> None: + framework_path = tmp_path / "idf" + framework_path.mkdir() + # Deliberately do NOT call _make_idf_tree — simulate a clone that + # returned 0 but produced no tools/idf_tools.py. + + with ( + patch("esphome.git.run_git_command", return_value=""), + pytest.raises(RuntimeError, match="no usable ESP-IDF tree"), + ): + _clone_idf_with_submodules( + framework_path, + "https://github.com/espressif/esp-idf.git", + None, + ) diff --git a/tests/unit_tests/test_external_files.py b/tests/unit_tests/test_external_files.py index 865487f617..dfa7d0d4e2 100644 --- a/tests/unit_tests/test_external_files.py +++ b/tests/unit_tests/test_external_files.py @@ -120,7 +120,7 @@ def test_is_file_recent_with_old_file(setup_core: Path) -> None: old_time = time.time() - 7200 mock_stat = MagicMock() - mock_stat.st_ctime = old_time + mock_stat.st_mtime = old_time with patch.object(Path, "stat", return_value=mock_stat): refresh = TimePeriod(seconds=3600) @@ -147,7 +147,7 @@ def test_is_file_recent_with_zero_refresh(setup_core: Path) -> None: # Mock stat to return a time 10 seconds ago mock_stat = MagicMock() - mock_stat.st_ctime = time.time() - 10 + mock_stat.st_mtime = time.time() - 10 with patch.object(Path, "stat", return_value=mock_stat): refresh = TimePeriod(seconds=0) result = external_files.is_file_recent(test_file, refresh) diff --git a/tests/unit_tests/test_git.py b/tests/unit_tests/test_git.py index eab6bfc2cb..62d2344069 100644 --- a/tests/unit_tests/test_git.py +++ b/tests/unit_tests/test_git.py @@ -1,10 +1,10 @@ """Tests for git.py module.""" -from datetime import datetime, timedelta import os from pathlib import Path +import time from typing import Any -from unittest.mock import Mock +from unittest.mock import Mock, patch import pytest @@ -34,9 +34,9 @@ def _setup_old_repo(repo_dir: Path, days_old: int = 2) -> None: # Create FETCH_HEAD file with old timestamp fetch_head = git_dir / "FETCH_HEAD" fetch_head.write_text("test") - old_time = datetime.now() - timedelta(days=days_old) + old_time = time.time() - days_old * 86400 fetch_head.touch() - os.utime(fetch_head, (old_time.timestamp(), old_time.timestamp())) + os.utime(fetch_head, (old_time, old_time)) def _get_git_command_type(cmd: list[str]) -> str | None: @@ -285,10 +285,10 @@ def test_clone_or_update_with_refresh_updates_old_repo( # Create FETCH_HEAD file with old timestamp (2 days ago) fetch_head = git_dir / "FETCH_HEAD" fetch_head.write_text("test") - old_time = datetime.now() - timedelta(days=2) + old_time = time.time() - 2 * 86400 fetch_head.touch() # Create the file # Set modification time to 2 days ago - os.utime(fetch_head, (old_time.timestamp(), old_time.timestamp())) + os.utime(fetch_head, (old_time, old_time)) # Mock git command responses mock_run_git_command.return_value = "abc123" # SHA for rev-parse @@ -333,10 +333,10 @@ def test_clone_or_update_with_refresh_skips_fresh_repo( # Create FETCH_HEAD file with recent timestamp (1 hour ago) fetch_head = git_dir / "FETCH_HEAD" fetch_head.write_text("test") - recent_time = datetime.now() - timedelta(hours=1) + recent_time = time.time() - 3600 fetch_head.touch() # Create the file # Set modification time to 1 hour ago - os.utime(fetch_head, (recent_time.timestamp(), recent_time.timestamp())) + os.utime(fetch_head, (recent_time, recent_time)) # Call with refresh=1d (1 day) refresh = TimePeriodSeconds(days=1) @@ -409,10 +409,10 @@ def test_clone_or_update_with_none_refresh_always_updates( # Create FETCH_HEAD file with very recent timestamp (1 second ago) fetch_head = git_dir / "FETCH_HEAD" fetch_head.write_text("test") - recent_time = datetime.now() - timedelta(seconds=1) + recent_time = time.time() - 1 fetch_head.touch() # Create the file # Set modification time to 1 second ago - os.utime(fetch_head, (recent_time.timestamp(), recent_time.timestamp())) + os.utime(fetch_head, (recent_time, recent_time)) # Mock git command responses mock_run_git_command.return_value = "abc123" # SHA for rev-parse @@ -1001,3 +1001,304 @@ def test_refresh_picks_up_new_remote_commits( "--hard", "old_sha", ] + + +def test_resolve_symlink_stub_returns_none_on_non_windows( + tmp_path: Path, mock_run_git_command: Mock +) -> None: + """On non-Windows, resolve_symlink_stub returns None without calling git.""" + repo_dir = tmp_path / "repo" + repo_dir.mkdir() + stub = repo_dir / "file.yaml" + stub.write_text("static/file.yaml") + + with patch("esphome.git.sys.platform", "linux"): + result = git.resolve_symlink_stub(repo_dir, stub) + + assert result is None + mock_run_git_command.assert_not_called() + + +def test_resolve_symlink_stub_returns_target_for_mode_120000( + tmp_path: Path, mock_run_git_command: Mock +) -> None: + """A mode-120000 file is recognised as a stub; its target Path is returned.""" + repo_dir = tmp_path / "repo" + repo_dir.mkdir() + (repo_dir / "static").mkdir() + + target = repo_dir / "static" / "real.yaml" + target.write_text("esphome:\n name: real\n") + + stub = repo_dir / "real.yaml" + stub.write_text("static/real.yaml") + + mock_run_git_command.return_value = "120000 abc123 0\treal.yaml" + + with patch("esphome.git.sys.platform", "win32"): + result = git.resolve_symlink_stub(repo_dir, stub) + + assert result == target.resolve() + # Stub file itself was not modified — only inspected. + assert stub.read_text() == "static/real.yaml" + + +def test_resolve_symlink_stub_resolves_relative_parent_paths( + tmp_path: Path, mock_run_git_command: Mock +) -> None: + """Symlink targets with ``..`` segments resolve correctly within the repo.""" + repo_dir = tmp_path / "repo" + (repo_dir / "subdir").mkdir(parents=True) + (repo_dir / "static").mkdir() + + target = repo_dir / "static" / "shared.yaml" + target.write_text("shared content") + + stub = repo_dir / "subdir" / "shared.yaml" + stub.write_text("../static/shared.yaml") + + mock_run_git_command.return_value = "120000 abc123 0\tsubdir/shared.yaml" + + with patch("esphome.git.sys.platform", "win32"): + result = git.resolve_symlink_stub(repo_dir, stub) + + assert result == target.resolve() + + +def test_resolve_symlink_stub_refuses_escape_outside_repo( + tmp_path: Path, mock_run_git_command: Mock +) -> None: + """A symlink pointing outside the repository is not followed.""" + outside = tmp_path / "outside.yaml" + outside.write_text("sensitive") + + repo_dir = tmp_path / "repo" + repo_dir.mkdir() + + stub = repo_dir / "escape.yaml" + stub.write_text("../outside.yaml") + + mock_run_git_command.return_value = "120000 abc123 0\tescape.yaml" + + with patch("esphome.git.sys.platform", "win32"): + result = git.resolve_symlink_stub(repo_dir, stub) + + assert result is None + + +def test_resolve_symlink_stub_returns_none_for_real_symlink( + tmp_path: Path, mock_run_git_command: Mock +) -> None: + """A real symlink already opens transparently, so the helper short-circuits. + + Skipped on Windows where symlink creation requires + SeCreateSymbolicLinkPrivilege. + """ + if os.name == "nt": + pytest.skip("Requires symlink-creation privilege on Windows") + + repo_dir = tmp_path / "repo" + repo_dir.mkdir() + target = repo_dir / "real.yaml" + target.write_text("real content") + + real_link = repo_dir / "link.yaml" + real_link.symlink_to("real.yaml") + + with patch("esphome.git.sys.platform", "win32"): + result = git.resolve_symlink_stub(repo_dir, real_link) + + assert result is None + # No git call needed for real symlinks. + mock_run_git_command.assert_not_called() + + +def test_resolve_symlink_stub_returns_none_for_regular_file( + tmp_path: Path, mock_run_git_command: Mock +) -> None: + """A regular file (mode 100644) whose content looks path-shaped is not + followed.""" + repo_dir = tmp_path / "repo" + repo_dir.mkdir() + + regular = repo_dir / "looks_like_path.txt" + regular.write_text("static/something.yaml") + + mock_run_git_command.return_value = "100644 abc123 0\tlooks_like_path.txt" + + with patch("esphome.git.sys.platform", "win32"): + result = git.resolve_symlink_stub(repo_dir, regular) + + assert result is None + + +def test_resolve_symlink_stub_returns_none_when_git_fails( + tmp_path: Path, mock_run_git_command: Mock +) -> None: + """If ``git ls-files`` fails (e.g. not a repo), the helper returns None.""" + repo_dir = tmp_path / "repo" + repo_dir.mkdir() + + stub = repo_dir / "real.yaml" + stub.write_text("static/real.yaml") + + mock_run_git_command.side_effect = GitCommandError("ls-files exploded") + + with patch("esphome.git.sys.platform", "win32"): + result = git.resolve_symlink_stub(repo_dir, stub) + + assert result is None + + +def test_resolve_symlink_stub_returns_none_for_non_utf8_content( + tmp_path: Path, mock_run_git_command: Mock +) -> None: + """A file whose bytes are not valid UTF-8 must not raise — return None.""" + repo_dir = tmp_path / "repo" + repo_dir.mkdir() + + stub = repo_dir / "binary.bin" + stub.write_bytes(b"\xff\xfe\x00\xff") + + mock_run_git_command.return_value = "120000 abc123 0\tbinary.bin" + + with patch("esphome.git.sys.platform", "win32"): + result = git.resolve_symlink_stub(repo_dir, stub) + + assert result is None + + +def test_resolve_symlink_stub_preserves_whitespace_in_target( + tmp_path: Path, mock_run_git_command: Mock +) -> None: + """Only trailing CR/LF is stripped — internal whitespace is preserved.""" + repo_dir = tmp_path / "repo" + repo_dir.mkdir() + target_dir = repo_dir / "dir with spaces" + target_dir.mkdir() + target = target_dir / "real.yaml" + target.write_text("hello") + + stub = repo_dir / "link.yaml" + # Trailing newline (as git's checkout may append) is stripped, but + # whitespace inside the target path itself must survive. + stub.write_bytes(b"dir with spaces/real.yaml\n") + + mock_run_git_command.return_value = "120000 abc123 0\tlink.yaml" + + with patch("esphome.git.sys.platform", "win32"): + result = git.resolve_symlink_stub(repo_dir, stub) + + assert result == target.resolve() + + +def test_resolve_symlink_stub_returns_none_for_directory_target( + tmp_path: Path, mock_run_git_command: Mock +) -> None: + """A symlink pointing at a directory has no file content to load.""" + repo_dir = tmp_path / "repo" + repo_dir.mkdir() + (repo_dir / "dir_target").mkdir() + + stub = repo_dir / "link_to_dir" + stub.write_text("dir_target") + + mock_run_git_command.return_value = "120000 abc123 0\tlink_to_dir" + + with patch("esphome.git.sys.platform", "win32"): + result = git.resolve_symlink_stub(repo_dir, stub) + + assert result is None + + +def test_resolve_symlink_stub_returns_none_when_resolve_raises( + tmp_path: Path, mock_run_git_command: Mock +) -> None: + """Path.resolve() raising (e.g. on a malformed target) must not propagate.""" + repo_dir = tmp_path / "repo" + repo_dir.mkdir() + + stub = repo_dir / "broken.yaml" + stub.write_text("ignored") + + mock_run_git_command.return_value = "120000 abc123 0\tbroken.yaml" + + with ( + patch("esphome.git.sys.platform", "win32"), + patch.object(Path, "resolve", side_effect=OSError("bad path")), + ): + result = git.resolve_symlink_stub(repo_dir, stub) + + assert result is None + + +def test_resolve_symlink_stub_returns_none_when_file_missing( + tmp_path: Path, mock_run_git_command: Mock +) -> None: + """A file path that doesn't exist is rejected before git is consulted.""" + repo_dir = tmp_path / "repo" + repo_dir.mkdir() + + missing = repo_dir / "ghost.yaml" # not created + + with patch("esphome.git.sys.platform", "win32"): + result = git.resolve_symlink_stub(repo_dir, missing) + + assert result is None + mock_run_git_command.assert_not_called() + + +def test_resolve_symlink_stub_returns_none_when_path_outside_repo( + tmp_path: Path, mock_run_git_command: Mock +) -> None: + """A file path that isn't under repo_dir is rejected (ValueError from relative_to).""" + repo_dir = tmp_path / "repo" + repo_dir.mkdir() + + outside = tmp_path / "stray.yaml" + outside.write_text("something") + + with patch("esphome.git.sys.platform", "win32"): + result = git.resolve_symlink_stub(repo_dir, outside) + + assert result is None + mock_run_git_command.assert_not_called() + + +def test_resolve_symlink_stub_returns_none_when_untracked( + tmp_path: Path, mock_run_git_command: Mock +) -> None: + """Empty `git ls-files` output (untracked file) makes the helper return None.""" + repo_dir = tmp_path / "repo" + repo_dir.mkdir() + + stub = repo_dir / "untracked.yaml" + stub.write_text("static/foo.yaml") + + mock_run_git_command.return_value = "" + + with patch("esphome.git.sys.platform", "win32"): + result = git.resolve_symlink_stub(repo_dir, stub) + + assert result is None + + +def test_resolve_symlink_stub_returns_none_when_read_bytes_raises( + tmp_path: Path, mock_run_git_command: Mock +) -> None: + """An OSError from read_bytes() (e.g. file vanished mid-call) must not propagate.""" + repo_dir = tmp_path / "repo" + repo_dir.mkdir() + + stub = repo_dir / "racy.yaml" + stub.write_text("static/racy.yaml") + + mock_run_git_command.return_value = "120000 abc123 0\tracy.yaml" + + with ( + patch("esphome.git.sys.platform", "win32"), + patch.object(Path, "read_bytes", side_effect=OSError("vanished")), + ): + result = git.resolve_symlink_stub(repo_dir, stub) + + assert result is None diff --git a/tests/unit_tests/test_helpers.py b/tests/unit_tests/test_helpers.py index bb00a15bee..efc2d8e42a 100644 --- a/tests/unit_tests/test_helpers.py +++ b/tests/unit_tests/test_helpers.py @@ -7,7 +7,7 @@ import stat from unittest.mock import MagicMock, patch from aioesphomeapi.host_resolver import AddrInfo, IPv4Sockaddr, IPv6Sockaddr -from hypothesis import given +from hypothesis import given, settings from hypothesis.strategies import ip_addresses import pytest @@ -151,6 +151,7 @@ def test_is_ip_address__invalid(host): assert actual is False +@settings(deadline=None) @given(value=ip_addresses(v=4).map(str)) def test_is_ip_address__valid(value): actual = helpers.is_ip_address(value) diff --git a/tests/unit_tests/test_main.py b/tests/unit_tests/test_main.py index 6ec0069b3a..26b550669f 100644 --- a/tests/unit_tests/test_main.py +++ b/tests/unit_tests/test_main.py @@ -11,7 +11,7 @@ from pathlib import Path import re import sys import time -from typing import Any +from typing import Any, Self from unittest.mock import AsyncMock, MagicMock, Mock, patch import pytest @@ -22,6 +22,7 @@ from esphome.__main__ import ( Purpose, _get_configured_xtal_freq, _make_crystal_freq_callback, + _redact_with_legacy_fallback, _resolve_network_devices, _validate_bootloader_binary, _validate_partition_table_binary, @@ -29,6 +30,7 @@ from esphome.__main__ import ( command_analyze_memory, command_bundle, command_clean_all, + command_config, command_config_hash, command_rename, command_run, @@ -340,6 +342,135 @@ def mock_ram_strings_analyzer() -> Generator[Mock]: yield mock_class +def test_redact_with_legacy_fallback__wraps_unmarked_field( + caplog: pytest.LogCaptureFixture, +) -> None: + """Unmarked sensitive-shaped fields are redacted; a deprecation warning + is emitted naming the field.""" + with caplog.at_level(logging.WARNING, logger="esphome.__main__"): + out = _redact_with_legacy_fallback("password: hunter2\n") + assert "password: \\033[8mhunter2\\033[28m" in out + assert any( + "password" in rec.message and "cv.sensitive" in rec.message + for rec in caplog.records + ) + + +def test_redact_with_legacy_fallback__skips_already_wrapped( + caplog: pytest.LogCaptureFixture, +) -> None: + """Values already wrapped by the SensitiveStr representer don't trigger + the heuristic or the warning.""" + wrapped = "password: \\033[8mhunter2\\033[28m\n" + with caplog.at_level(logging.WARNING, logger="esphome.__main__"): + out = _redact_with_legacy_fallback(wrapped) + assert out == wrapped + assert not any("legacy substring" in rec.message for rec in caplog.records) + + +def test_redact_with_legacy_fallback__captures_full_field_name( + caplog: pytest.LogCaptureFixture, +) -> None: + """The warning names the actual field, not just the matched fragment.""" + with caplog.at_level(logging.WARNING, logger="esphome.__main__"): + _redact_with_legacy_fallback("encryption_key: abc\n") + assert any("encryption_key" in rec.message for rec in caplog.records) + + +def test_redact_with_legacy_fallback__deduplicates_warnings( + caplog: pytest.LogCaptureFixture, +) -> None: + """One warning per unique field name even if it appears many times.""" + text = "password: a\npassword: b\npassword: c\n" + with caplog.at_level(logging.WARNING, logger="esphome.__main__"): + _redact_with_legacy_fallback(text) + password_warnings = [rec for rec in caplog.records if "'password'" in rec.message] + assert len(password_warnings) == 1 + + +def test_redact_with_legacy_fallback__skips_lambda_values( + caplog: pytest.LogCaptureFixture, +) -> None: + """``!lambda`` first line is structural, body is unreachable by a + single-line regex anyway, and tagged fields shouldn't trigger a warning.""" + text = ' ssid: !lambda |-\n return "x";\n' + with caplog.at_level(logging.WARNING, logger="esphome.__main__"): + out = _redact_with_legacy_fallback(text) + assert out == text + assert not any("legacy substring" in rec.message for rec in caplog.records) + + +def test_redact_with_legacy_fallback__skips_secret_references( + caplog: pytest.LogCaptureFixture, +) -> None: + """``!secret name`` is the dumper's user-friendly representation; the + name isn't the secret, so wrapping it would clobber the round-trip.""" + text = " password: !secret wifi_password\n" + with caplog.at_level(logging.WARNING, logger="esphome.__main__"): + out = _redact_with_legacy_fallback(text) + assert out == text + assert not any("legacy substring" in rec.message for rec in caplog.records) + + +def test_redact_with_legacy_fallback__does_not_match_fragment_in_middle( + caplog: pytest.LogCaptureFixture, +) -> None: + """Fragment must end the field name; embedded matches like + ``key_value_pair`` are unrelated to a sensitive key and must not be + redacted (matching the prior regex's scope).""" + with caplog.at_level(logging.WARNING, logger="esphome.__main__"): + out = _redact_with_legacy_fallback("key_value_pair: abc\n") + assert "\\033[8m" not in out + assert not any("legacy substring" in rec.message for rec in caplog.records) + + +def test_redact_with_legacy_fallback__does_not_match_fragment_as_suffix( + caplog: pytest.LogCaptureFixture, +) -> None: + """Fragment must start the name or follow ``_``; ``monkey:`` shouldn't + fire a 'legacy heuristic' warning because there's no sensitive field + here — the user has nothing to migrate.""" + with caplog.at_level(logging.WARNING, logger="esphome.__main__"): + out = _redact_with_legacy_fallback("monkey: 1234\n") + assert "\\033[8m" not in out + assert not any("legacy substring" in rec.message for rec in caplog.records) + + +def test_command_config__invokes_legacy_fallback_when_redacting( + tmp_path: Path, capfd: CaptureFixture[str] +) -> None: + """``command_config`` runs the legacy fallback on the dumped output when + ``--show-secrets`` is off. Cover the wiring (not just the helper). + """ + setup_core(tmp_path=tmp_path, config={"esphome": {"name": "test"}}) + args = MockArgs() + args.show_secrets = False + + result = command_config(args, {"wifi": {"password": "hunter2"}}) + + assert result == 0 + output = capfd.readouterr().out + assert "\\033[8mhunter2\\033[28m" in output + + +def test_command_config__show_secrets_skips_redaction( + tmp_path: Path, capfd: CaptureFixture[str] +) -> None: + """With ``--show-secrets`` the helper isn't invoked and the value + renders raw. + """ + setup_core(tmp_path=tmp_path, config={"esphome": {"name": "test"}}) + args = MockArgs() + args.show_secrets = True + + result = command_config(args, {"wifi": {"password": "hunter2"}}) + + assert result == 0 + output = capfd.readouterr().out + assert "hunter2" in output + assert "\\033[8m" not in output + + def test_choose_upload_log_host_with_string_default() -> None: """Test with a single string default device.""" setup_core() @@ -5110,11 +5241,11 @@ class MockSerial: self.timeout = 0.1 self._is_open = False - def __enter__(self) -> MockSerial: + def __enter__(self) -> Self: self._is_open = True return self - def __exit__(self, *args: Any) -> None: + def __exit__(self, *args: object) -> None: self._is_open = False @property diff --git a/tests/unit_tests/test_storage_json.py b/tests/unit_tests/test_storage_json.py index ea37492cf4..b3f8a05605 100644 --- a/tests/unit_tests/test_storage_json.py +++ b/tests/unit_tests/test_storage_json.py @@ -576,8 +576,8 @@ def test_esphome_storage_json_last_update_check_property() -> None: assert result.hour == 10 assert result.minute == 30 - # Test setter - new_date = datetime(2024, 2, 20, 15, 45, 30) + # Test setter — naive datetime matches the storage round-trip format. + new_date = datetime(2024, 2, 20, 15, 45, 30) # noqa: DTZ001 storage.last_update_check = new_date assert storage.last_update_check_str == "2024-02-20T15:45:30" diff --git a/tests/unit_tests/test_substitutions.py b/tests/unit_tests/test_substitutions.py index 4783112578..b5816f742e 100644 --- a/tests/unit_tests/test_substitutions.py +++ b/tests/unit_tests/test_substitutions.py @@ -1,4 +1,3 @@ -import glob import logging from pathlib import Path from typing import Any @@ -106,7 +105,7 @@ REMOTES = { # Collect all input YAML files for test_substitutions_fixtures parametrized tests: HERE = Path(__file__).parent BASE_DIR = HERE / "fixtures" / "substitutions" -SOURCES = sorted(glob.glob(str(BASE_DIR / "*.input.yaml"))) +SOURCES = sorted(str(p) for p in BASE_DIR.glob("*.input.yaml")) assert SOURCES, f"test_substitutions_fixtures: No input YAML files found in {BASE_DIR}" @@ -838,3 +837,86 @@ def test_include_vars_applied_to_lambda_value(tmp_path: Path) -> None: assert isinstance(result["value"], Lambda) assert result["value"].value == 'return "bar";' + + +@patch("esphome.git.resolve_symlink_stub") +@patch("esphome.git.clone_or_update") +def test_remote_package_symlink_stub_is_followed( + mock_clone_or_update: MagicMock, + mock_resolve_symlink_stub: MagicMock, + tmp_path: Path, +) -> None: + """When a package YAML is a scalar (symlink stub) and resolve_symlink_stub + returns a target, the loader follows the target and uses its content.""" + CORE.config_path = tmp_path / "test.yaml" + + repo_dir = tmp_path / "repo" + repo_dir.mkdir() + (repo_dir / "static").mkdir() + + # Stub file: content is the target path string (simulating Windows behavior). + stub = repo_dir / "file1.yaml" + stub.write_text("static/file1.yaml") + + # Real target with valid YAML mapping. + target = repo_dir / "static" / "file1.yaml" + target.write_text("substitutions:\n hello: world\n") + + mock_clone_or_update.return_value = (repo_dir, None) + mock_resolve_symlink_stub.return_value = target + + config: dict[str, Any] = { + "packages": { + "test_package": { + "url": "https://github.com/esphome/repo1", + "ref": "main", + "files": ["file1.yaml"], + } + } + } + + # Must succeed (does not raise the helpful cv.Invalid) because the stub + # was followed and a valid mapping was loaded from the target. + do_packages_pass(config) + assert mock_resolve_symlink_stub.called + + +@patch("esphome.git.clone_or_update") +def test_remote_package_scalar_yaml_raises_helpful_error( + mock_clone_or_update: MagicMock, tmp_path: Path +) -> None: + """A remote package YAML that is a top-level scalar (e.g. an unmaterialized + git symlink on Windows) raises a clear cv.Invalid, not AttributeError. + + Regression test for the case where a repo containing a YAML symlink, + checked out on Windows without symlink privilege, lands as a short text + file containing the symlink target path. PyYAML parses that as a bare + string scalar; the package loader must reject it with a human-readable + error instead of dying inside ``.get()``. + """ + CORE.config_path = tmp_path / "test.yaml" + + repo_dir = tmp_path / "repo" + repo_dir.mkdir() + # Simulate the broken-symlink state: a YAML file whose entire content is + # the symlink target string. PyYAML parses this as a top-level scalar. + (repo_dir / "file1.yaml").write_text("static/file1.yaml") + + mock_clone_or_update.return_value = (repo_dir, None) + + config: dict[str, Any] = { + "packages": { + "test_package": { + "url": "https://github.com/esphome/repo1", + "ref": "main", + "files": ["file1.yaml"], + } + } + } + + with pytest.raises(cv.Invalid) as exc_info: + do_packages_pass(config) + + msg = str(exc_info.value) + assert "mapping at the top level" in msg + assert "file1.yaml" in msg diff --git a/tests/unit_tests/test_writer.py b/tests/unit_tests/test_writer.py index afc5c26edf..df463749ab 100644 --- a/tests/unit_tests/test_writer.py +++ b/tests/unit_tests/test_writer.py @@ -1361,7 +1361,7 @@ def test_clean_build_handles_readonly_files( # Create a read-only file (simulating git pack files on Windows) readonly_file = git_dir / "pack-abc123.pack" readonly_file.write_text("pack data") - os.chmod(readonly_file, stat.S_IRUSR) # Read-only + readonly_file.chmod(stat.S_IRUSR) # Read-only # Setup mocks mock_core.relative_pioenvs_path.return_value = pioenvs_dir @@ -1396,7 +1396,7 @@ def test_clean_all_handles_readonly_files( subdir.mkdir() readonly_file = subdir / "readonly.txt" readonly_file.write_text("content") - os.chmod(readonly_file, stat.S_IRUSR) # Read-only + readonly_file.chmod(stat.S_IRUSR) # Read-only # Verify file is read-only assert not os.access(readonly_file, os.W_OK) @@ -1425,7 +1425,7 @@ def test_clean_build_reraises_for_other_errors( test_file.write_text("content") # Make subdir read-only so files inside can't be deleted - os.chmod(subdir, stat.S_IRUSR | stat.S_IXUSR) + subdir.chmod(stat.S_IRUSR | stat.S_IXUSR) # Setup mocks mock_core.relative_pioenvs_path.return_value = pioenvs_dir @@ -1443,7 +1443,7 @@ def test_clean_build_reraises_for_other_errors( clean_build() finally: # Cleanup - restore write permission so tmp_path cleanup works - os.chmod(subdir, stat.S_IRWXU) + subdir.chmod(stat.S_IRWXU) # Tests for get_build_info() diff --git a/tests/unit_tests/test_yaml_util.py b/tests/unit_tests/test_yaml_util.py index de70a5307d..6be090b869 100644 --- a/tests/unit_tests/test_yaml_util.py +++ b/tests/unit_tests/test_yaml_util.py @@ -15,6 +15,7 @@ from esphome.yaml_util import ( DiscoveredYamlFiles, ESPHomeDataBase, ESPLiteralValue, + SensitiveStr, discover_user_yaml_files, force_load_include_files, format_path, @@ -907,7 +908,7 @@ def test_format_path_current_obj_without_location_falls_back_to_key(): """An ESPHomeDataBase current_obj with no esp_range falls back to the key's location.""" class _NoRange(ESPHomeDataBase, str): - pass + __slots__ = () obj = _NoRange.__new__(_NoRange, "value") str.__init__(obj) @@ -1340,3 +1341,57 @@ def test_frontmatter_included_file_stored(tmp_path: Path) -> None: assert main.resolve() not in core.CORE.frontmatter # Included file's frontmatter is captured assert core.CORE.frontmatter[inc.resolve()]["child_meta"] == "hello" + + +def test_sensitive_str__is_a_str_subclass() -> None: + value = SensitiveStr("hunter2") + assert isinstance(value, str) + assert value == "hunter2" + + +def test_dump__redacts_sensitive_str_by_default() -> None: + out = yaml_util.dump({"password": SensitiveStr("hunter2")}) + assert "\\033[8mhunter2\\033[28m" in out + assert "hunter2" not in out.replace( + "\\033[8mhunter2\\033[28m", "" + ) # the raw value is only present inside the wrap + + +def test_dump__show_secrets_emits_sensitive_str_raw() -> None: + out = yaml_util.dump({"password": SensitiveStr("hunter2")}, show_secrets=True) + assert "hunter2" in out + assert "\\033[8m" not in out + assert "\\033[28m" not in out + + +def test_dump__plain_str_is_not_redacted() -> None: + out = yaml_util.dump({"hostname": "myserver"}) + assert "myserver" in out + assert "\\033[8m" not in out + + +def test_dump__secret_reference_wins_over_redaction() -> None: + # If the value also has an entry in _SECRET_VALUES (i.e., it was loaded + # via !secret), the dump should render it as !secret , not as a + # redacted scalar. SensitiveStr layered on top must not change that. + value = SensitiveStr("hunter2") + yaml_util._SECRET_VALUES[str(value)] = "my_secret_name" + try: + out = yaml_util.dump({"password": value}) + assert "!secret" in out + assert "my_secret_name" in out + assert "\\033[8m" not in out + finally: + yaml_util._SECRET_VALUES.clear() + + +def test_dump__redaction_flag_does_not_leak_between_calls() -> None: + # Per-call _Dumper subclass means show_secrets in one call doesn't + # affect another. Run them in both orders to catch any leakage. + redacted = yaml_util.dump({"password": SensitiveStr("hunter2")}) + raw = yaml_util.dump({"password": SensitiveStr("hunter2")}, show_secrets=True) + redacted_again = yaml_util.dump({"password": SensitiveStr("hunter2")}) + + assert "\\033[8m" in redacted + assert "\\033[8m" not in raw + assert "\\033[8m" in redacted_again