From 5c5ea8824edebb74e80a3ff9306add66b76b4b95 Mon Sep 17 00:00:00 2001 From: Kevin Ahrendt Date: Thu, 5 Mar 2026 13:51:08 -0600 Subject: [PATCH 1/7] [audio_file] New component for embedding files into firmware (#14434) Co-authored-by: Claude Opus 4.6 Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> Co-authored-by: pre-commit-ci-lite[bot] <117423508+pre-commit-ci-lite[bot]@users.noreply.github.com> Co-authored-by: J. Nick Koston --- CODEOWNERS | 1 + esphome/components/audio_file/__init__.py | 255 ++++++++++++++++++ esphome/components/audio_file/audio_file.h | 28 ++ esphome/core/defines.h | 1 + script/ci-custom.py | 1 + tests/components/audio_file/common.yaml | 5 + .../components/audio_file/test.esp32-idf.yaml | 1 + tests/components/audio_file/test.wav | Bin 0 -> 46 bytes 8 files changed, 292 insertions(+) create mode 100644 esphome/components/audio_file/__init__.py create mode 100644 esphome/components/audio_file/audio_file.h create mode 100644 tests/components/audio_file/common.yaml create mode 100644 tests/components/audio_file/test.esp32-idf.yaml create mode 100644 tests/components/audio_file/test.wav diff --git a/CODEOWNERS b/CODEOWNERS index b22f85b71d7..7c37b20e099 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -54,6 +54,7 @@ esphome/components/atm90e32/* @circuitsetup @descipher esphome/components/audio/* @kahrendt esphome/components/audio_adc/* @kbx81 esphome/components/audio_dac/* @kbx81 +esphome/components/audio_file/* @kahrendt esphome/components/axs15231/* @clydebarrow esphome/components/b_parasite/* @rbaron esphome/components/ballu/* @bazuchan diff --git a/esphome/components/audio_file/__init__.py b/esphome/components/audio_file/__init__.py new file mode 100644 index 00000000000..3ed6c1cd928 --- /dev/null +++ b/esphome/components/audio_file/__init__.py @@ -0,0 +1,255 @@ +from dataclasses import dataclass, field +import hashlib +import logging +from pathlib import Path + +import puremagic + +from esphome import external_files +import esphome.codegen as cg +from esphome.components import audio +import esphome.config_validation as cv +from esphome.const import ( + CONF_FILE, + CONF_ID, + CONF_PATH, + CONF_RAW_DATA_ID, + CONF_TYPE, + CONF_URL, +) +from esphome.core import CORE, ID, HexInt +from esphome.cpp_generator import MockObj +from esphome.external_files import download_content +from esphome.types import ConfigType + +_LOGGER = logging.getLogger(__name__) + +CODEOWNERS = ["@kahrendt"] + +AUTO_LOAD = ["audio"] + +DOMAIN = "audio_file" + +audio_file_ns = cg.esphome_ns.namespace("audio_file") + +TYPE_LOCAL = "local" +TYPE_WEB = "web" + + +@dataclass +class AudioFileData: + file_ids: dict[str, ID] = field(default_factory=dict) + file_cache: dict[str, tuple[bytes, MockObj]] = field(default_factory=dict) + + +def _get_data() -> AudioFileData: + if DOMAIN not in CORE.data: + CORE.data[DOMAIN] = AudioFileData() + return CORE.data[DOMAIN] + + +def get_audio_file_ids() -> dict[str, ID]: + """Get all registered audio file IDs for cross-component access.""" + return _get_data().file_ids + + +def _compute_local_file_path(value: ConfigType) -> Path: + url = value[CONF_URL] + h = hashlib.new("sha256") + h.update(url.encode()) + key = h.hexdigest()[:8] + base_dir = external_files.compute_local_file_dir(DOMAIN) + _LOGGER.debug("_compute_local_file_path: base_dir=%s", base_dir / key) + return base_dir / key + + +def _download_web_file(value: ConfigType) -> ConfigType: + url = value[CONF_URL] + path = _compute_local_file_path(value) + + download_content(url, path) + _LOGGER.debug("download_web_file: path=%s", path) + return value + + +def _file_schema(value: ConfigType | str) -> ConfigType: + if isinstance(value, str): + return _validate_file_shorthand(value) + return TYPED_FILE_SCHEMA(value) + + +def _validate_file_shorthand(value: str) -> ConfigType: + value = cv.string_strict(value) + if value.startswith("http://") or value.startswith("https://"): + return _file_schema( + { + CONF_TYPE: TYPE_WEB, + CONF_URL: value, + } + ) + return _file_schema( + { + CONF_TYPE: TYPE_LOCAL, + CONF_PATH: value, + } + ) + + +def read_audio_file_and_type(file_config: ConfigType) -> tuple[bytes, MockObj]: + """Read an audio file and determine its type. Used by this component and media_source platform.""" + conf_file = file_config[CONF_FILE] + file_source = conf_file[CONF_TYPE] + if file_source == TYPE_LOCAL: + path = CORE.relative_config_path(conf_file[CONF_PATH]) + elif file_source == TYPE_WEB: + path = _compute_local_file_path(conf_file) + else: + raise cv.Invalid("Unsupported file source") + + with open(path, "rb") as f: + data = f.read() + + try: + file_type: str = puremagic.from_string(data) + file_type = file_type.removeprefix(".") + except puremagic.PureError as e: + raise cv.Invalid( + f"Unable to determine audio file type of '{path}'. " + f"Try re-encoding the file into a supported format. Details: {e}" + ) + + media_file_type = audio.AUDIO_FILE_TYPE_ENUM["NONE"] + if file_type == "wav": + media_file_type = audio.AUDIO_FILE_TYPE_ENUM["WAV"] + elif file_type in ("mp3", "mpeg", "mpga"): + media_file_type = audio.AUDIO_FILE_TYPE_ENUM["MP3"] + elif file_type == "flac": + media_file_type = audio.AUDIO_FILE_TYPE_ENUM["FLAC"] + elif ( + file_type == "ogg" + and len(data) >= 36 + and data.startswith(b"OggS") + and data[28:36] == b"OpusHead" + ): + media_file_type = audio.AUDIO_FILE_TYPE_ENUM["OPUS"] + + return data, media_file_type + + +LOCAL_SCHEMA = cv.Schema( + { + cv.Required(CONF_PATH): cv.file_, + } +) + +WEB_SCHEMA = cv.All( + { + cv.Required(CONF_URL): cv.url, + }, + _download_web_file, +) + + +TYPED_FILE_SCHEMA = cv.typed_schema( + { + TYPE_LOCAL: LOCAL_SCHEMA, + TYPE_WEB: WEB_SCHEMA, + }, +) + + +MEDIA_FILE_TYPE_SCHEMA = cv.Schema( + { + cv.Required(CONF_ID): cv.declare_id(audio.AudioFile), + cv.Required(CONF_FILE): _file_schema, + cv.GenerateID(CONF_RAW_DATA_ID): cv.declare_id(cg.uint8), + } +) + + +MAX_FILE_SIZE = 5 * 1024 * 1024 # 5 MB + + +def _validate_supported_local_file(config: list[ConfigType]) -> list[ConfigType]: + for file_config in config: + data, media_file_type = read_audio_file_and_type(file_config) + + if len(data) > MAX_FILE_SIZE: + file_info = file_config.get(CONF_FILE, {}) + source = ( + file_info.get(CONF_PATH) or file_info.get(CONF_URL) or "unknown source" + ) + raise cv.Invalid( + f"Audio file {source!r} is too large ({len(data)} bytes, max {MAX_FILE_SIZE} bytes)" + ) + + if str(media_file_type) == str(audio.AUDIO_FILE_TYPE_ENUM["NONE"]): + file_info = file_config.get(CONF_FILE, {}) + source = ( + file_info.get(CONF_PATH) or file_info.get(CONF_URL) or "unknown source" + ) + raise cv.Invalid( + f"Unsupported media file from {source!r} (detected type: {media_file_type})" + ) + + # Cache the file data so to_code() doesn't need to re-read it + _get_data().file_cache[str(file_config[CONF_ID])] = (data, media_file_type) + + media_file_type_str = str(media_file_type) + if media_file_type_str == str(audio.AUDIO_FILE_TYPE_ENUM["FLAC"]): + audio.request_flac_support() + elif media_file_type_str == str(audio.AUDIO_FILE_TYPE_ENUM["MP3"]): + audio.request_mp3_support() + elif media_file_type_str == str(audio.AUDIO_FILE_TYPE_ENUM["OPUS"]): + audio.request_opus_support() + + return config + + +CONFIG_SCHEMA = cv.All( + cv.only_on_esp32, + cv.ensure_list(MEDIA_FILE_TYPE_SCHEMA), + _validate_supported_local_file, +) + + +async def to_code(config: list[ConfigType]) -> None: + cache = _get_data().file_cache + + for file_config in config: + file_id = str(file_config[CONF_ID]) + data, media_file_type = cache[file_id] + + rhs = [HexInt(x) for x in data] + prog_arr = cg.progmem_array(file_config[CONF_RAW_DATA_ID], rhs) + + media_files_struct = cg.StructInitializer( + audio.AudioFile, + ( + "data", + prog_arr, + ), + ( + "length", + len(rhs), + ), + ( + "file_type", + media_file_type, + ), + ) + + cg.new_Pvariable( + file_config[CONF_ID], + media_files_struct, + ) + + # Store file ID for cross-component access + _get_data().file_ids[file_id] = file_config[CONF_ID] + + # Register all files in the shared C++ registry + cg.add_define("AUDIO_FILE_MAX_FILES", len(config)) + for file_config in config: + file_id = str(file_config[CONF_ID]) + file_var = await cg.get_variable(file_config[CONF_ID]) + cg.add(audio_file_ns.add_named_audio_file(file_var, file_id)) diff --git a/esphome/components/audio_file/audio_file.h b/esphome/components/audio_file/audio_file.h new file mode 100644 index 00000000000..537e19fb3ca --- /dev/null +++ b/esphome/components/audio_file/audio_file.h @@ -0,0 +1,28 @@ +#pragma once + +#include "esphome/core/defines.h" + +#ifdef AUDIO_FILE_MAX_FILES + +#include "esphome/components/audio/audio.h" +#include "esphome/core/helpers.h" + +namespace esphome::audio_file { + +struct NamedAudioFile { + audio::AudioFile *file; + const char *file_id; +}; + +inline StaticVector + named_audio_files; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) + +inline void add_named_audio_file(audio::AudioFile *file, const char *file_id) { + named_audio_files.push_back({file, file_id}); +} + +inline const StaticVector &get_named_audio_files() { return named_audio_files; } + +} // namespace esphome::audio_file + +#endif // AUDIO_FILE_MAX_FILES diff --git a/esphome/core/defines.h b/esphome/core/defines.h index 07afefd91aa..1a6d9b3a803 100644 --- a/esphome/core/defines.h +++ b/esphome/core/defines.h @@ -137,6 +137,7 @@ // Feature flags which do not work for zephyr #ifndef USE_ZEPHYR +#define AUDIO_FILE_MAX_FILES 4 #define USE_AUDIO_DAC #define USE_AUDIO_FLAC_SUPPORT #define USE_AUDIO_MP3_SUPPORT diff --git a/script/ci-custom.py b/script/ci-custom.py index b60d7d77401..8e1652b505d 100755 --- a/script/ci-custom.py +++ b/script/ci-custom.py @@ -72,6 +72,7 @@ ignore_types = ( ".gif", ".webp", ".bin", + ".wav", ) LINT_FILE_CHECKS = [] diff --git a/tests/components/audio_file/common.yaml b/tests/components/audio_file/common.yaml new file mode 100644 index 00000000000..94042080946 --- /dev/null +++ b/tests/components/audio_file/common.yaml @@ -0,0 +1,5 @@ +audio_file: + - id: test_audio + file: + type: local + path: $component_dir/test.wav diff --git a/tests/components/audio_file/test.esp32-idf.yaml b/tests/components/audio_file/test.esp32-idf.yaml new file mode 100644 index 00000000000..dade44d145b --- /dev/null +++ b/tests/components/audio_file/test.esp32-idf.yaml @@ -0,0 +1 @@ +<<: !include common.yaml diff --git a/tests/components/audio_file/test.wav b/tests/components/audio_file/test.wav new file mode 100644 index 0000000000000000000000000000000000000000..f9d07ef2238eb2fcb355055466d3789ee1a1fe0b GIT binary patch literal 46 ycmWIYbaPW Date: Thu, 5 Mar 2026 14:51:32 -0500 Subject: [PATCH 2/7] [wled][lcd_base][touchscreen][ee895] Fix off-by-one, buffer overrun, empty deref, and uninitialized pointers (#14513) Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- esphome/components/ee895/ee895.h | 6 +++--- esphome/components/lcd_base/lcd_display.cpp | 2 +- esphome/components/touchscreen/touchscreen.h | 7 ++++++- esphome/components/wled/wled_light_effect.cpp | 2 +- 4 files changed, 11 insertions(+), 6 deletions(-) diff --git a/esphome/components/ee895/ee895.h b/esphome/components/ee895/ee895.h index 259b7c524b5..ff1085e05d8 100644 --- a/esphome/components/ee895/ee895.h +++ b/esphome/components/ee895/ee895.h @@ -22,9 +22,9 @@ class EE895Component : public PollingComponent, public i2c::I2CDevice { void write_command_(uint16_t addr, uint16_t reg_cnt); float read_float_(); uint16_t calc_crc16_(const uint8_t buf[], uint8_t len); - sensor::Sensor *co2_sensor_; - sensor::Sensor *temperature_sensor_; - sensor::Sensor *pressure_sensor_; + sensor::Sensor *co2_sensor_{nullptr}; + sensor::Sensor *temperature_sensor_{nullptr}; + sensor::Sensor *pressure_sensor_{nullptr}; enum ErrorCode { NONE = 0, COMMUNICATION_FAILED, CRC_CHECK_FAILED } error_code_{NONE}; }; diff --git a/esphome/components/lcd_base/lcd_display.cpp b/esphome/components/lcd_base/lcd_display.cpp index cd08a739eb6..1f0ba482d74 100644 --- a/esphome/components/lcd_base/lcd_display.cpp +++ b/esphome/components/lcd_base/lcd_display.cpp @@ -99,7 +99,7 @@ void HOT LCDDisplay::display() { this->send(this->buffer_[this->columns_ * 2 + i], true); } - if (this->rows_ >= 1) { + if (this->rows_ >= 2) { this->command_(LCD_DISPLAY_COMMAND_SET_DDRAM_ADDR | 0x40); for (uint8_t i = 0; i < this->columns_; i++) diff --git a/esphome/components/touchscreen/touchscreen.h b/esphome/components/touchscreen/touchscreen.h index 8016323d493..7451c207ec3 100644 --- a/esphome/components/touchscreen/touchscreen.h +++ b/esphome/components/touchscreen/touchscreen.h @@ -65,7 +65,12 @@ class Touchscreen : public PollingComponent { void register_listener(TouchListener *listener) { this->touch_listeners_.push_back(listener); } - optional get_touch() { return this->touches_.begin()->second; } + optional get_touch() { + if (this->touches_.empty()) { + return {}; + } + return this->touches_.begin()->second; + } TouchPoints_t get_touches() { TouchPoints_t touches; diff --git a/esphome/components/wled/wled_light_effect.cpp b/esphome/components/wled/wled_light_effect.cpp index 87bae5b1da4..db2708d6d02 100644 --- a/esphome/components/wled/wled_light_effect.cpp +++ b/esphome/components/wled/wled_light_effect.cpp @@ -161,7 +161,7 @@ bool WLEDLightEffect::parse_notifier_frame_(light::AddressableLight &it, const u // https://kno.wled.ge/interfaces/udp-notifier/ // https://github.com/Aircoookie/WLED/blob/main/wled00/udp.cpp - if (size < 34) { + if (size <= 34) { return false; } From 99a805cba675ab2399d432dd39d142a0852fafc8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 5 Mar 2026 09:52:10 -1000 Subject: [PATCH 3/7] Bump the docker-actions group across 1 directory with 2 updates (#14520) Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/ci-docker.yml | 2 +- .github/workflows/release.yml | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci-docker.yml b/.github/workflows/ci-docker.yml index a83bcae0b06..4009ac1e174 100644 --- a/.github/workflows/ci-docker.yml +++ b/.github/workflows/ci-docker.yml @@ -49,7 +49,7 @@ jobs: with: python-version: "3.11" - name: Set up Docker Buildx - uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0 + uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0 - name: Set TAG run: | diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 17a2616dffc..8f68e9c873f 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -99,15 +99,15 @@ jobs: python-version: "3.11" - name: Set up Docker Buildx - uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0 + uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0 - name: Log in to docker hub - uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0 + uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v4.0.0 with: username: ${{ secrets.DOCKER_USER }} password: ${{ secrets.DOCKER_PASSWORD }} - name: Log in to the GitHub container registry - uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0 + uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v4.0.0 with: registry: ghcr.io username: ${{ github.actor }} @@ -178,17 +178,17 @@ jobs: merge-multiple: true - name: Set up Docker Buildx - uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0 + uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0 - name: Log in to docker hub if: matrix.registry == 'dockerhub' - uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0 + uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v4.0.0 with: username: ${{ secrets.DOCKER_USER }} password: ${{ secrets.DOCKER_PASSWORD }} - name: Log in to the GitHub container registry if: matrix.registry == 'ghcr' - uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0 + uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v4.0.0 with: registry: ghcr.io username: ${{ github.actor }} From 291679126f532a6b036c10984c97c47db0be9cd3 Mon Sep 17 00:00:00 2001 From: Jonathan Swoboda <154711427+swoboda1337@users.noreply.github.com> Date: Thu, 5 Mar 2026 14:55:54 -0500 Subject: [PATCH 4/7] [nfc] Fix off-by-one in NDEF message parsing (#14485) Co-authored-by: Claude Opus 4.6 --- esphome/components/nfc/ndef_message.cpp | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/esphome/components/nfc/ndef_message.cpp b/esphome/components/nfc/ndef_message.cpp index e7304445c55..35028555c55 100644 --- a/esphome/components/nfc/ndef_message.cpp +++ b/esphome/components/nfc/ndef_message.cpp @@ -8,8 +8,14 @@ static const char *const TAG = "nfc.ndef_message"; NdefMessage::NdefMessage(std::vector &data) { ESP_LOGV(TAG, "Building NdefMessage with %zu bytes", data.size()); - uint8_t index = 0; - while (index <= data.size()) { + size_t index = 0; + while (index < data.size()) { + // Minimum record: TNF byte + type length byte + payload length (1 or 4 bytes) + if (index + 2 >= data.size()) { + ESP_LOGE(TAG, "Truncated record header; aborting"); + break; + } + uint8_t tnf_byte = data[index++]; bool me = tnf_byte & 0x40; // Message End bit (is set if this is the last record of the message) bool sr = tnf_byte & 0x10; // Short record bit (is set if payload size is less or equal to 255 bytes) @@ -23,6 +29,10 @@ NdefMessage::NdefMessage(std::vector &data) { if (sr) { payload_length = data[index++]; } else { + if (index + 4 > data.size()) { + ESP_LOGE(TAG, "Truncated payload length; aborting"); + break; + } payload_length = (static_cast(data[index]) << 24) | (static_cast(data[index + 1]) << 16) | (static_cast(data[index + 2]) << 8) | static_cast(data[index + 3]); index += 4; @@ -30,6 +40,10 @@ NdefMessage::NdefMessage(std::vector &data) { uint8_t id_length = 0; if (il) { + if (index >= data.size()) { + ESP_LOGE(TAG, "Truncated ID length; aborting"); + break; + } id_length = data[index++]; } From e25d740968f656b8e50dedb793fbc5370573f3ef Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Thu, 5 Mar 2026 09:58:58 -1000 Subject: [PATCH 5/7] [wifi] Cache is_connected() for cheap inline access (#14463) Co-authored-by: Jonathan Swoboda <154711427+swoboda1337@users.noreply.github.com> Co-authored-by: pre-commit-ci-lite[bot] <117423508+pre-commit-ci-lite[bot]@users.noreply.github.com> --- esphome/components/wifi/wifi_component.cpp | 8 +++++--- esphome/components/wifi/wifi_component.h | 5 ++++- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/esphome/components/wifi/wifi_component.cpp b/esphome/components/wifi/wifi_component.cpp index 852ff922f1f..8b60810d28a 100644 --- a/esphome/components/wifi/wifi_component.cpp +++ b/esphome/components/wifi/wifi_component.cpp @@ -725,6 +725,7 @@ void WiFiComponent::restart_adapter() { void WiFiComponent::loop() { this->wifi_loop_(); const uint32_t now = App.get_loop_component_start_time(); + this->update_connected_state_(); if (this->has_sta()) { #if defined(USE_WIFI_CONNECT_TRIGGER) || defined(USE_WIFI_DISCONNECT_TRIGGER) @@ -776,7 +777,7 @@ void WiFiComponent::loop() { } case WIFI_COMPONENT_STATE_STA_CONNECTED: { - if (!this->is_connected()) { + if (!this->is_connected_()) { ESP_LOGW(TAG, "Connection lost; reconnecting"); this->state_ = WIFI_COMPONENT_STATE_STA_CONNECTING; this->retry_connect(); @@ -2118,15 +2119,16 @@ bool WiFiComponent::can_proceed() { if (!this->has_sta() || this->state_ == WIFI_COMPONENT_STATE_DISABLED || this->ap_setup_) { return true; } - return this->is_connected(); + return this->is_connected_(); } #endif void WiFiComponent::set_reboot_timeout(uint32_t reboot_timeout) { this->reboot_timeout_ = reboot_timeout; } -bool WiFiComponent::is_connected() const { +bool WiFiComponent::is_connected_() const { return this->state_ == WIFI_COMPONENT_STATE_STA_CONNECTED && this->wifi_sta_connect_status_() == WiFiSTAConnectStatus::CONNECTED && !this->error_from_callback_; } +void WiFiComponent::update_connected_state_() { this->connected_ = this->is_connected_(); } void WiFiComponent::set_power_save_mode(WiFiPowerSaveMode power_save) { this->power_save_ = power_save; #if defined(USE_ESP32) && defined(USE_WIFI_RUNTIME_POWER_SAVE) diff --git a/esphome/components/wifi/wifi_component.h b/esphome/components/wifi/wifi_component.h index a6f03a08d9d..f340b708c90 100644 --- a/esphome/components/wifi/wifi_component.h +++ b/esphome/components/wifi/wifi_component.h @@ -443,7 +443,7 @@ class WiFiComponent : public Component { void set_reboot_timeout(uint32_t reboot_timeout); - bool is_connected() const; + bool is_connected() const { return this->connected_; } void set_power_save_mode(WiFiPowerSaveMode power_save); void set_min_auth_mode(WifiMinAuthMode min_auth_mode) { min_auth_mode_ = min_auth_mode; } @@ -678,6 +678,8 @@ class WiFiComponent : public Component { bool wifi_sta_connect_(const WiFiAP &ap); void wifi_pre_setup_(); WiFiSTAConnectStatus wifi_sta_connect_status_() const; + bool is_connected_() const; + void update_connected_state_(); bool wifi_scan_start_(bool passive); #ifdef USE_WIFI_AP @@ -854,6 +856,7 @@ class WiFiComponent : public Component { bool has_completed_scan_after_captive_portal_start_{ false}; // Tracks if we've completed a scan after captive portal started bool skip_cooldown_next_cycle_{false}; + bool connected_{false}; bool post_connect_roaming_{true}; // Enabled by default #if defined(USE_ESP32) && defined(USE_WIFI_RUNTIME_POWER_SAVE) bool is_high_performance_mode_{false}; From d11e7cab464a007bccf37cf0cf3da4d8b1f71861 Mon Sep 17 00:00:00 2001 From: Jonathan Swoboda <154711427+swoboda1337@users.noreply.github.com> Date: Thu, 5 Mar 2026 15:18:54 -0500 Subject: [PATCH 6/7] [xiaomi_ble][pvvx_mithermometer][atc_mithermometer] Add BLE service data bounds checks (#14514) Co-authored-by: Claude Opus 4.6 --- .../components/atc_mithermometer/atc_mithermometer.cpp | 4 ++++ .../components/pvvx_mithermometer/pvvx_mithermometer.cpp | 4 ++++ esphome/components/xiaomi_ble/xiaomi_ble.cpp | 9 +++++++++ 3 files changed, 17 insertions(+) diff --git a/esphome/components/atc_mithermometer/atc_mithermometer.cpp b/esphome/components/atc_mithermometer/atc_mithermometer.cpp index b4d2929742a..9afd6334f5b 100644 --- a/esphome/components/atc_mithermometer/atc_mithermometer.cpp +++ b/esphome/components/atc_mithermometer/atc_mithermometer.cpp @@ -61,6 +61,10 @@ optional ATCMiThermometer::parse_header_(const esp32_ble_tracker::S } auto raw = service_data.data; + if (raw.size() < 13) { + ESP_LOGVV(TAG, "parse_header_(): service data too short (%zu).", raw.size()); + return {}; + } static uint8_t last_frame_count = 0; if (last_frame_count == raw[12]) { diff --git a/esphome/components/pvvx_mithermometer/pvvx_mithermometer.cpp b/esphome/components/pvvx_mithermometer/pvvx_mithermometer.cpp index 57124479090..239a1e74fe6 100644 --- a/esphome/components/pvvx_mithermometer/pvvx_mithermometer.cpp +++ b/esphome/components/pvvx_mithermometer/pvvx_mithermometer.cpp @@ -61,6 +61,10 @@ optional PVVXMiThermometer::parse_header_(const esp32_ble_tracker:: } auto raw = service_data.data; + if (raw.size() < 14) { + ESP_LOGVV(TAG, "parse_header_(): service data too short (%zu).", raw.size()); + return {}; + } static uint8_t last_frame_count = 0; if (last_frame_count == raw[13]) { diff --git a/esphome/components/xiaomi_ble/xiaomi_ble.cpp b/esphome/components/xiaomi_ble/xiaomi_ble.cpp index 0018d35f1f5..97a660f0e3a 100644 --- a/esphome/components/xiaomi_ble/xiaomi_ble.cpp +++ b/esphome/components/xiaomi_ble/xiaomi_ble.cpp @@ -121,6 +121,11 @@ bool parse_xiaomi_message(const std::vector &message, XiaomiParseResult // Byte 2: length // Byte 3..3+len-1: data point value + if (result.raw_offset < 0 || static_cast(result.raw_offset) >= message.size()) { + ESP_LOGVV(TAG, "parse_xiaomi_message(): raw_offset (%d) exceeds message size (%d)!", result.raw_offset, + message.size()); + return false; + } const uint8_t *payload = message.data() + result.raw_offset; uint8_t payload_length = message.size() - result.raw_offset; uint8_t payload_offset = 0; @@ -165,6 +170,10 @@ optional parse_xiaomi_header(const esp32_ble_tracker::Service } auto raw = service_data.data; + if (raw.size() < 5) { + ESP_LOGVV(TAG, "parse_xiaomi_header(): service data too short (%d).", raw.size()); + return {}; + } result.has_data = raw[0] & 0x40; result.has_capability = raw[0] & 0x20; result.has_encryption = raw[0] & 0x08; From b0be02e16d8b53efee692543c9be2c6da2118da7 Mon Sep 17 00:00:00 2001 From: Bonne Eggleston Date: Thu, 5 Mar 2026 12:54:17 -0800 Subject: [PATCH 7/7] [modbus] Fix timing bugs and better adhere to spec (#8032) Co-authored-by: brambo123 <52667932+brambo123@users.noreply.github.com> Co-authored-by: Keith Burzinski Co-authored-by: J. Nick Koston Co-authored-by: J. Nick Koston Co-authored-by: J. Nick Koston --- .../growatt_solar/growatt_solar.cpp | 2 +- esphome/components/modbus/__init__.py | 5 + esphome/components/modbus/modbus.cpp | 294 ++++++++++++------ esphome/components/modbus/modbus.h | 55 +++- .../components/modbus/modbus_definitions.h | 2 + .../components/modbus_controller/__init__.py | 3 +- .../modbus_controller/modbus_controller.cpp | 2 +- tests/components/modbus/common.yaml | 2 + .../external_components/uart_mock/__init__.py | 4 +- .../uart_mock/uart_mock.cpp | 26 +- .../external_components/uart_mock/uart_mock.h | 5 +- .../fixtures/uart_mock_modbus.yaml | 48 ++- .../fixtures/uart_mock_modbus_timing.yaml | 2 + tests/integration/test_uart_mock_modbus.py | 67 +++- 14 files changed, 396 insertions(+), 121 deletions(-) diff --git a/esphome/components/growatt_solar/growatt_solar.cpp b/esphome/components/growatt_solar/growatt_solar.cpp index 686c1c232e7..29974258723 100644 --- a/esphome/components/growatt_solar/growatt_solar.cpp +++ b/esphome/components/growatt_solar/growatt_solar.cpp @@ -26,7 +26,7 @@ void GrowattSolar::update() { } // The bus might be slow, or there might be other devices, or other components might be talking to our device. - if (this->waiting_for_response()) { + if (!this->ready_for_immediate_send()) { this->waiting_to_update_ = true; return; } diff --git a/esphome/components/modbus/__init__.py b/esphome/components/modbus/__init__.py index 2bd85c6121f..f6e0f98857d 100644 --- a/esphome/components/modbus/__init__.py +++ b/esphome/components/modbus/__init__.py @@ -20,6 +20,7 @@ MULTI_CONF = True CONF_ROLE = "role" CONF_MODBUS_ID = "modbus_id" CONF_SEND_WAIT_TIME = "send_wait_time" +CONF_TURNAROUND_TIME = "turnaround_time" ModbusRole = modbus_ns.enum("ModbusRole") MODBUS_ROLES = { @@ -36,6 +37,9 @@ CONFIG_SCHEMA = ( cv.Optional( CONF_SEND_WAIT_TIME, default="250ms" ): cv.positive_time_period_milliseconds, + cv.Optional( + CONF_TURNAROUND_TIME, default="100ms" + ): cv.positive_time_period_milliseconds, cv.Optional(CONF_DISABLE_CRC, default=False): cv.boolean, } ) @@ -57,6 +61,7 @@ async def to_code(config): cg.add(var.set_flow_control_pin(pin)) cg.add(var.set_send_wait_time(config[CONF_SEND_WAIT_TIME])) + cg.add(var.set_turnaround_time(config[CONF_TURNAROUND_TIME])) cg.add(var.set_disable_crc(config[CONF_DISABLE_CRC])) diff --git a/esphome/components/modbus/modbus.cpp b/esphome/components/modbus/modbus.cpp index d40343db33d..28e26e307e3 100644 --- a/esphome/components/modbus/modbus.cpp +++ b/esphome/components/modbus/modbus.cpp @@ -15,10 +15,69 @@ void Modbus::setup() { if (this->flow_control_pin_ != nullptr) { this->flow_control_pin_->setup(); } -} -void Modbus::loop() { - const uint32_t now = App.get_loop_component_start_time(); + this->frame_delay_ms_ = + std::max(2, // 1750us minimum per spec - rounded up to 2ms. + // 3.5 characters * 11 bits per character * 1000ms/sec / (bits/sec) (Standard modbus frame delay) + (uint16_t) (3.5 * 11 * 1000 / this->parent_->get_baud_rate()) + 1); + + this->long_rx_buffer_delay_ms_ = + (this->parent_->get_rx_full_threshold() * 11 * 1000 / this->parent_->get_baud_rate()) + 1; +} + +void Modbus::loop() { + // First process all available incoming data. + this->receive_and_parse_modbus_bytes_(); + + // If the response frame is finished (including interframe delay) - we timeout. + // The long_rx_buffer_delay accounts for long responses (larger than the UART rx_full_threshold) to avoid timeouts + // when the buffer is filling the back half of the response + const uint16_t timeout = std::max( + (uint16_t) this->frame_delay_ms_, + (uint16_t) (this->rx_buffer_.size() >= this->parent_->get_rx_full_threshold() ? this->long_rx_buffer_delay_ms_ + : 0)); + // We use millis() here and elsewhere instead of App.get_loop_component_start_time() to avoid stale timestamps + // It's critical in all timestamp comparisons that the left timestamp comes before the right one in time + // If we use a cached value in place of millis() and last_modbus_byte_ is updated inside our loop + // then the comparison is backwards (small negative which wraps to large positive) and will cause a false timeout + // So in this component we don't use any cached timestamp values to avoid these annoying bugs + if (millis() - this->last_modbus_byte_ > timeout) { + this->clear_rx_buffer_(LOG_STR("timeout after partial response"), true); + } + + // If we're past the send_wait_time timeout and response buffer doesn't have the start of the expected response + if (this->waiting_for_response_ != 0 && + millis() - this->last_send_ > this->last_send_tx_offset_ + this->send_wait_time_ && + (this->rx_buffer_.empty() || this->rx_buffer_[0] != this->waiting_for_response_)) { + ESP_LOGW(TAG, "Stop waiting for response from %" PRIu8 " %" PRIu32 "ms after last send", + this->waiting_for_response_, millis() - this->last_send_); + this->waiting_for_response_ = 0; + } + + // If there's no response pending and there's commands in the buffer + this->send_next_frame_(); +} + +bool Modbus::tx_blocked() { + const uint32_t now = millis(); + + // We block transmission in any of these case: + // 1. There are bytes in the UART Rx buffer + // 2. There are bytes in our Rx buffer + // 3. We're waiting for a response + // 4. The last sent byte isn't more than frame_delay ms ago (i.e. wait to tell receivers that our previous Tx is done) + // 5. The last received byte isn't more than frame_delay ms ago (i.e. wait to be sure there isn't more Rx coming) + // 6. If we're a client - also wait for the turnaround delay, to give the servers time to process the previous message + return this->available() || !this->rx_buffer_.empty() || (this->waiting_for_response_ != 0) || + (now - this->last_send_ < this->last_send_tx_offset_ + this->frame_delay_ms_ + + (this->role == ModbusRole::CLIENT ? this->turnaround_delay_ms_ : 0)) || + (now - this->last_modbus_byte_ < + this->frame_delay_ms_ + (this->role == ModbusRole::CLIENT ? this->turnaround_delay_ms_ : 0)); +} + +bool Modbus::tx_buffer_empty() { return this->tx_buffer_.empty(); } + +void Modbus::receive_and_parse_modbus_bytes_() { // Read all available bytes in batches to reduce UART call overhead. size_t avail = this->available(); uint8_t buf[64]; @@ -28,33 +87,20 @@ void Modbus::loop() { break; } avail -= to_read; - for (size_t i = 0; i < to_read; i++) { - if (this->parse_modbus_byte_(buf[i])) { - this->last_modbus_byte_ = now; + if (this->rx_buffer_.empty()) { + ESP_LOGV(TAG, "Received first byte %" PRIu8 " (0X%x) %" PRIu32 "ms after last send", buf[i], buf[i], + millis() - this->last_send_); } else { - size_t at = this->rx_buffer_.size(); - if (at > 0) { - ESP_LOGV(TAG, "Clearing buffer of %d bytes - parse failed", at); - this->rx_buffer_.clear(); - } + ESP_LOGVV(TAG, "Received byte %" PRIu8 " (0X%x) %" PRIu32 "ms after last send", buf[i], buf[i], + millis() - this->last_send_); } - } - } - if (now - this->last_modbus_byte_ > 50) { - size_t at = this->rx_buffer_.size(); - if (at > 0) { - ESP_LOGV(TAG, "Clearing buffer of %d bytes - timeout", at); - this->rx_buffer_.clear(); - } - - // stop blocking new send commands after sent_wait_time_ ms after response received - if (now - this->last_send_ > send_wait_time_) { - if (waiting_for_response > 0) { - ESP_LOGV(TAG, "Stop waiting for response from %d", waiting_for_response); + // If the bytes in the rx buffer do not parse, clear out the buffer + if (!this->parse_modbus_byte_(buf[i])) { + this->clear_rx_buffer_(LOG_STR("parse failed"), true); } - waiting_for_response = 0; + this->last_modbus_byte_ = millis(); } } } @@ -63,7 +109,7 @@ bool Modbus::parse_modbus_byte_(uint8_t byte) { size_t at = this->rx_buffer_.size(); this->rx_buffer_.push_back(byte); const uint8_t *raw = &this->rx_buffer_[0]; - ESP_LOGVV(TAG, "Modbus received Byte %d (0X%x)", byte, byte); + // Byte 0: modbus address (match all) if (at == 0) return true; @@ -101,7 +147,7 @@ bool Modbus::parse_modbus_byte_(uint8_t byte) { if (computed_crc != remote_crc) return true; - ESP_LOGD(TAG, "Modbus user-defined function %02X found", function_code); + ESP_LOGD(TAG, "User-defined function %02X found", function_code); } else { // data starts at 2 and length is 4 for read registers commands @@ -152,9 +198,19 @@ bool Modbus::parse_modbus_byte_(uint8_t byte) { uint16_t remote_crc = uint16_t(raw[data_offset + data_len]) | (uint16_t(raw[data_offset + data_len + 1]) << 8); if (computed_crc != remote_crc) { if (this->disable_crc_) { - ESP_LOGD(TAG, "Modbus CRC Check failed, but ignored! %02X!=%02X", computed_crc, remote_crc); + ESP_LOGD(TAG, "CRC check failed %" PRIu32 "ms after last send; ignoring", millis() - this->last_send_); +#if ESPHOME_LOG_LEVEL >= ESPHOME_LOG_LEVEL_VERY_VERBOSE + char hex_buf[format_hex_pretty_size(MODBUS_MAX_LOG_BYTES)]; +#endif + ESP_LOGVV(TAG, " (%02X != %02X) %s", computed_crc, remote_crc, + format_hex_pretty_to(hex_buf, this->rx_buffer_.data(), this->rx_buffer_.size())); } else { - ESP_LOGW(TAG, "Modbus CRC Check failed! %02X!=%02X", computed_crc, remote_crc); + ESP_LOGW(TAG, "CRC check failed %" PRIu32 "ms after last send", millis() - this->last_send_); +#if ESPHOME_LOG_LEVEL >= ESPHOME_LOG_LEVEL_VERY_VERBOSE + char hex_buf[format_hex_pretty_size(MODBUS_MAX_LOG_BYTES)]; +#endif + ESP_LOGVV(TAG, " (%02X != %02X) %s", computed_crc, remote_crc, + format_hex_pretty_to(hex_buf, this->rx_buffer_.data(), this->rx_buffer_.size())); return false; } } @@ -164,52 +220,101 @@ bool Modbus::parse_modbus_byte_(uint8_t byte) { for (auto *device : this->devices_) { if (device->address_ == address) { found = true; - // Is it an error response? - if ((function_code & FUNCTION_CODE_EXCEPTION_MASK) == FUNCTION_CODE_EXCEPTION_MASK) { - ESP_LOGD(TAG, "Modbus error function code: 0x%X exception: %d", function_code, raw[2]); - if (waiting_for_response != 0) { - device->on_modbus_error(function_code & FUNCTION_CODE_MASK, raw[2]); - } else { - // Ignore modbus exception not related to a pending command - ESP_LOGD(TAG, "Ignoring Modbus error - not expecting a response"); - } - continue; - } if (this->role == ModbusRole::SERVER) { if (function_code == ModbusFunctionCode::READ_HOLDING_REGISTERS || function_code == ModbusFunctionCode::READ_INPUT_REGISTERS) { device->on_modbus_read_registers(function_code, uint16_t(data[1]) | (uint16_t(data[0]) << 8), uint16_t(data[3]) | (uint16_t(data[2]) << 8)); - continue; - } - if (function_code == ModbusFunctionCode::WRITE_SINGLE_REGISTER || - function_code == ModbusFunctionCode::WRITE_MULTIPLE_REGISTERS) { + } else if (function_code == ModbusFunctionCode::WRITE_SINGLE_REGISTER || + function_code == ModbusFunctionCode::WRITE_MULTIPLE_REGISTERS) { device->on_modbus_write_registers(function_code, data); - continue; + } + } else { // We're a client + // Is it an error response? + if ((function_code & FUNCTION_CODE_EXCEPTION_MASK) == FUNCTION_CODE_EXCEPTION_MASK) { + uint8_t exception = raw[2]; + ESP_LOGW(TAG, + "Error function code: 0x%X exception: %" PRIu8 ", address: %" PRIu8 ", %" PRIu32 + "ms after last send", + function_code, exception, address, millis() - this->last_send_); + if (this->waiting_for_response_ == address) { + device->on_modbus_error(function_code & FUNCTION_CODE_MASK, exception); + } else { + // Ignore modbus exception not related to a pending command + ESP_LOGD(TAG, "Ignoring error - not expecting a response from %" PRIu8 "", address); + } + } else { // Not an error response + if (this->waiting_for_response_ == address) { + device->on_modbus_data(data); + } else { + // Ignore modbus response not related to a pending command + ESP_LOGW(TAG, "Ignoring response - not expecting a response from %" PRIu8 ", %" PRIu32 "ms after last send", + address, millis() - this->last_send_); + } } } - // fallthrough for other function codes - device->on_modbus_data(data); } } - waiting_for_response = 0; - if (!found) { - ESP_LOGW(TAG, "Got Modbus frame from unknown address 0x%02X! ", address); + if (!found && this->role == ModbusRole::CLIENT) { + ESP_LOGW(TAG, "Got frame from unknown address %" PRIu8 ", %" PRIu32 "ms after last send", address, + millis() - this->last_send_); } - // reset buffer - ESP_LOGV(TAG, "Clearing buffer of %d bytes - parse succeeded", at); - this->rx_buffer_.clear(); + this->clear_rx_buffer_(LOG_STR("parse succeeded")); + + if (this->waiting_for_response_ == address) + this->waiting_for_response_ = 0; + return true; } +void Modbus::send_next_frame_() { + if (this->tx_buffer_.empty()) + return; + + if (this->tx_blocked()) + return; + + const ModbusDeviceCommand &frame = this->tx_buffer_.front(); + + if (this->role == ModbusRole::CLIENT) { + this->waiting_for_response_ = frame.data.get()[0]; + } + + if (this->flow_control_pin_ != nullptr) { + this->flow_control_pin_->digital_write(true); + this->write_array(frame.data.get(), frame.size); + this->flush(); + this->flow_control_pin_->digital_write(false); + this->last_send_tx_offset_ = 0; + } else { + this->write_array(frame.data.get(), frame.size); + this->last_send_tx_offset_ = frame.size * 11 * 1000 / this->parent_->get_baud_rate() + 1; + } + +#if ESPHOME_LOG_LEVEL >= ESPHOME_LOG_LEVEL_VERBOSE + char hex_buf[format_hex_pretty_size(MODBUS_MAX_LOG_BYTES)]; +#endif + ESP_LOGV(TAG, "Write: %s %" PRIu32 "ms after last send", format_hex_pretty_to(hex_buf, frame.data.get(), frame.size), + millis() - this->last_send_); + this->last_send_ = millis(); + this->tx_buffer_.pop_front(); + if (!this->tx_buffer_.empty()) { + ESP_LOGV(TAG, "Write queue contains %" PRIu32 " items.", this->tx_buffer_.size()); + } +} + void Modbus::dump_config() { ESP_LOGCONFIG(TAG, "Modbus:\n" " Send Wait Time: %d ms\n" + " Turnaround Time: %d ms\n" + " Frame Delay: %d ms\n" + " Long Rx Buffer Delay: %d ms\n" " CRC Disabled: %s", - this->send_wait_time_, YESNO(this->disable_crc_)); + this->send_wait_time_, this->turnaround_delay_ms_, this->frame_delay_ms_, + this->long_rx_buffer_delay_ms_, YESNO(this->disable_crc_)); LOG_PIN(" Flow Control Pin: ", this->flow_control_pin_); } float Modbus::get_setup_priority() const { @@ -228,15 +333,6 @@ void Modbus::send(uint8_t address, uint8_t function_code, uint16_t start_address return; } - static constexpr size_t ADDR_SIZE = 1; - static constexpr size_t FC_SIZE = 1; - static constexpr size_t START_ADDR_SIZE = 2; - static constexpr size_t NUM_ENTITIES_SIZE = 2; - static constexpr size_t BYTE_COUNT_SIZE = 1; - static constexpr size_t MAX_PAYLOAD_SIZE = std::numeric_limits::max(); - static constexpr size_t CRC_SIZE = 2; - static constexpr size_t MAX_FRAME_SIZE = - ADDR_SIZE + FC_SIZE + START_ADDR_SIZE + NUM_ENTITIES_SIZE + BYTE_COUNT_SIZE + MAX_PAYLOAD_SIZE + CRC_SIZE; uint8_t data[MAX_FRAME_SIZE]; size_t pos = 0; @@ -259,29 +355,16 @@ void Modbus::send(uint8_t address, uint8_t function_code, uint16_t start_address } else { payload_len = 2; // Write single register or coil } + if (payload_len + pos + 2 > MAX_FRAME_SIZE) { // Check if payload fits (accounting for CRC) + ESP_LOGE(TAG, "Payload too large to send: %d bytes", payload_len); + return; + } for (int i = 0; i < payload_len; i++) { data[pos++] = payload[i]; } } - auto crc = crc16(data, pos); - data[pos++] = crc >> 0; - data[pos++] = crc >> 8; - - if (this->flow_control_pin_ != nullptr) - this->flow_control_pin_->digital_write(true); - - this->write_array(data, pos); - this->flush(); - - if (this->flow_control_pin_ != nullptr) - this->flow_control_pin_->digital_write(false); - waiting_for_response = address; - last_send_ = millis(); -#if ESPHOME_LOG_LEVEL >= ESPHOME_LOG_LEVEL_VERBOSE - char hex_buf[format_hex_pretty_size(MODBUS_MAX_LOG_BYTES)]; -#endif - ESP_LOGV(TAG, "Modbus write: %s", format_hex_pretty_to(hex_buf, data, pos)); + this->queue_raw_(data, pos); } // Helper function for lambdas @@ -290,23 +373,44 @@ void Modbus::send_raw(const std::vector &payload) { if (payload.empty()) { return; } + // Frame size: payload + CRC(2) + if (payload.size() + 2 > MAX_FRAME_SIZE) { + ESP_LOGE(TAG, "Attempted to send frame larger than max frame size of %d bytes", MAX_FRAME_SIZE); + return; + } + // Use stack buffer - Modbus frames are small and bounded + uint8_t data[MAX_FRAME_SIZE]; - if (this->flow_control_pin_ != nullptr) - this->flow_control_pin_->digital_write(true); + std::memcpy(data, payload.data(), payload.size()); - auto crc = crc16(payload.data(), payload.size()); - this->write_array(payload); - this->write_byte(crc & 0xFF); - this->write_byte((crc >> 8) & 0xFF); - this->flush(); - if (this->flow_control_pin_ != nullptr) - this->flow_control_pin_->digital_write(false); - waiting_for_response = payload[0]; -#if ESPHOME_LOG_LEVEL >= ESPHOME_LOG_LEVEL_VERBOSE - char hex_buf[format_hex_pretty_size(MODBUS_MAX_LOG_BYTES)]; + this->queue_raw_(data, payload.size()); +} + +// Assume data and length is valid and append CRC, then queue for sending. Used internally to avoid unnecessary copying +// of data into vectors +void Modbus::queue_raw_(const uint8_t *data, uint16_t len) { + if (this->tx_buffer_.size() < MODBUS_TX_BUFFER_SIZE) { + this->tx_buffer_.emplace_back(data, len); + } else { +#if ESPHOME_LOG_LEVEL >= ESPHOME_LOG_LEVEL_ERROR + char hex_buf[format_hex_pretty_size(MODBUS_MAX_LOG_BYTES)]; #endif - ESP_LOGV(TAG, "Modbus write raw: %s", format_hex_pretty_to(hex_buf, payload.data(), payload.size())); - last_send_ = millis(); + ESP_LOGE(TAG, "Write buffer full, dropped: %s", format_hex_pretty_to(hex_buf, data, len)); + } +} + +void Modbus::clear_rx_buffer_(const LogString *reason, bool warn) { + size_t at = this->rx_buffer_.size(); + if (at > 0) { + if (warn) { + ESP_LOGW(TAG, "Clearing buffer of %" PRIu32 " bytes - %s %" PRIu32 "ms after last send", at, LOG_STR_ARG(reason), + millis() - this->last_send_); + } else { + ESP_LOGV(TAG, "Clearing buffer of %" PRIu32 " bytes - %s %" PRIu32 "ms after last send", at, LOG_STR_ARG(reason), + millis() - this->last_send_); + } + this->rx_buffer_.clear(); + } } } // namespace modbus diff --git a/esphome/components/modbus/modbus.h b/esphome/components/modbus/modbus.h index fac74aaadfd..c90d4c78ae2 100644 --- a/esphome/components/modbus/modbus.h +++ b/esphome/components/modbus/modbus.h @@ -5,11 +5,16 @@ #include "esphome/components/modbus/modbus_definitions.h" +#include +#include #include +#include namespace esphome { namespace modbus { +static constexpr uint16_t MODBUS_TX_BUFFER_SIZE = 15; + enum ModbusRole { CLIENT, SERVER, @@ -17,6 +22,19 @@ enum ModbusRole { class ModbusDevice; +struct ModbusDeviceCommand { + // Frame with exact-size allocation to avoid std::vector overhead + std::unique_ptr data; + uint16_t size; // Modbus RTU max is 256 bytes + + ModbusDeviceCommand(const uint8_t *src, uint16_t len) : data(std::make_unique(len + 2)), size(len + 2) { + std::memcpy(this->data.get(), src, len); + auto crc = crc16(data.get(), len); + data[len + 0] = crc >> 0; + data[len + 1] = crc >> 8; + } +}; + class Modbus : public uart::UARTDevice, public Component { public: Modbus() = default; @@ -30,28 +48,45 @@ class Modbus : public uart::UARTDevice, public Component { void register_device(ModbusDevice *device) { this->devices_.push_back(device); } float get_setup_priority() const override; + bool tx_buffer_empty(); + bool tx_blocked(); void send(uint8_t address, uint8_t function_code, uint16_t start_address, uint16_t number_of_entities, uint8_t payload_len = 0, const uint8_t *payload = nullptr); void send_raw(const std::vector &payload); void set_role(ModbusRole role) { this->role = role; } void set_flow_control_pin(GPIOPin *flow_control_pin) { this->flow_control_pin_ = flow_control_pin; } - uint8_t waiting_for_response{0}; - void set_send_wait_time(uint16_t time_in_ms) { send_wait_time_ = time_in_ms; } - void set_disable_crc(bool disable_crc) { disable_crc_ = disable_crc; } + void set_send_wait_time(uint16_t time_in_ms) { this->send_wait_time_ = time_in_ms; } + void set_turnaround_time(uint16_t time_in_ms) { this->turnaround_delay_ms_ = time_in_ms; } + void set_disable_crc(bool disable_crc) { this->disable_crc_ = disable_crc; } ModbusRole role; protected: - GPIOPin *flow_control_pin_{nullptr}; - bool parse_modbus_byte_(uint8_t byte); - uint16_t send_wait_time_{250}; - bool disable_crc_; - std::vector rx_buffer_; + void receive_and_parse_modbus_bytes_(); + void clear_rx_buffer_(const LogString *reason, bool warn = false); + void send_next_frame_(); + void queue_raw_(const uint8_t *data, uint16_t len); + uint32_t last_modbus_byte_{0}; uint32_t last_send_{0}; + uint32_t last_send_tx_offset_{0}; + uint16_t frame_delay_ms_{5}; + uint16_t long_rx_buffer_delay_ms_{0}; + uint16_t send_wait_time_{250}; + uint16_t turnaround_delay_ms_{100}; + uint8_t waiting_for_response_{0}; + bool disable_crc_{false}; + + GPIOPin *flow_control_pin_{nullptr}; + + std::vector rx_buffer_; std::vector devices_; + // std::deque is appropriate here since we need a FIFO buffer, and we can't know ahead of time how many + // requests will be queued. Each modbus component may queue multiple requests, and the sequence of scheduling + // may change at run time. + std::deque tx_buffer_; }; class ModbusDevice { @@ -76,7 +111,9 @@ class ModbusDevice { this->send_raw(error_response); } // If more than one device is connected block sending a new command before a response is received - bool waiting_for_response() { return parent_->waiting_for_response != 0; } + ESPDEPRECATED("Use ready_for_immediate_send() instead. Removed in 2026.9.0", "2026.3.0") + bool waiting_for_response() { return !ready_for_immediate_send(); } + bool ready_for_immediate_send() { return parent_->tx_buffer_empty() && !parent_->tx_blocked(); } protected: friend Modbus; diff --git a/esphome/components/modbus/modbus_definitions.h b/esphome/components/modbus/modbus_definitions.h index 07f101ae4c5..c86d548578a 100644 --- a/esphome/components/modbus/modbus_definitions.h +++ b/esphome/components/modbus/modbus_definitions.h @@ -81,6 +81,8 @@ const uint8_t MAX_NUM_OF_REGISTERS_TO_WRITE = 123; // 0x7B // 6.3 03 (0x03) Read Holding Registers // 6.4 04 (0x04) Read Input Registers const uint8_t MAX_NUM_OF_REGISTERS_TO_READ = 125; // 0x7D + +static constexpr uint16_t MAX_FRAME_SIZE = 256; /// End of Modbus definitions } // namespace modbus } // namespace esphome diff --git a/esphome/components/modbus_controller/__init__.py b/esphome/components/modbus_controller/__init__.py index c45c338bb32..aea79b20536 100644 --- a/esphome/components/modbus_controller/__init__.py +++ b/esphome/components/modbus_controller/__init__.py @@ -48,6 +48,7 @@ CONF_SERVER_REGISTERS = "server_registers" MULTI_CONF = True modbus_controller_ns = cg.esphome_ns.namespace("modbus_controller") +modbus_ns = cg.esphome_ns.namespace("modbus") ModbusController = modbus_controller_ns.class_( "ModbusController", cg.PollingComponent, modbus.ModbusDevice ) @@ -56,7 +57,7 @@ SensorItem = modbus_controller_ns.struct("SensorItem") ServerCourtesyResponse = modbus_controller_ns.struct("ServerCourtesyResponse") ServerRegister = modbus_controller_ns.struct("ServerRegister") -ModbusFunctionCode_ns = modbus_controller_ns.namespace("ModbusFunctionCode") +ModbusFunctionCode_ns = modbus_ns.namespace("ModbusFunctionCode") ModbusFunctionCode = ModbusFunctionCode_ns.enum("ModbusFunctionCode") MODBUS_FUNCTION_CODE = { "read_coils": ModbusFunctionCode.READ_COILS, diff --git a/esphome/components/modbus_controller/modbus_controller.cpp b/esphome/components/modbus_controller/modbus_controller.cpp index 50bd9f45cbd..7f0eb230e0a 100644 --- a/esphome/components/modbus_controller/modbus_controller.cpp +++ b/esphome/components/modbus_controller/modbus_controller.cpp @@ -18,7 +18,7 @@ void ModbusController::setup() { this->create_register_ranges_(); } bool ModbusController::send_next_command_() { uint32_t last_send = millis() - this->last_command_timestamp_; - if ((last_send > this->command_throttle_) && !waiting_for_response() && !this->command_queue_.empty()) { + if ((last_send > this->command_throttle_) && this->ready_for_immediate_send() && !this->command_queue_.empty()) { auto &command = this->command_queue_.front(); // remove from queue if command was sent too often diff --git a/tests/components/modbus/common.yaml b/tests/components/modbus/common.yaml index d636143ec98..221aab4ed82 100644 --- a/tests/components/modbus/common.yaml +++ b/tests/components/modbus/common.yaml @@ -1,3 +1,5 @@ modbus: id: mod_bus1 flow_control_pin: ${flow_control_pin} + send_wait_time: 500ms + turnaround_time: 100ms diff --git a/tests/integration/fixtures/external_components/uart_mock/__init__.py b/tests/integration/fixtures/external_components/uart_mock/__init__.py index abb3abcc419..c10d73354e2 100644 --- a/tests/integration/fixtures/external_components/uart_mock/__init__.py +++ b/tests/integration/fixtures/external_components/uart_mock/__init__.py @@ -71,6 +71,7 @@ RESPONSE_SCHEMA = cv.Schema( { cv.Required(CONF_EXPECT_TX): [cv.hex_uint8_t], cv.Required(CONF_INJECT_RX): [cv.hex_uint8_t], + cv.Optional(CONF_DELAY, default="0ms"): cv.positive_time_period_milliseconds, } ) @@ -151,7 +152,8 @@ async def to_code(config): for response in config[CONF_RESPONSES]: tx_data = response[CONF_EXPECT_TX] rx_data = response[CONF_INJECT_RX] - cg.add(var.add_response(tx_data, rx_data)) + delay_ms = response[CONF_DELAY] + cg.add(var.add_response(tx_data, rx_data, delay_ms)) for periodic in config[CONF_PERIODIC_RX]: data = periodic[CONF_DATA] diff --git a/tests/integration/fixtures/external_components/uart_mock/uart_mock.cpp b/tests/integration/fixtures/external_components/uart_mock/uart_mock.cpp index 83a13793be7..affcc8d908d 100644 --- a/tests/integration/fixtures/external_components/uart_mock/uart_mock.cpp +++ b/tests/integration/fixtures/external_components/uart_mock/uart_mock.cpp @@ -36,8 +36,8 @@ void MockUartComponent::loop() { // component (e.g., LD2410) a chance to process each batch independently. if (this->injection_index_ < this->injections_.size()) { auto &injection = this->injections_[this->injection_index_]; - uint32_t target_time = this->scenario_start_ms_ + this->cumulative_delay_ms_ + injection.delay_ms; - if (now >= target_time) { + uint32_t total_delay = this->cumulative_delay_ms_ + injection.delay_ms; + if (now - this->scenario_start_ms_ >= total_delay) { ESP_LOGD(TAG, "Injecting %zu RX bytes (injection %u)", injection.rx_data.size(), this->injection_index_); this->inject_to_rx_buffer(injection.rx_data); this->cumulative_delay_ms_ += injection.delay_ms; @@ -52,6 +52,15 @@ void MockUartComponent::loop() { periodic.last_inject_ms = now; } } + + // Process delayed responses + for (auto &response : this->responses_) { + if (response.delay_ms > 0 && response.last_match_ms > 0 && now - response.last_match_ms >= response.delay_ms) { + ESP_LOGD(TAG, "Injecting %zu RX bytes for delayed response", response.inject_rx.size()); + this->inject_to_rx_buffer(response.inject_rx); + response.last_match_ms = 0; // Reset to prevent repeated injection + } + } } void MockUartComponent::start_scenario() { @@ -149,8 +158,9 @@ void MockUartComponent::add_injection(const std::vector &rx_data, uint3 this->injections_.push_back({rx_data, delay_ms}); } -void MockUartComponent::add_response(const std::vector &expect_tx, const std::vector &inject_rx) { - this->responses_.push_back({expect_tx, inject_rx}); +void MockUartComponent::add_response(const std::vector &expect_tx, const std::vector &inject_rx, + uint32_t delay_ms) { + this->responses_.push_back({expect_tx, inject_rx, delay_ms, 0}); } void MockUartComponent::add_periodic_rx(const std::vector &data, uint32_t interval_ms) { @@ -166,7 +176,13 @@ void MockUartComponent::try_match_response_() { size_t offset = this->tx_buffer_.size() - response.expect_tx.size(); if (std::equal(response.expect_tx.begin(), response.expect_tx.end(), this->tx_buffer_.begin() + offset)) { ESP_LOGD(TAG, "TX match found, injecting %zu RX bytes", response.inject_rx.size()); - this->inject_to_rx_buffer(response.inject_rx); + if (response.delay_ms > 0) { + ESP_LOGD(TAG, "Delaying response by %u ms", response.delay_ms); + // Schedule the response injection as a future injection + response.last_match_ms = App.get_loop_component_start_time(); + } else { + this->inject_to_rx_buffer(response.inject_rx); + } this->tx_buffer_.clear(); return; } diff --git a/tests/integration/fixtures/external_components/uart_mock/uart_mock.h b/tests/integration/fixtures/external_components/uart_mock/uart_mock.h index b721512f96c..901e371dec4 100644 --- a/tests/integration/fixtures/external_components/uart_mock/uart_mock.h +++ b/tests/integration/fixtures/external_components/uart_mock/uart_mock.h @@ -34,7 +34,8 @@ class MockUartComponent : public uart::UARTComponent, public Component { // Scenario configuration - called from generated code void add_injection(const std::vector &rx_data, uint32_t delay_ms); - void add_response(const std::vector &expect_tx, const std::vector &inject_rx); + void add_response(const std::vector &expect_tx, const std::vector &inject_rx, + uint32_t delay_ms = 0); void add_periodic_rx(const std::vector &data, uint32_t interval_ms); void start_scenario(); @@ -64,6 +65,8 @@ class MockUartComponent : public uart::UARTComponent, public Component { struct Response { std::vector expect_tx; std::vector inject_rx; + uint32_t delay_ms; + uint32_t last_match_ms{0}; }; std::vector responses_; std::vector tx_buffer_; diff --git a/tests/integration/fixtures/uart_mock_modbus.yaml b/tests/integration/fixtures/uart_mock_modbus.yaml index 0a3492a0d2f..3ff7ab01bdf 100644 --- a/tests/integration/fixtures/uart_mock_modbus.yaml +++ b/tests/integration/fixtures/uart_mock_modbus.yaml @@ -25,20 +25,64 @@ uart_mock: auto_start: false debug: responses: - - expect_tx: [0x01, 0x03, 0x00, 0x03, 0x00, 0x01, 0x74, 0x0A] # Read holding register 1 on device 1 + - expect_tx: [0x01, 0x03, 0x00, 0x03, 0x00, 0x01, 0x74, 0x0A] # Read holding register 3 on device 1 (basic_register) inject_rx: [0x01, 0x03, 0x02, 0x01, 0x03, 0xF9, 0xD5] # Return value 0x0103 (hex) = 259 (dec) + - expect_tx: [0x01, 0x03, 0x00, 0x05, 0x00, 0x01, 0x94, 0x0B] # Read holding register 5 on device 1 (delayed_response) + delay: 100ms # Shorter than modbus send_wait_time of 200ms, should succeed + inject_rx: [0x01, 0x03, 0x02, 0x00, 0xFF, 0xF8, 0x04] # Return value 0x00FF (hex) = 255 (dec) + - expect_tx: [0x02, 0x03, 0x00, 0x07, 0x00, 0x01, 0x35, 0xF8] # Read holding register 7 on device 2 (late_response) + delay: 300ms # Longer than modbus send_wait_time of 200ms, should cause timeout + inject_rx: [0x02, 0x03, 0x02, 0x00, 0xF0, 0xFC, 0x00] # Return value 0x00F0 (hex) = 240 (dec) + - expect_tx: [0x03, 0x03, 0x00, 0x09, 0x00, 0x01, 0x55, 0xEA] # Read holding register 9 on device 3 (no_response) + inject_rx: [] # No response, should cause timeout + - expect_tx: [0x01, 0x03, 0x00, 0x0A, 0x00, 0x01, 0xA4, 0x08] # Read holding register A on device 1 (exception_response) + inject_rx: [0x01, 0x83, 0x02, 0xC0, 0xF1] # Exception response with code 2 (illegal data address) modbus: uart_id: virtual_uart_dev + send_wait_time: 200ms + turnaround_time: 10ms modbus_controller: - address: 1 + - address: 1 + id: modbus_controller_ok + max_cmd_retries: 0 + update_interval: 1s + - address: 2 + id: modbus_controller_slow + max_cmd_retries: 0 + update_interval: 1s + - address: 3 + id: modbus_controller_offline + max_cmd_retries: 0 + update_interval: 1s sensor: - platform: modbus_controller name: "basic_register" address: 0x03 register_type: holding + modbus_controller_id: modbus_controller_ok + - platform: modbus_controller + name: "delayed_response" + address: 0x05 + register_type: holding + modbus_controller_id: modbus_controller_ok + - platform: modbus_controller + name: "late_response" + address: 0x07 + register_type: holding + modbus_controller_id: modbus_controller_slow + - platform: modbus_controller + name: "no_response" + address: 0x09 + register_type: holding + modbus_controller_id: modbus_controller_offline + - platform: modbus_controller + name: "exception_response" + address: 0x0A + register_type: holding + modbus_controller_id: modbus_controller_ok button: - platform: template diff --git a/tests/integration/fixtures/uart_mock_modbus_timing.yaml b/tests/integration/fixtures/uart_mock_modbus_timing.yaml index c4e29e5fe8d..f4cf0bde37e 100644 --- a/tests/integration/fixtures/uart_mock_modbus_timing.yaml +++ b/tests/integration/fixtures/uart_mock_modbus_timing.yaml @@ -46,10 +46,12 @@ uart_mock: modbus: uart_id: virtual_uart_dev + turnaround_time: 10ms sensor: - platform: sdm_meter address: 2 + update_interval: 1s phase_a: voltage: name: sdm_voltage diff --git a/tests/integration/test_uart_mock_modbus.py b/tests/integration/test_uart_mock_modbus.py index bf3c0697502..6901dc27fe1 100644 --- a/tests/integration/test_uart_mock_modbus.py +++ b/tests/integration/test_uart_mock_modbus.py @@ -39,9 +39,17 @@ async def test_uart_mock_modbus( # Track sensor state updates (after initial state is swallowed) sensor_states: dict[str, list[float]] = { "basic_register": [], + "delayed_response": [], + "late_response": [], + "no_response": [], + "exception_response": [], } basic_register_changed = loop.create_future() + delayed_response_changed = loop.create_future() + late_response_changed = loop.create_future() + no_response_changed = loop.create_future() + exception_response_changed = loop.create_future() def on_state(state: EntityState) -> None: if isinstance(state, SensorState) and not state.missing_state: @@ -54,6 +62,23 @@ async def test_uart_mock_modbus( and not basic_register_changed.done() ): basic_register_changed.set_result(True) + elif ( + sensor_name == "delayed_response" + and state.state == 255.0 + and not delayed_response_changed.done() + ): + delayed_response_changed.set_result(True) + elif ( + sensor_name == "late_response" and not late_response_changed.done() + ): + late_response_changed.set_result(True) + elif sensor_name == "no_response" and not no_response_changed.done(): + no_response_changed.set_result(True) + elif ( + sensor_name == "exception_response" + and not exception_response_changed.done() + ): + exception_response_changed.set_result(True) async with ( run_compiled(yaml_config), @@ -79,20 +104,52 @@ async def test_uart_mock_modbus( assert start_btn is not None, "Start Scenario button not found" client.button_command(start_btn.key) + try: + await asyncio.wait_for(delayed_response_changed, timeout=2.0) + except TimeoutError: + pytest.fail( + f"Timeout waiting for delayed_response change. Received sensor states:\n" + f" delayed_response: {sensor_states['delayed_response']}\n" + ) + + try: + await asyncio.wait_for(late_response_changed, timeout=2.0) + pytest.fail( + f"late_response change should not have been triggered, but was. Received sensor states:\n" + f" late_response: {sensor_states['late_response']}\n" + ) + except TimeoutError: + pass # Expected timeout since we never inject a response for late_response + + try: + await asyncio.wait_for(no_response_changed, timeout=2.0) + pytest.fail( + f"no_response change should not have been triggered, but was. Received sensor states:\n" + f" no_response: {sensor_states['no_response']}\n" + ) + except TimeoutError: + pass # Expected timeout since we never inject a response for no_response + # Wait for basic register to be updated with successful parse try: - await asyncio.wait_for(basic_register_changed, timeout=15.0) + await asyncio.wait_for(basic_register_changed, timeout=2.0) except TimeoutError: pytest.fail( f"Timeout waiting for Basic Register change. Received sensor states:\n" f" basic_register: {sensor_states['basic_register']}\n" ) + try: + await asyncio.wait_for(exception_response_changed, timeout=2.0) + pytest.fail( + f"exception_response change should not have been triggered, but was. Received sensor states:\n" + f" exception_response: {sensor_states['exception_response']}\n" + ) + except TimeoutError: + pass + @pytest.mark.asyncio -@pytest.mark.xfail( - reason="There is a bug in UART which will timeout for long responses." -) async def test_uart_mock_modbus_timing( yaml_config: str, run_compiled: RunCompiledFunction, @@ -155,7 +212,7 @@ async def test_uart_mock_modbus_timing( # Wait for voltage to be updated with successful parse try: - await asyncio.wait_for(voltage_changed, timeout=15.0) + await asyncio.wait_for(voltage_changed, timeout=2.0) except TimeoutError: pytest.fail( f"Timeout waiting for SDM voltage change. Received sensor states:\n"