diff --git a/.github/scripts/auto-label-pr/detectors.js b/.github/scripts/auto-label-pr/detectors.js index a45a84f2196..80d8847bc1f 100644 --- a/.github/scripts/auto-label-pr/detectors.js +++ b/.github/scripts/auto-label-pr/detectors.js @@ -1,5 +1,12 @@ const fs = require('fs'); const { DOCS_PR_PATTERNS } = require('./constants'); +const { + COMPONENT_REGEX, + detectComponents, + hasCoreChanges, + hasDashboardChanges, + hasGitHubActionsChanges, +} = require('../detect-tags'); // Strategy: Merge branch detection async function detectMergeBranch(context) { @@ -20,15 +27,13 @@ async function detectMergeBranch(context) { // Strategy: Component and platform labeling async function detectComponentPlatforms(changedFiles, apiData) { const labels = new Set(); - const componentRegex = /^esphome\/components\/([^\/]+)\//; const targetPlatformRegex = new RegExp(`^esphome\/components\/(${apiData.targetPlatforms.join('|')})/`); - for (const file of changedFiles) { - const componentMatch = file.match(componentRegex); - if (componentMatch) { - labels.add(`component: ${componentMatch[1]}`); - } + for (const comp of detectComponents(changedFiles)) { + labels.add(`component: ${comp}`); + } + for (const file of changedFiles) { const platformMatch = file.match(targetPlatformRegex); if (platformMatch) { labels.add(`platform: ${platformMatch[1]}`); @@ -90,15 +95,9 @@ async function detectNewPlatforms(prFiles, apiData) { // Strategy: Core files detection async function detectCoreChanges(changedFiles) { const labels = new Set(); - const coreFiles = changedFiles.filter(file => - file.startsWith('esphome/core/') || - (file.startsWith('esphome/') && file.split('/').length === 2) - ); - - if (coreFiles.length > 0) { + if (hasCoreChanges(changedFiles)) { labels.add('core'); } - return labels; } @@ -131,29 +130,18 @@ async function detectPRSize(prFiles, totalAdditions, totalDeletions, totalChange // Strategy: Dashboard changes async function detectDashboardChanges(changedFiles) { const labels = new Set(); - const dashboardFiles = changedFiles.filter(file => - file.startsWith('esphome/dashboard/') || - file.startsWith('esphome/components/dashboard_import/') - ); - - if (dashboardFiles.length > 0) { + if (hasDashboardChanges(changedFiles)) { labels.add('dashboard'); } - return labels; } // Strategy: GitHub Actions changes async function detectGitHubActionsChanges(changedFiles) { const labels = new Set(); - const githubActionsFiles = changedFiles.filter(file => - file.startsWith('.github/workflows/') - ); - - if (githubActionsFiles.length > 0) { + if (hasGitHubActionsChanges(changedFiles)) { labels.add('github-actions'); } - return labels; } @@ -259,7 +247,7 @@ async function detectDeprecatedComponents(github, context, changedFiles) { const { owner, repo } = context.repo; // Compile regex once for better performance - const componentFileRegex = /^esphome\/components\/([^\/]+)\//; + const componentFileRegex = COMPONENT_REGEX; // Get files that are modified or added in components directory const componentFiles = changedFiles.filter(file => componentFileRegex.test(file)); diff --git a/.github/scripts/detect-tags.js b/.github/scripts/detect-tags.js new file mode 100644 index 00000000000..3933776c616 --- /dev/null +++ b/.github/scripts/detect-tags.js @@ -0,0 +1,66 @@ +/** + * Shared tag detection from changed file paths. + * Used by pr-title-check and auto-label-pr workflows. + */ + +const COMPONENT_REGEX = /^esphome\/components\/([^\/]+)\//; + +/** + * Detect component names from changed files. + * @param {string[]} changedFiles - List of changed file paths + * @returns {Set} Set of component names + */ +function detectComponents(changedFiles) { + const components = new Set(); + for (const file of changedFiles) { + const match = file.match(COMPONENT_REGEX); + if (match) { + components.add(match[1]); + } + } + return components; +} + +/** + * Detect if core files were changed. + * Core files are in esphome/core/ or top-level esphome/ directory. + * @param {string[]} changedFiles - List of changed file paths + * @returns {boolean} + */ +function hasCoreChanges(changedFiles) { + return changedFiles.some(file => + file.startsWith('esphome/core/') || + (file.startsWith('esphome/') && file.split('/').length === 2) + ); +} + +/** + * Detect if dashboard files were changed. + * @param {string[]} changedFiles - List of changed file paths + * @returns {boolean} + */ +function hasDashboardChanges(changedFiles) { + return changedFiles.some(file => + file.startsWith('esphome/dashboard/') || + file.startsWith('esphome/components/dashboard_import/') + ); +} + +/** + * Detect if GitHub Actions files were changed. + * @param {string[]} changedFiles - List of changed file paths + * @returns {boolean} + */ +function hasGitHubActionsChanges(changedFiles) { + return changedFiles.some(file => + file.startsWith('.github/workflows/') + ); +} + +module.exports = { + COMPONENT_REGEX, + detectComponents, + hasCoreChanges, + hasDashboardChanges, + hasGitHubActionsChanges, +}; diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b752701920a..4c9e8c58bcc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -945,13 +945,13 @@ jobs: python-version: ${{ env.DEFAULT_PYTHON }} cache-key: ${{ needs.common.outputs.cache-key }} - name: Download target analysis JSON - uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 + uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0 with: name: memory-analysis-target path: ./memory-analysis continue-on-error: true - name: Download PR analysis JSON - uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 + uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0 with: name: memory-analysis-pr path: ./memory-analysis diff --git a/.github/workflows/pr-title-check.yml b/.github/workflows/pr-title-check.yml new file mode 100644 index 00000000000..f23c2c870ee --- /dev/null +++ b/.github/workflows/pr-title-check.yml @@ -0,0 +1,76 @@ +name: PR Title Check + +on: + pull_request: + types: [opened, edited, synchronize, reopened] + +permissions: + contents: read + pull-requests: read + +jobs: + check: + name: Validate PR title + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0 + with: + script: | + const { + detectComponents, + hasCoreChanges, + hasDashboardChanges, + hasGitHubActionsChanges, + } = require('./.github/scripts/detect-tags.js'); + + const title = context.payload.pull_request.title; + + // Block titles starting with "word:" or "word(scope):" patterns + const commitStylePattern = /^\w+(\(.*?\))?[!]?\s*:/; + if (commitStylePattern.test(title)) { + core.setFailed( + `PR title should not start with a "prefix:" style format.\n` + + `Please use the format: [component] Brief description\n` + + `Example: [pn532] Add health checking and auto-reset` + ); + return; + } + + // Get changed files to detect tags + const files = await github.paginate(github.rest.pulls.listFiles, { + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: context.issue.number, + }); + const filenames = files.map(f => f.filename); + + // Detect tags from changed files using shared logic + const tags = new Set(); + + for (const comp of detectComponents(filenames)) { + tags.add(comp); + } + if (hasCoreChanges(filenames)) tags.add('core'); + if (hasDashboardChanges(filenames)) tags.add('dashboard'); + if (hasGitHubActionsChanges(filenames)) tags.add('ci'); + + if (tags.size === 0) { + return; + } + + // Check title starts with [tag] prefix + const bracketPattern = /^\[\w+\]/; + if (!bracketPattern.test(title)) { + const suggestion = [...tags].map(c => `[${c}]`).join(''); + // Skip if the suggested prefix would be too long for a readable title + if (suggestion.length > 40) { + return; + } + core.setFailed( + `PR modifies: ${[...tags].join(', ')}\n` + + `Title must start with a [tag] prefix.\n` + + `Suggested: ${suggestion} ` + ); + } diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 0cca4dcaf6b..17a2616dffc 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -171,7 +171,7 @@ jobs: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Download digests - uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 + uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0 with: pattern: digests-* path: /tmp/digests diff --git a/esphome/components/esp32/__init__.py b/esphome/components/esp32/__init__.py index a34747a183a..e2efcff587a 100644 --- a/esphome/components/esp32/__init__.py +++ b/esphome/components/esp32/__init__.py @@ -890,10 +890,10 @@ def final_validate(config): ) ) if advanced[CONF_EXECUTE_FROM_PSRAM]: - if config[CONF_VARIANT] != VARIANT_ESP32S3: + if config[CONF_VARIANT] not in {VARIANT_ESP32S3, VARIANT_ESP32P4}: errs.append( cv.Invalid( - f"'{CONF_EXECUTE_FROM_PSRAM}' is only supported on {VARIANT_ESP32S3} variant", + f"'{CONF_EXECUTE_FROM_PSRAM}' is not available on this esp32 variant", path=[CONF_FRAMEWORK, CONF_ADVANCED, CONF_EXECUTE_FROM_PSRAM], ) ) @@ -1436,6 +1436,7 @@ async def to_code(config): cg.set_cpp_standard("gnu++20") cg.add_build_flag("-DUSE_ESP32") + cg.add_define("USE_NATIVE_64BIT_TIME") cg.add_build_flag("-Wl,-z,noexecstack") cg.add_define("ESPHOME_BOARD", config[CONF_BOARD]) variant = config[CONF_VARIANT] @@ -1627,8 +1628,13 @@ async def to_code(config): _configure_lwip_max_sockets(conf) if advanced[CONF_EXECUTE_FROM_PSRAM]: - add_idf_sdkconfig_option("CONFIG_SPIRAM_FETCH_INSTRUCTIONS", True) - add_idf_sdkconfig_option("CONFIG_SPIRAM_RODATA", True) + if variant == VARIANT_ESP32S3: + add_idf_sdkconfig_option("CONFIG_SPIRAM_FETCH_INSTRUCTIONS", True) + add_idf_sdkconfig_option("CONFIG_SPIRAM_RODATA", True) + elif variant == VARIANT_ESP32P4: + add_idf_sdkconfig_option("CONFIG_SPIRAM_XIP_FROM_PSRAM", True) + else: + raise ValueError("Unhandled ESP32 variant") # Apply LWIP core locking for better socket performance # This is already enabled by default in Arduino framework, where it provides diff --git a/esphome/components/esp8266/core.cpp b/esphome/components/esp8266/core.cpp index 06d5ed0f2b0..a43e917b1c6 100644 --- a/esphome/components/esp8266/core.cpp +++ b/esphome/components/esp8266/core.cpp @@ -3,7 +3,7 @@ #include "core.h" #include "esphome/core/defines.h" #include "esphome/core/hal.h" -#include "esphome/core/application.h" +#include "esphome/core/time_64.h" #include "esphome/core/helpers.h" #include "preferences.h" #include @@ -17,7 +17,7 @@ namespace esphome { void HOT yield() { ::yield(); } uint32_t IRAM_ATTR HOT millis() { return ::millis(); } -uint64_t millis_64() { return App.scheduler.millis_64_impl_(::millis()); } +uint64_t millis_64() { return Millis64Impl::compute(::millis()); } void HOT delay(uint32_t ms) { ::delay(ms); } uint32_t IRAM_ATTR HOT micros() { return ::micros(); } void IRAM_ATTR HOT delayMicroseconds(uint32_t us) { delay_microseconds_safe(us); } diff --git a/esphome/components/host/__init__.py b/esphome/components/host/__init__.py index ba05e497c8b..8adbfb02ec5 100644 --- a/esphome/components/host/__init__.py +++ b/esphome/components/host/__init__.py @@ -41,6 +41,7 @@ CONFIG_SCHEMA = cv.All( async def to_code(config): cg.add_build_flag("-DUSE_HOST") + cg.add_define("USE_NATIVE_64BIT_TIME") cg.add_define("USE_ESPHOME_HOST_MAC_ADDRESS", config[CONF_MAC_ADDRESS].parts) cg.add_build_flag("-std=gnu++20") cg.add_define("ESPHOME_BOARD", "host") diff --git a/esphome/components/libretiny/core.cpp b/esphome/components/libretiny/core.cpp index 8d6ae379828..47f7b6d2fa2 100644 --- a/esphome/components/libretiny/core.cpp +++ b/esphome/components/libretiny/core.cpp @@ -3,7 +3,7 @@ #include "core.h" #include "esphome/core/defines.h" #include "esphome/core/hal.h" -#include "esphome/core/application.h" +#include "esphome/core/time_64.h" #include "esphome/core/helpers.h" #include "preferences.h" @@ -14,7 +14,7 @@ namespace esphome { void HOT yield() { ::yield(); } uint32_t IRAM_ATTR HOT millis() { return ::millis(); } -uint64_t millis_64() { return App.scheduler.millis_64_impl_(::millis()); } +uint64_t millis_64() { return Millis64Impl::compute(::millis()); } uint32_t IRAM_ATTR HOT micros() { return ::micros(); } void HOT delay(uint32_t ms) { ::delay(ms); } void IRAM_ATTR HOT delayMicroseconds(uint32_t us) { ::delayMicroseconds(us); } diff --git a/esphome/components/rp2040/__init__.py b/esphome/components/rp2040/__init__.py index 23f12e651f3..ea269a47c58 100644 --- a/esphome/components/rp2040/__init__.py +++ b/esphome/components/rp2040/__init__.py @@ -169,6 +169,7 @@ async def to_code(config): cg.add_platformio_option("lib_compat_mode", "strict") cg.add_platformio_option("board", config[CONF_BOARD]) cg.add_build_flag("-DUSE_RP2040") + cg.add_define("USE_NATIVE_64BIT_TIME") cg.set_cpp_standard("gnu++20") cg.add_define("ESPHOME_BOARD", config[CONF_BOARD]) cg.add_define("ESPHOME_VARIANT", "RP2040") diff --git a/esphome/components/zephyr/__init__.py b/esphome/components/zephyr/__init__.py index 43d5cebebb2..4cc71bddca8 100644 --- a/esphome/components/zephyr/__init__.py +++ b/esphome/components/zephyr/__init__.py @@ -112,6 +112,7 @@ def add_extra_script(stage: str, filename: str, path: Path) -> None: def zephyr_to_code(config): cg.add_build_flag("-DUSE_ZEPHYR") + cg.add_define("USE_NATIVE_64BIT_TIME") cg.set_cpp_standard("gnu++20") # build is done by west so bypass board checking in platformio cg.add_platformio_option("boards_dir", CORE.relative_build_path("boards")) diff --git a/esphome/core/config.py b/esphome/core/config.py index 215432835a0..593b4022303 100644 --- a/esphome/core/config.py +++ b/esphome/core/config.py @@ -650,6 +650,12 @@ FILTER_SOURCE_FILES = filter_source_files_from_platform( PlatformFramework.ESP32_ARDUINO, PlatformFramework.ESP32_IDF, }, + "time_64.cpp": { + PlatformFramework.ESP8266_ARDUINO, + PlatformFramework.BK72XX_ARDUINO, + PlatformFramework.RTL87XX_ARDUINO, + PlatformFramework.LN882X_ARDUINO, + }, # Note: lock_free_queue.h and event_pool.h are header files and don't need to be filtered # as they are only included when needed by the preprocessor } diff --git a/esphome/core/defines.h b/esphome/core/defines.h index cc73aa7fe05..3a557b9886c 100644 --- a/esphome/core/defines.h +++ b/esphome/core/defines.h @@ -181,6 +181,11 @@ #define USE_I2S_LEGACY #endif +// Platforms with native 64-bit time sources (no rollover tracking needed) +#if defined(USE_ESP32) || defined(USE_HOST) || defined(USE_ZEPHYR) || defined(USE_RP2040) +#define USE_NATIVE_64BIT_TIME +#endif + // ESP32-specific feature flags #ifdef USE_ESP32 #define USE_MQTT_IDF_ENQUEUE diff --git a/esphome/core/scheduler.cpp b/esphome/core/scheduler.cpp index 2c10e7e2dad..ca560e8250a 100644 --- a/esphome/core/scheduler.cpp +++ b/esphome/core/scheduler.cpp @@ -9,7 +9,6 @@ #include #include #include -#include namespace esphome { @@ -28,10 +27,6 @@ static constexpr size_t MAX_POOL_SIZE = 5; // Set to 5 to match the pool size - when we have as many cancelled items as our // pool can hold, it's time to clean up and recycle them. static constexpr uint32_t MAX_LOGICALLY_DELETED_ITEMS = 5; -#if !defined(USE_ESP32) && !defined(USE_HOST) && !defined(USE_ZEPHYR) && !defined(USE_RP2040) -// Half the 32-bit range - used to detect rollovers vs normal time progression -static constexpr uint32_t HALF_MAX_UINT32 = std::numeric_limits::max() / 2; -#endif // max delay to start an interval sequence static constexpr uint32_t MAX_INTERVAL_DELAY = 5000; @@ -152,9 +147,6 @@ void HOT Scheduler::set_timer_common_(Component *component, SchedulerItem::Type return; } - // Get fresh 64-bit timestamp BEFORE taking lock - const uint64_t now_64 = millis_64(); - // Take lock early to protect scheduler_item_pool_ access LockGuard guard{this->lock_}; @@ -181,6 +173,9 @@ void HOT Scheduler::set_timer_common_(Component *component, SchedulerItem::Type } else #endif /* not ESPHOME_THREAD_SINGLE */ { + // Only non-defer items need a timestamp for scheduling + const uint64_t now_64 = millis_64(); + // Type-specific setup if (type == SchedulerItem::INTERVAL) { item->interval = delay; @@ -475,19 +470,8 @@ void HOT Scheduler::call(uint32_t now) { if (now_64 - last_print > 2000) { last_print = now_64; std::vector old_items; -#if !defined(USE_ESP32) && !defined(USE_HOST) && !defined(USE_ZEPHYR) && !defined(USE_RP2040) && \ - defined(ESPHOME_THREAD_MULTI_ATOMICS) - const auto last_dbg = this->last_millis_.load(std::memory_order_relaxed); - const auto major_dbg = this->millis_major_.load(std::memory_order_relaxed); - ESP_LOGD(TAG, "Items: count=%zu, pool=%zu, now=%" PRIu64 " (%" PRIu16 ", %" PRIu32 ")", this->items_.size(), - this->scheduler_item_pool_.size(), now_64, major_dbg, last_dbg); -#elif !defined(USE_ESP32) && !defined(USE_HOST) && !defined(USE_ZEPHYR) && !defined(USE_RP2040) - ESP_LOGD(TAG, "Items: count=%zu, pool=%zu, now=%" PRIu64 " (%" PRIu16 ", %" PRIu32 ")", this->items_.size(), - this->scheduler_item_pool_.size(), now_64, this->millis_major_, this->last_millis_); -#else ESP_LOGD(TAG, "Items: count=%zu, pool=%zu, now=%" PRIu64, this->items_.size(), this->scheduler_item_pool_.size(), now_64); -#endif // Cleanup before debug output this->cleanup_(); while (!this->items_.empty()) { @@ -715,166 +699,6 @@ bool HOT Scheduler::cancel_item_locked_(Component *component, NameType name_type return total_cancelled > 0; } -#if !defined(USE_ESP32) && !defined(USE_HOST) && !defined(USE_ZEPHYR) && !defined(USE_RP2040) -uint64_t Scheduler::millis_64_impl_(uint32_t now) { - // THREAD SAFETY NOTE: - // This function has three implementations, based on the precompiler flags - // - ESPHOME_THREAD_SINGLE - Runs on single-threaded platforms (ESP8266, RP2040, etc.) - // - ESPHOME_THREAD_MULTI_NO_ATOMICS - Runs on multi-threaded platforms without atomics (LibreTiny BK72xx) - // - ESPHOME_THREAD_MULTI_ATOMICS - Runs on multi-threaded platforms with atomics (ESP32, HOST, LibreTiny - // RTL87xx/LN882x, etc.) - // - // Make sure all changes are synchronized if you edit this function. - // - // IMPORTANT: Always pass fresh millis() values to this function. The implementation - // handles out-of-order timestamps between threads, but minimizing time differences - // helps maintain accuracy. - // - -#ifdef ESPHOME_THREAD_SINGLE - // This is the single core implementation. - // - // Single-core platforms have no concurrency, so this is a simple implementation - // that just tracks 32-bit rollover (every 49.7 days) without any locking or atomics. - - uint16_t major = this->millis_major_; - uint32_t last = this->last_millis_; - - // Check for rollover - if (now < last && (last - now) > HALF_MAX_UINT32) { - this->millis_major_++; - major++; - this->last_millis_ = now; -#ifdef ESPHOME_DEBUG_SCHEDULER - ESP_LOGD(TAG, "Detected true 32-bit rollover at %" PRIu32 "ms (was %" PRIu32 ")", now, last); -#endif /* ESPHOME_DEBUG_SCHEDULER */ - } else if (now > last) { - // Only update if time moved forward - this->last_millis_ = now; - } - - // Combine major (high 32 bits) and now (low 32 bits) into 64-bit time - return now + (static_cast(major) << 32); - -#elif defined(ESPHOME_THREAD_MULTI_NO_ATOMICS) - // This is the multi core no atomics implementation. - // - // Without atomics, this implementation uses locks more aggressively: - // 1. Always locks when near the rollover boundary (within 10 seconds) - // 2. Always locks when detecting a large backwards jump - // 3. Updates without lock in normal forward progression (accepting minor races) - // This is less efficient but necessary without atomic operations. - uint16_t major = this->millis_major_; - uint32_t last = this->last_millis_; - - // Define a safe window around the rollover point (10 seconds) - // This covers any reasonable scheduler delays or thread preemption - static constexpr uint32_t ROLLOVER_WINDOW = 10000; // 10 seconds in milliseconds - - // Check if we're near the rollover boundary (close to std::numeric_limits::max() or just past 0) - bool near_rollover = (last > (std::numeric_limits::max() - ROLLOVER_WINDOW)) || (now < ROLLOVER_WINDOW); - - if (near_rollover || (now < last && (last - now) > HALF_MAX_UINT32)) { - // Near rollover or detected a rollover - need lock for safety - LockGuard guard{this->lock_}; - // Re-read with lock held - last = this->last_millis_; - - if (now < last && (last - now) > HALF_MAX_UINT32) { - // True rollover detected (happens every ~49.7 days) - this->millis_major_++; - major++; -#ifdef ESPHOME_DEBUG_SCHEDULER - ESP_LOGD(TAG, "Detected true 32-bit rollover at %" PRIu32 "ms (was %" PRIu32 ")", now, last); -#endif /* ESPHOME_DEBUG_SCHEDULER */ - } - // Update last_millis_ while holding lock - this->last_millis_ = now; - } else if (now > last) { - // Normal case: Not near rollover and time moved forward - // Update without lock. While this may cause minor races (microseconds of - // backwards time movement), they're acceptable because: - // 1. The scheduler operates at millisecond resolution, not microsecond - // 2. We've already prevented the critical rollover race condition - // 3. Any backwards movement is orders of magnitude smaller than scheduler delays - this->last_millis_ = now; - } - // If now <= last and we're not near rollover, don't update - // This minimizes backwards time movement - - // Combine major (high 32 bits) and now (low 32 bits) into 64-bit time - return now + (static_cast(major) << 32); - -#elif defined(ESPHOME_THREAD_MULTI_ATOMICS) - // This is the multi core with atomics implementation. - // - // Uses atomic operations with acquire/release semantics to ensure coherent - // reads of millis_major_ and last_millis_ across cores. Features: - // 1. Epoch-coherency retry loop to handle concurrent updates - // 2. Lock only taken for actual rollover detection and update - // 3. Lock-free CAS updates for normal forward time progression - // 4. Memory ordering ensures cores see consistent time values - - for (;;) { - uint16_t major = this->millis_major_.load(std::memory_order_acquire); - - /* - * Acquire so that if we later decide **not** to take the lock we still - * observe a `millis_major_` value coherent with the loaded `last_millis_`. - * The acquire load ensures any later read of `millis_major_` sees its - * corresponding increment. - */ - uint32_t last = this->last_millis_.load(std::memory_order_acquire); - - // If we might be near a rollover (large backwards jump), take the lock for the entire operation - // This ensures rollover detection and last_millis_ update are atomic together - if (now < last && (last - now) > HALF_MAX_UINT32) { - // Potential rollover - need lock for atomic rollover detection + update - LockGuard guard{this->lock_}; - // Re-read with lock held; mutex already provides ordering - last = this->last_millis_.load(std::memory_order_relaxed); - - if (now < last && (last - now) > HALF_MAX_UINT32) { - // True rollover detected (happens every ~49.7 days) - this->millis_major_.fetch_add(1, std::memory_order_relaxed); - major++; -#ifdef ESPHOME_DEBUG_SCHEDULER - ESP_LOGD(TAG, "Detected true 32-bit rollover at %" PRIu32 "ms (was %" PRIu32 ")", now, last); -#endif /* ESPHOME_DEBUG_SCHEDULER */ - } - /* - * Update last_millis_ while holding the lock to prevent races - * Publish the new low-word *after* bumping `millis_major_` (done above) - * so readers never see a mismatched pair. - */ - this->last_millis_.store(now, std::memory_order_release); - } else { - // Normal case: Try lock-free update, but only allow forward movement within same epoch - // This prevents accidentally moving backwards across a rollover boundary - while (now > last && (now - last) < HALF_MAX_UINT32) { - if (this->last_millis_.compare_exchange_weak(last, now, - std::memory_order_release, // success - std::memory_order_relaxed)) { // failure - break; - } - // CAS failure means no data was published; relaxed is fine - // last is automatically updated by compare_exchange_weak if it fails - } - } - uint16_t major_end = this->millis_major_.load(std::memory_order_relaxed); - if (major_end == major) - return now + (static_cast(major) << 32); - } - // Unreachable - the loop always returns when major_end == major - __builtin_unreachable(); - -#else -#error \ - "No platform threading model defined. One of ESPHOME_THREAD_SINGLE, ESPHOME_THREAD_MULTI_NO_ATOMICS, or ESPHOME_THREAD_MULTI_ATOMICS must be defined." -#endif -} -#endif // !USE_ESP32 && !USE_HOST && !USE_ZEPHYR && !USE_RP2040 - bool HOT Scheduler::SchedulerItem::cmp(const SchedulerItemPtr &a, const SchedulerItemPtr &b) { // High bits are almost always equal (change only on 32-bit rollover ~49 days) // Optimize for common case: check low bits first when high bits are equal diff --git a/esphome/core/scheduler.h b/esphome/core/scheduler.h index 4b2b125e19b..9f48ed7e927 100644 --- a/esphome/core/scheduler.h +++ b/esphome/core/scheduler.h @@ -12,6 +12,7 @@ #include "esphome/core/component.h" #include "esphome/core/hal.h" #include "esphome/core/helpers.h" +#include "esphome/core/time_64.h" namespace esphome { @@ -283,23 +284,16 @@ class Scheduler { bool cancel_retry_(Component *component, NameType name_type, const char *static_name, uint32_t hash_or_id); // Extend a 32-bit millis() value to 64-bit. Use when the caller already has a fresh now. - // On ESP32, ignores now and uses esp_timer_get_time() directly (native 64-bit). - // On non-ESP32, extends now to 64-bit using rollover tracking. + // On platforms with native 64-bit time, ignores now and uses millis_64() directly. + // On other platforms, extends now to 64-bit using rollover tracking. uint64_t millis_64_from_(uint32_t now) { -#if defined(USE_ESP32) || defined(USE_HOST) || defined(USE_ZEPHYR) || defined(USE_RP2040) +#ifdef USE_NATIVE_64BIT_TIME (void) now; return millis_64(); #else - return this->millis_64_impl_(now); + return Millis64Impl::compute(now); #endif } - -#if !defined(USE_ESP32) && !defined(USE_HOST) && !defined(USE_ZEPHYR) && !defined(USE_RP2040) - // On platforms without native 64-bit time, millis_64() HAL function delegates to this - // method which tracks 32-bit millis() rollover using millis_major_ and last_millis_. - friend uint64_t millis_64(); - uint64_t millis_64_impl_(uint32_t now); -#endif // Cleanup logically deleted items from the scheduler // Returns the number of items remaining after cleanup // IMPORTANT: This method should only be called from the main thread (loop task). @@ -565,39 +559,6 @@ class Scheduler { // can stall the entire system, causing timing issues and dropped events for any components that need // to synchronize between tasks (see https://github.com/esphome/backlog/issues/52) std::vector scheduler_item_pool_; - -#if !defined(USE_ESP32) && !defined(USE_HOST) && !defined(USE_ZEPHYR) && !defined(USE_RP2040) - // On platforms with native 64-bit time (ESP32, Host, Zephyr, RP2040), no rollover tracking needed. - // On other platforms, these fields track 32-bit millis() rollover for millis_64_impl_(). -#ifdef ESPHOME_THREAD_MULTI_ATOMICS - /* - * Multi-threaded platforms with atomic support: last_millis_ needs atomic for lock-free updates - * - * MEMORY-ORDERING NOTE - * -------------------- - * `last_millis_` and `millis_major_` form a single 64-bit timestamp split in half. - * Writers publish `last_millis_` with memory_order_release and readers use - * memory_order_acquire. This ensures that once a reader sees the new low word, - * it also observes the corresponding increment of `millis_major_`. - */ - std::atomic last_millis_{0}; -#else /* not ESPHOME_THREAD_MULTI_ATOMICS */ - // Platforms without atomic support or single-threaded platforms - uint32_t last_millis_{0}; -#endif /* else ESPHOME_THREAD_MULTI_ATOMICS */ - - /* - * Upper 16 bits of the 64-bit millis counter. Incremented only while holding - * `lock_`; read concurrently. Atomic (relaxed) avoids a formal data race. - * Ordering relative to `last_millis_` is provided by its release store and the - * corresponding acquire loads. - */ -#ifdef ESPHOME_THREAD_MULTI_ATOMICS - std::atomic millis_major_{0}; -#else /* not ESPHOME_THREAD_MULTI_ATOMICS */ - uint16_t millis_major_{0}; -#endif /* else ESPHOME_THREAD_MULTI_ATOMICS */ -#endif /* !USE_ESP32 && !USE_HOST && !USE_ZEPHYR && !USE_RP2040 */ }; } // namespace esphome diff --git a/esphome/core/time_64.cpp b/esphome/core/time_64.cpp new file mode 100644 index 00000000000..db5df25eb93 --- /dev/null +++ b/esphome/core/time_64.cpp @@ -0,0 +1,207 @@ +#include "esphome/core/defines.h" + +#ifndef USE_NATIVE_64BIT_TIME + +#include "time_64.h" + +#include "esphome/core/helpers.h" +#ifdef ESPHOME_DEBUG_SCHEDULER +#include "esphome/core/log.h" +#include +#endif +#ifdef ESPHOME_THREAD_MULTI_ATOMICS +#include +#endif +#include + +namespace esphome { + +#ifdef ESPHOME_DEBUG_SCHEDULER +static const char *const TAG = "time_64"; +#endif + +uint64_t Millis64Impl::compute(uint32_t now) { + // Half the 32-bit range - used to detect rollovers vs normal time progression + static constexpr uint32_t HALF_MAX_UINT32 = std::numeric_limits::max() / 2; + + // State variables for rollover tracking - static to persist across calls +#ifdef ESPHOME_THREAD_MULTI_ATOMICS + // Mutex for rollover serialization (taken only every ~49.7 days). + // A spinlock would be smaller (~1 byte vs ~80-100 bytes) but is unsafe on + // preemptive single-core RTOS platforms due to priority inversion: a high-priority + // task spinning would prevent the lock holder from running to release it. + static Mutex lock; + /* + * Multi-threaded platforms with atomic support: last_millis needs atomic for lock-free updates. + * Writers publish last_millis with memory_order_release and readers use memory_order_acquire. + * This ensures that once a reader sees the new low word, it also observes the corresponding + * increment of millis_major. + */ + static std::atomic last_millis{0}; + /* + * Upper 16 bits of the 64-bit millis counter. Incremented only while holding lock; + * read concurrently. Atomic (relaxed) avoids a formal data race. Ordering relative + * to last_millis is provided by its release store and the corresponding acquire loads. + */ + static std::atomic millis_major{0}; +#elif !defined(ESPHOME_THREAD_SINGLE) /* ESPHOME_THREAD_MULTI_NO_ATOMICS */ + static Mutex lock; + static uint32_t last_millis{0}; + static uint16_t millis_major{0}; +#else /* ESPHOME_THREAD_SINGLE */ + static uint32_t last_millis{0}; + static uint16_t millis_major{0}; +#endif + + // THREAD SAFETY NOTE: + // This function has three implementations, based on the precompiler flags + // - ESPHOME_THREAD_SINGLE - Runs on single-threaded platforms (ESP8266, etc.) + // - ESPHOME_THREAD_MULTI_NO_ATOMICS - Runs on multi-threaded platforms without atomics (LibreTiny BK72xx) + // - ESPHOME_THREAD_MULTI_ATOMICS - Runs on multi-threaded platforms with atomics (LibreTiny RTL87xx/LN882x, etc.) + // + // Make sure all changes are synchronized if you edit this function. + // + // IMPORTANT: Always pass fresh millis() values to this function. The implementation + // handles out-of-order timestamps between threads, but minimizing time differences + // helps maintain accuracy. + +#ifdef ESPHOME_THREAD_SINGLE + // Single-core platforms have no concurrency, so this is a simple implementation + // that just tracks 32-bit rollover (every 49.7 days) without any locking or atomics. + + uint16_t major = millis_major; + uint32_t last = last_millis; + + // Check for rollover + if (now < last && (last - now) > HALF_MAX_UINT32) { + millis_major++; + major++; + last_millis = now; +#ifdef ESPHOME_DEBUG_SCHEDULER + ESP_LOGD(TAG, "Detected true 32-bit rollover at %" PRIu32 "ms (was %" PRIu32 ")", now, last); +#endif /* ESPHOME_DEBUG_SCHEDULER */ + } else if (now > last) { + // Only update if time moved forward + last_millis = now; + } + + // Combine major (high 32 bits) and now (low 32 bits) into 64-bit time + return now + (static_cast(major) << 32); + +#elif defined(ESPHOME_THREAD_MULTI_NO_ATOMICS) + // Without atomics, this implementation uses locks more aggressively: + // 1. Always locks when near the rollover boundary (within 10 seconds) + // 2. Always locks when detecting a large backwards jump + // 3. Updates without lock in normal forward progression (accepting minor races) + // This is less efficient but necessary without atomic operations. + uint16_t major = millis_major; + uint32_t last = last_millis; + + // Define a safe window around the rollover point (10 seconds) + // This covers any reasonable scheduler delays or thread preemption + static constexpr uint32_t ROLLOVER_WINDOW = 10000; // 10 seconds in milliseconds + + // Check if we're near the rollover boundary (close to std::numeric_limits::max() or just past 0) + bool near_rollover = (last > (std::numeric_limits::max() - ROLLOVER_WINDOW)) || (now < ROLLOVER_WINDOW); + + if (near_rollover || (now < last && (last - now) > HALF_MAX_UINT32)) { + // Near rollover or detected a rollover - need lock for safety + LockGuard guard{lock}; + // Re-read with lock held + last = last_millis; + + if (now < last && (last - now) > HALF_MAX_UINT32) { + // True rollover detected (happens every ~49.7 days) + millis_major++; + major++; +#ifdef ESPHOME_DEBUG_SCHEDULER + ESP_LOGD(TAG, "Detected true 32-bit rollover at %" PRIu32 "ms (was %" PRIu32 ")", now, last); +#endif /* ESPHOME_DEBUG_SCHEDULER */ + } + // Update last_millis while holding lock + last_millis = now; + } else if (now > last) { + // Normal case: Not near rollover and time moved forward + // Update without lock. While this may cause minor races (microseconds of + // backwards time movement), they're acceptable because: + // 1. The scheduler operates at millisecond resolution, not microsecond + // 2. We've already prevented the critical rollover race condition + // 3. Any backwards movement is orders of magnitude smaller than scheduler delays + last_millis = now; + } + // If now <= last and we're not near rollover, don't update + // This minimizes backwards time movement + + // Combine major (high 32 bits) and now (low 32 bits) into 64-bit time + return now + (static_cast(major) << 32); + +#elif defined(ESPHOME_THREAD_MULTI_ATOMICS) + // Uses atomic operations with acquire/release semantics to ensure coherent + // reads of millis_major and last_millis across cores. Features: + // 1. Epoch-coherency retry loop to handle concurrent updates + // 2. Lock only taken for actual rollover detection and update + // 3. Lock-free CAS updates for normal forward time progression + // 4. Memory ordering ensures cores see consistent time values + + for (;;) { + uint16_t major = millis_major.load(std::memory_order_acquire); + + /* + * Acquire so that if we later decide **not** to take the lock we still + * observe a millis_major value coherent with the loaded last_millis. + * The acquire load ensures any later read of millis_major sees its + * corresponding increment. + */ + uint32_t last = last_millis.load(std::memory_order_acquire); + + // If we might be near a rollover (large backwards jump), take the lock + // This ensures rollover detection and last_millis update are atomic together + if (now < last && (last - now) > HALF_MAX_UINT32) { + // Potential rollover - need lock for atomic rollover detection + update + LockGuard guard{lock}; + // Re-read with lock held; mutex already provides ordering + last = last_millis.load(std::memory_order_relaxed); + + if (now < last && (last - now) > HALF_MAX_UINT32) { + // True rollover detected (happens every ~49.7 days) + millis_major.fetch_add(1, std::memory_order_relaxed); + major++; +#ifdef ESPHOME_DEBUG_SCHEDULER + ESP_LOGD(TAG, "Detected true 32-bit rollover at %" PRIu32 "ms (was %" PRIu32 ")", now, last); +#endif /* ESPHOME_DEBUG_SCHEDULER */ + } + /* + * Update last_millis while holding the lock to prevent races. + * Publish the new low-word *after* bumping millis_major (done above) + * so readers never see a mismatched pair. + */ + last_millis.store(now, std::memory_order_release); + } else { + // Normal case: Try lock-free update, but only allow forward movement within same epoch + // This prevents accidentally moving backwards across a rollover boundary + while (now > last && (now - last) < HALF_MAX_UINT32) { + if (last_millis.compare_exchange_weak(last, now, + std::memory_order_release, // success + std::memory_order_relaxed)) { // failure + break; + } + // CAS failure means no data was published; relaxed is fine + // last is automatically updated by compare_exchange_weak if it fails + } + } + uint16_t major_end = millis_major.load(std::memory_order_relaxed); + if (major_end == major) + return now + (static_cast(major) << 32); + } + // Unreachable - the loop always returns when major_end == major + __builtin_unreachable(); + +#else +#error \ + "No platform threading model defined. One of ESPHOME_THREAD_SINGLE, ESPHOME_THREAD_MULTI_NO_ATOMICS, or ESPHOME_THREAD_MULTI_ATOMICS must be defined." +#endif +} + +} // namespace esphome + +#endif // !USE_NATIVE_64BIT_TIME diff --git a/esphome/core/time_64.h b/esphome/core/time_64.h new file mode 100644 index 00000000000..42d4b041e51 --- /dev/null +++ b/esphome/core/time_64.h @@ -0,0 +1,24 @@ +#pragma once +#include "esphome/core/defines.h" + +#ifndef USE_NATIVE_64BIT_TIME + +#include + +namespace esphome { + +class Scheduler; + +/// Extends 32-bit millis() to 64-bit using rollover tracking. +/// Access restricted to platform HAL (millis_64()) and Scheduler. +/// All other code should call millis_64() from hal.h instead. +class Millis64Impl { + friend uint64_t millis_64(); + friend class Scheduler; + + static uint64_t compute(uint32_t now); +}; + +} // namespace esphome + +#endif // !USE_NATIVE_64BIT_TIME diff --git a/requirements_test.txt b/requirements_test.txt index 88a38ffa995..6b2617b6563 100644 --- a/requirements_test.txt +++ b/requirements_test.txt @@ -1,6 +1,6 @@ pylint==4.0.5 flake8==7.3.0 # also change in .pre-commit-config.yaml when updating -ruff==0.15.3 # also change in .pre-commit-config.yaml when updating +ruff==0.15.4 # also change in .pre-commit-config.yaml when updating pyupgrade==3.21.2 # also change in .pre-commit-config.yaml when updating pre-commit diff --git a/tests/component_tests/esp32/config/execute_from_psram_disabled.yaml b/tests/component_tests/esp32/config/execute_from_psram_disabled.yaml new file mode 100644 index 00000000000..b52255c5adb --- /dev/null +++ b/tests/component_tests/esp32/config/execute_from_psram_disabled.yaml @@ -0,0 +1,10 @@ +esphome: + name: test + +esp32: + variant: esp32s3 + framework: + type: esp-idf + +psram: + mode: octal diff --git a/tests/component_tests/esp32/config/execute_from_psram_p4.yaml b/tests/component_tests/esp32/config/execute_from_psram_p4.yaml new file mode 100644 index 00000000000..e9070a5ae18 --- /dev/null +++ b/tests/component_tests/esp32/config/execute_from_psram_p4.yaml @@ -0,0 +1,11 @@ +esphome: + name: test + +esp32: + variant: esp32p4 + framework: + type: esp-idf + advanced: + execute_from_psram: true + +psram: diff --git a/tests/component_tests/esp32/config/execute_from_psram_s3.yaml b/tests/component_tests/esp32/config/execute_from_psram_s3.yaml new file mode 100644 index 00000000000..aec020ddffb --- /dev/null +++ b/tests/component_tests/esp32/config/execute_from_psram_s3.yaml @@ -0,0 +1,12 @@ +esphome: + name: test + +esp32: + variant: esp32s3 + framework: + type: esp-idf + advanced: + execute_from_psram: true + +psram: + mode: octal diff --git a/tests/component_tests/esp32/test_esp32.py b/tests/component_tests/esp32/test_esp32.py index 68bd3a5965b..bd4f9828ce5 100644 --- a/tests/component_tests/esp32/test_esp32.py +++ b/tests/component_tests/esp32/test_esp32.py @@ -2,13 +2,17 @@ Test ESP32 configuration """ +from collections.abc import Callable +from pathlib import Path from typing import Any import pytest from esphome.components.esp32 import VARIANTS +from esphome.components.esp32.const import KEY_ESP32, KEY_SDKCONFIG_OPTIONS import esphome.config_validation as cv from esphome.const import CONF_ESPHOME, PlatformFramework +from esphome.core import CORE from tests.component_tests.types import SetCoreConfigCallable @@ -70,7 +74,7 @@ def test_esp32_config( "advanced": {"execute_from_psram": True}, }, }, - r"'execute_from_psram' is only supported on ESP32S3 variant @ data\['framework'\]\['advanced'\]\['execute_from_psram'\]", + r"'execute_from_psram' is not available on this esp32 variant @ data\['framework'\]\['advanced'\]\['execute_from_psram'\]", id="execute_from_psram_invalid_for_variant_config", ), pytest.param( @@ -82,7 +86,18 @@ def test_esp32_config( }, }, r"'execute_from_psram' requires PSRAM to be configured @ data\['framework'\]\['advanced'\]\['execute_from_psram'\]", - id="execute_from_psram_requires_psram_config", + id="execute_from_psram_requires_psram_s3_config", + ), + pytest.param( + { + "variant": "esp32p4", + "framework": { + "type": "esp-idf", + "advanced": {"execute_from_psram": True}, + }, + }, + r"'execute_from_psram' requires PSRAM to be configured @ data\['framework'\]\['advanced'\]\['execute_from_psram'\]", + id="execute_from_psram_requires_psram_p4_config", ), pytest.param( { @@ -108,3 +123,39 @@ def test_esp32_configuration_errors( with pytest.raises(cv.Invalid, match=error_match): FINAL_VALIDATE_SCHEMA(CONFIG_SCHEMA(config)) + + +def test_execute_from_psram_s3_sdkconfig( + generate_main: Callable[[str | Path], str], + component_config_path: Callable[[str], Path], +) -> None: + """Test that execute_from_psram on ESP32-S3 sets the correct sdkconfig options.""" + generate_main(component_config_path("execute_from_psram_s3.yaml")) + sdkconfig = CORE.data[KEY_ESP32][KEY_SDKCONFIG_OPTIONS] + assert sdkconfig.get("CONFIG_SPIRAM_FETCH_INSTRUCTIONS") is True + assert sdkconfig.get("CONFIG_SPIRAM_RODATA") is True + assert "CONFIG_SPIRAM_XIP_FROM_PSRAM" not in sdkconfig + + +def test_execute_from_psram_p4_sdkconfig( + generate_main: Callable[[str | Path], str], + component_config_path: Callable[[str], Path], +) -> None: + """Test that execute_from_psram on ESP32-P4 sets the correct sdkconfig options.""" + generate_main(component_config_path("execute_from_psram_p4.yaml")) + sdkconfig = CORE.data[KEY_ESP32][KEY_SDKCONFIG_OPTIONS] + assert sdkconfig.get("CONFIG_SPIRAM_XIP_FROM_PSRAM") is True + assert "CONFIG_SPIRAM_FETCH_INSTRUCTIONS" not in sdkconfig + assert "CONFIG_SPIRAM_RODATA" not in sdkconfig + + +def test_execute_from_psram_disabled_sdkconfig( + generate_main: Callable[[str | Path], str], + component_config_path: Callable[[str], Path], +) -> None: + """Test that without execute_from_psram, no XIP sdkconfig options are set.""" + generate_main(component_config_path("execute_from_psram_disabled.yaml")) + sdkconfig = CORE.data[KEY_ESP32][KEY_SDKCONFIG_OPTIONS] + assert "CONFIG_SPIRAM_FETCH_INSTRUCTIONS" not in sdkconfig + assert "CONFIG_SPIRAM_RODATA" not in sdkconfig + assert "CONFIG_SPIRAM_XIP_FROM_PSRAM" not in sdkconfig