Merge remote-tracking branch 'upstream/move-millis64-impl' into integration

This commit is contained in:
J. Nick Koston
2026-02-27 11:41:19 -10:00
22 changed files with 514 additions and 264 deletions
+15 -27
View File
@@ -1,5 +1,12 @@
const fs = require('fs');
const { DOCS_PR_PATTERNS } = require('./constants');
const {
COMPONENT_REGEX,
detectComponents,
hasCoreChanges,
hasDashboardChanges,
hasGitHubActionsChanges,
} = require('../detect-tags');
// Strategy: Merge branch detection
async function detectMergeBranch(context) {
@@ -20,15 +27,13 @@ async function detectMergeBranch(context) {
// Strategy: Component and platform labeling
async function detectComponentPlatforms(changedFiles, apiData) {
const labels = new Set();
const componentRegex = /^esphome\/components\/([^\/]+)\//;
const targetPlatformRegex = new RegExp(`^esphome\/components\/(${apiData.targetPlatforms.join('|')})/`);
for (const file of changedFiles) {
const componentMatch = file.match(componentRegex);
if (componentMatch) {
labels.add(`component: ${componentMatch[1]}`);
}
for (const comp of detectComponents(changedFiles)) {
labels.add(`component: ${comp}`);
}
for (const file of changedFiles) {
const platformMatch = file.match(targetPlatformRegex);
if (platformMatch) {
labels.add(`platform: ${platformMatch[1]}`);
@@ -90,15 +95,9 @@ async function detectNewPlatforms(prFiles, apiData) {
// Strategy: Core files detection
async function detectCoreChanges(changedFiles) {
const labels = new Set();
const coreFiles = changedFiles.filter(file =>
file.startsWith('esphome/core/') ||
(file.startsWith('esphome/') && file.split('/').length === 2)
);
if (coreFiles.length > 0) {
if (hasCoreChanges(changedFiles)) {
labels.add('core');
}
return labels;
}
@@ -131,29 +130,18 @@ async function detectPRSize(prFiles, totalAdditions, totalDeletions, totalChange
// Strategy: Dashboard changes
async function detectDashboardChanges(changedFiles) {
const labels = new Set();
const dashboardFiles = changedFiles.filter(file =>
file.startsWith('esphome/dashboard/') ||
file.startsWith('esphome/components/dashboard_import/')
);
if (dashboardFiles.length > 0) {
if (hasDashboardChanges(changedFiles)) {
labels.add('dashboard');
}
return labels;
}
// Strategy: GitHub Actions changes
async function detectGitHubActionsChanges(changedFiles) {
const labels = new Set();
const githubActionsFiles = changedFiles.filter(file =>
file.startsWith('.github/workflows/')
);
if (githubActionsFiles.length > 0) {
if (hasGitHubActionsChanges(changedFiles)) {
labels.add('github-actions');
}
return labels;
}
@@ -259,7 +247,7 @@ async function detectDeprecatedComponents(github, context, changedFiles) {
const { owner, repo } = context.repo;
// Compile regex once for better performance
const componentFileRegex = /^esphome\/components\/([^\/]+)\//;
const componentFileRegex = COMPONENT_REGEX;
// Get files that are modified or added in components directory
const componentFiles = changedFiles.filter(file => componentFileRegex.test(file));
+66
View File
@@ -0,0 +1,66 @@
/**
* Shared tag detection from changed file paths.
* Used by pr-title-check and auto-label-pr workflows.
*/
const COMPONENT_REGEX = /^esphome\/components\/([^\/]+)\//;
/**
* Detect component names from changed files.
* @param {string[]} changedFiles - List of changed file paths
* @returns {Set<string>} Set of component names
*/
function detectComponents(changedFiles) {
const components = new Set();
for (const file of changedFiles) {
const match = file.match(COMPONENT_REGEX);
if (match) {
components.add(match[1]);
}
}
return components;
}
/**
* Detect if core files were changed.
* Core files are in esphome/core/ or top-level esphome/ directory.
* @param {string[]} changedFiles - List of changed file paths
* @returns {boolean}
*/
function hasCoreChanges(changedFiles) {
return changedFiles.some(file =>
file.startsWith('esphome/core/') ||
(file.startsWith('esphome/') && file.split('/').length === 2)
);
}
/**
* Detect if dashboard files were changed.
* @param {string[]} changedFiles - List of changed file paths
* @returns {boolean}
*/
function hasDashboardChanges(changedFiles) {
return changedFiles.some(file =>
file.startsWith('esphome/dashboard/') ||
file.startsWith('esphome/components/dashboard_import/')
);
}
/**
* Detect if GitHub Actions files were changed.
* @param {string[]} changedFiles - List of changed file paths
* @returns {boolean}
*/
function hasGitHubActionsChanges(changedFiles) {
return changedFiles.some(file =>
file.startsWith('.github/workflows/')
);
}
module.exports = {
COMPONENT_REGEX,
detectComponents,
hasCoreChanges,
hasDashboardChanges,
hasGitHubActionsChanges,
};
+2 -2
View File
@@ -945,13 +945,13 @@ jobs:
python-version: ${{ env.DEFAULT_PYTHON }}
cache-key: ${{ needs.common.outputs.cache-key }}
- name: Download target analysis JSON
uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0
uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0
with:
name: memory-analysis-target
path: ./memory-analysis
continue-on-error: true
- name: Download PR analysis JSON
uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0
uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0
with:
name: memory-analysis-pr
path: ./memory-analysis
+76
View File
@@ -0,0 +1,76 @@
name: PR Title Check
on:
pull_request:
types: [opened, edited, synchronize, reopened]
permissions:
contents: read
pull-requests: read
jobs:
check:
name: Validate PR title
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0
with:
script: |
const {
detectComponents,
hasCoreChanges,
hasDashboardChanges,
hasGitHubActionsChanges,
} = require('./.github/scripts/detect-tags.js');
const title = context.payload.pull_request.title;
// Block titles starting with "word:" or "word(scope):" patterns
const commitStylePattern = /^\w+(\(.*?\))?[!]?\s*:/;
if (commitStylePattern.test(title)) {
core.setFailed(
`PR title should not start with a "prefix:" style format.\n` +
`Please use the format: [component] Brief description\n` +
`Example: [pn532] Add health checking and auto-reset`
);
return;
}
// Get changed files to detect tags
const files = await github.paginate(github.rest.pulls.listFiles, {
owner: context.repo.owner,
repo: context.repo.repo,
pull_number: context.issue.number,
});
const filenames = files.map(f => f.filename);
// Detect tags from changed files using shared logic
const tags = new Set();
for (const comp of detectComponents(filenames)) {
tags.add(comp);
}
if (hasCoreChanges(filenames)) tags.add('core');
if (hasDashboardChanges(filenames)) tags.add('dashboard');
if (hasGitHubActionsChanges(filenames)) tags.add('ci');
if (tags.size === 0) {
return;
}
// Check title starts with [tag] prefix
const bracketPattern = /^\[\w+\]/;
if (!bracketPattern.test(title)) {
const suggestion = [...tags].map(c => `[${c}]`).join('');
// Skip if the suggested prefix would be too long for a readable title
if (suggestion.length > 40) {
return;
}
core.setFailed(
`PR modifies: ${[...tags].join(', ')}\n` +
`Title must start with a [tag] prefix.\n` +
`Suggested: ${suggestion} <description>`
);
}
+1 -1
View File
@@ -171,7 +171,7 @@ jobs:
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- name: Download digests
uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0
uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0
with:
pattern: digests-*
path: /tmp/digests
+10 -4
View File
@@ -890,10 +890,10 @@ def final_validate(config):
)
)
if advanced[CONF_EXECUTE_FROM_PSRAM]:
if config[CONF_VARIANT] != VARIANT_ESP32S3:
if config[CONF_VARIANT] not in {VARIANT_ESP32S3, VARIANT_ESP32P4}:
errs.append(
cv.Invalid(
f"'{CONF_EXECUTE_FROM_PSRAM}' is only supported on {VARIANT_ESP32S3} variant",
f"'{CONF_EXECUTE_FROM_PSRAM}' is not available on this esp32 variant",
path=[CONF_FRAMEWORK, CONF_ADVANCED, CONF_EXECUTE_FROM_PSRAM],
)
)
@@ -1436,6 +1436,7 @@ async def to_code(config):
cg.set_cpp_standard("gnu++20")
cg.add_build_flag("-DUSE_ESP32")
cg.add_define("USE_NATIVE_64BIT_TIME")
cg.add_build_flag("-Wl,-z,noexecstack")
cg.add_define("ESPHOME_BOARD", config[CONF_BOARD])
variant = config[CONF_VARIANT]
@@ -1627,8 +1628,13 @@ async def to_code(config):
_configure_lwip_max_sockets(conf)
if advanced[CONF_EXECUTE_FROM_PSRAM]:
add_idf_sdkconfig_option("CONFIG_SPIRAM_FETCH_INSTRUCTIONS", True)
add_idf_sdkconfig_option("CONFIG_SPIRAM_RODATA", True)
if variant == VARIANT_ESP32S3:
add_idf_sdkconfig_option("CONFIG_SPIRAM_FETCH_INSTRUCTIONS", True)
add_idf_sdkconfig_option("CONFIG_SPIRAM_RODATA", True)
elif variant == VARIANT_ESP32P4:
add_idf_sdkconfig_option("CONFIG_SPIRAM_XIP_FROM_PSRAM", True)
else:
raise ValueError("Unhandled ESP32 variant")
# Apply LWIP core locking for better socket performance
# This is already enabled by default in Arduino framework, where it provides
+2 -2
View File
@@ -3,7 +3,7 @@
#include "core.h"
#include "esphome/core/defines.h"
#include "esphome/core/hal.h"
#include "esphome/core/application.h"
#include "esphome/core/time_64.h"
#include "esphome/core/helpers.h"
#include "preferences.h"
#include <Arduino.h>
@@ -17,7 +17,7 @@ namespace esphome {
void HOT yield() { ::yield(); }
uint32_t IRAM_ATTR HOT millis() { return ::millis(); }
uint64_t millis_64() { return App.scheduler.millis_64_impl_(::millis()); }
uint64_t millis_64() { return Millis64Impl::compute(::millis()); }
void HOT delay(uint32_t ms) { ::delay(ms); }
uint32_t IRAM_ATTR HOT micros() { return ::micros(); }
void IRAM_ATTR HOT delayMicroseconds(uint32_t us) { delay_microseconds_safe(us); }
+1
View File
@@ -41,6 +41,7 @@ CONFIG_SCHEMA = cv.All(
async def to_code(config):
cg.add_build_flag("-DUSE_HOST")
cg.add_define("USE_NATIVE_64BIT_TIME")
cg.add_define("USE_ESPHOME_HOST_MAC_ADDRESS", config[CONF_MAC_ADDRESS].parts)
cg.add_build_flag("-std=gnu++20")
cg.add_define("ESPHOME_BOARD", "host")
+2 -2
View File
@@ -3,7 +3,7 @@
#include "core.h"
#include "esphome/core/defines.h"
#include "esphome/core/hal.h"
#include "esphome/core/application.h"
#include "esphome/core/time_64.h"
#include "esphome/core/helpers.h"
#include "preferences.h"
@@ -14,7 +14,7 @@ namespace esphome {
void HOT yield() { ::yield(); }
uint32_t IRAM_ATTR HOT millis() { return ::millis(); }
uint64_t millis_64() { return App.scheduler.millis_64_impl_(::millis()); }
uint64_t millis_64() { return Millis64Impl::compute(::millis()); }
uint32_t IRAM_ATTR HOT micros() { return ::micros(); }
void HOT delay(uint32_t ms) { ::delay(ms); }
void IRAM_ATTR HOT delayMicroseconds(uint32_t us) { ::delayMicroseconds(us); }
+1
View File
@@ -169,6 +169,7 @@ async def to_code(config):
cg.add_platformio_option("lib_compat_mode", "strict")
cg.add_platformio_option("board", config[CONF_BOARD])
cg.add_build_flag("-DUSE_RP2040")
cg.add_define("USE_NATIVE_64BIT_TIME")
cg.set_cpp_standard("gnu++20")
cg.add_define("ESPHOME_BOARD", config[CONF_BOARD])
cg.add_define("ESPHOME_VARIANT", "RP2040")
+1
View File
@@ -112,6 +112,7 @@ def add_extra_script(stage: str, filename: str, path: Path) -> None:
def zephyr_to_code(config):
cg.add_build_flag("-DUSE_ZEPHYR")
cg.add_define("USE_NATIVE_64BIT_TIME")
cg.set_cpp_standard("gnu++20")
# build is done by west so bypass board checking in platformio
cg.add_platformio_option("boards_dir", CORE.relative_build_path("boards"))
+6
View File
@@ -650,6 +650,12 @@ FILTER_SOURCE_FILES = filter_source_files_from_platform(
PlatformFramework.ESP32_ARDUINO,
PlatformFramework.ESP32_IDF,
},
"time_64.cpp": {
PlatformFramework.ESP8266_ARDUINO,
PlatformFramework.BK72XX_ARDUINO,
PlatformFramework.RTL87XX_ARDUINO,
PlatformFramework.LN882X_ARDUINO,
},
# Note: lock_free_queue.h and event_pool.h are header files and don't need to be filtered
# as they are only included when needed by the preprocessor
}
+5
View File
@@ -181,6 +181,11 @@
#define USE_I2S_LEGACY
#endif
// Platforms with native 64-bit time sources (no rollover tracking needed)
#if defined(USE_ESP32) || defined(USE_HOST) || defined(USE_ZEPHYR) || defined(USE_RP2040)
#define USE_NATIVE_64BIT_TIME
#endif
// ESP32-specific feature flags
#ifdef USE_ESP32
#define USE_MQTT_IDF_ENQUEUE
+3 -179
View File
@@ -9,7 +9,6 @@
#include <algorithm>
#include <cinttypes>
#include <cstring>
#include <limits>
namespace esphome {
@@ -28,10 +27,6 @@ static constexpr size_t MAX_POOL_SIZE = 5;
// Set to 5 to match the pool size - when we have as many cancelled items as our
// pool can hold, it's time to clean up and recycle them.
static constexpr uint32_t MAX_LOGICALLY_DELETED_ITEMS = 5;
#if !defined(USE_ESP32) && !defined(USE_HOST) && !defined(USE_ZEPHYR) && !defined(USE_RP2040)
// Half the 32-bit range - used to detect rollovers vs normal time progression
static constexpr uint32_t HALF_MAX_UINT32 = std::numeric_limits<uint32_t>::max() / 2;
#endif
// max delay to start an interval sequence
static constexpr uint32_t MAX_INTERVAL_DELAY = 5000;
@@ -152,9 +147,6 @@ void HOT Scheduler::set_timer_common_(Component *component, SchedulerItem::Type
return;
}
// Get fresh 64-bit timestamp BEFORE taking lock
const uint64_t now_64 = millis_64();
// Take lock early to protect scheduler_item_pool_ access
LockGuard guard{this->lock_};
@@ -181,6 +173,9 @@ void HOT Scheduler::set_timer_common_(Component *component, SchedulerItem::Type
} else
#endif /* not ESPHOME_THREAD_SINGLE */
{
// Only non-defer items need a timestamp for scheduling
const uint64_t now_64 = millis_64();
// Type-specific setup
if (type == SchedulerItem::INTERVAL) {
item->interval = delay;
@@ -475,19 +470,8 @@ void HOT Scheduler::call(uint32_t now) {
if (now_64 - last_print > 2000) {
last_print = now_64;
std::vector<SchedulerItemPtr> old_items;
#if !defined(USE_ESP32) && !defined(USE_HOST) && !defined(USE_ZEPHYR) && !defined(USE_RP2040) && \
defined(ESPHOME_THREAD_MULTI_ATOMICS)
const auto last_dbg = this->last_millis_.load(std::memory_order_relaxed);
const auto major_dbg = this->millis_major_.load(std::memory_order_relaxed);
ESP_LOGD(TAG, "Items: count=%zu, pool=%zu, now=%" PRIu64 " (%" PRIu16 ", %" PRIu32 ")", this->items_.size(),
this->scheduler_item_pool_.size(), now_64, major_dbg, last_dbg);
#elif !defined(USE_ESP32) && !defined(USE_HOST) && !defined(USE_ZEPHYR) && !defined(USE_RP2040)
ESP_LOGD(TAG, "Items: count=%zu, pool=%zu, now=%" PRIu64 " (%" PRIu16 ", %" PRIu32 ")", this->items_.size(),
this->scheduler_item_pool_.size(), now_64, this->millis_major_, this->last_millis_);
#else
ESP_LOGD(TAG, "Items: count=%zu, pool=%zu, now=%" PRIu64, this->items_.size(), this->scheduler_item_pool_.size(),
now_64);
#endif
// Cleanup before debug output
this->cleanup_();
while (!this->items_.empty()) {
@@ -715,166 +699,6 @@ bool HOT Scheduler::cancel_item_locked_(Component *component, NameType name_type
return total_cancelled > 0;
}
#if !defined(USE_ESP32) && !defined(USE_HOST) && !defined(USE_ZEPHYR) && !defined(USE_RP2040)
uint64_t Scheduler::millis_64_impl_(uint32_t now) {
// THREAD SAFETY NOTE:
// This function has three implementations, based on the precompiler flags
// - ESPHOME_THREAD_SINGLE - Runs on single-threaded platforms (ESP8266, RP2040, etc.)
// - ESPHOME_THREAD_MULTI_NO_ATOMICS - Runs on multi-threaded platforms without atomics (LibreTiny BK72xx)
// - ESPHOME_THREAD_MULTI_ATOMICS - Runs on multi-threaded platforms with atomics (ESP32, HOST, LibreTiny
// RTL87xx/LN882x, etc.)
//
// Make sure all changes are synchronized if you edit this function.
//
// IMPORTANT: Always pass fresh millis() values to this function. The implementation
// handles out-of-order timestamps between threads, but minimizing time differences
// helps maintain accuracy.
//
#ifdef ESPHOME_THREAD_SINGLE
// This is the single core implementation.
//
// Single-core platforms have no concurrency, so this is a simple implementation
// that just tracks 32-bit rollover (every 49.7 days) without any locking or atomics.
uint16_t major = this->millis_major_;
uint32_t last = this->last_millis_;
// Check for rollover
if (now < last && (last - now) > HALF_MAX_UINT32) {
this->millis_major_++;
major++;
this->last_millis_ = now;
#ifdef ESPHOME_DEBUG_SCHEDULER
ESP_LOGD(TAG, "Detected true 32-bit rollover at %" PRIu32 "ms (was %" PRIu32 ")", now, last);
#endif /* ESPHOME_DEBUG_SCHEDULER */
} else if (now > last) {
// Only update if time moved forward
this->last_millis_ = now;
}
// Combine major (high 32 bits) and now (low 32 bits) into 64-bit time
return now + (static_cast<uint64_t>(major) << 32);
#elif defined(ESPHOME_THREAD_MULTI_NO_ATOMICS)
// This is the multi core no atomics implementation.
//
// Without atomics, this implementation uses locks more aggressively:
// 1. Always locks when near the rollover boundary (within 10 seconds)
// 2. Always locks when detecting a large backwards jump
// 3. Updates without lock in normal forward progression (accepting minor races)
// This is less efficient but necessary without atomic operations.
uint16_t major = this->millis_major_;
uint32_t last = this->last_millis_;
// Define a safe window around the rollover point (10 seconds)
// This covers any reasonable scheduler delays or thread preemption
static constexpr uint32_t ROLLOVER_WINDOW = 10000; // 10 seconds in milliseconds
// Check if we're near the rollover boundary (close to std::numeric_limits<uint32_t>::max() or just past 0)
bool near_rollover = (last > (std::numeric_limits<uint32_t>::max() - ROLLOVER_WINDOW)) || (now < ROLLOVER_WINDOW);
if (near_rollover || (now < last && (last - now) > HALF_MAX_UINT32)) {
// Near rollover or detected a rollover - need lock for safety
LockGuard guard{this->lock_};
// Re-read with lock held
last = this->last_millis_;
if (now < last && (last - now) > HALF_MAX_UINT32) {
// True rollover detected (happens every ~49.7 days)
this->millis_major_++;
major++;
#ifdef ESPHOME_DEBUG_SCHEDULER
ESP_LOGD(TAG, "Detected true 32-bit rollover at %" PRIu32 "ms (was %" PRIu32 ")", now, last);
#endif /* ESPHOME_DEBUG_SCHEDULER */
}
// Update last_millis_ while holding lock
this->last_millis_ = now;
} else if (now > last) {
// Normal case: Not near rollover and time moved forward
// Update without lock. While this may cause minor races (microseconds of
// backwards time movement), they're acceptable because:
// 1. The scheduler operates at millisecond resolution, not microsecond
// 2. We've already prevented the critical rollover race condition
// 3. Any backwards movement is orders of magnitude smaller than scheduler delays
this->last_millis_ = now;
}
// If now <= last and we're not near rollover, don't update
// This minimizes backwards time movement
// Combine major (high 32 bits) and now (low 32 bits) into 64-bit time
return now + (static_cast<uint64_t>(major) << 32);
#elif defined(ESPHOME_THREAD_MULTI_ATOMICS)
// This is the multi core with atomics implementation.
//
// Uses atomic operations with acquire/release semantics to ensure coherent
// reads of millis_major_ and last_millis_ across cores. Features:
// 1. Epoch-coherency retry loop to handle concurrent updates
// 2. Lock only taken for actual rollover detection and update
// 3. Lock-free CAS updates for normal forward time progression
// 4. Memory ordering ensures cores see consistent time values
for (;;) {
uint16_t major = this->millis_major_.load(std::memory_order_acquire);
/*
* Acquire so that if we later decide **not** to take the lock we still
* observe a `millis_major_` value coherent with the loaded `last_millis_`.
* The acquire load ensures any later read of `millis_major_` sees its
* corresponding increment.
*/
uint32_t last = this->last_millis_.load(std::memory_order_acquire);
// If we might be near a rollover (large backwards jump), take the lock for the entire operation
// This ensures rollover detection and last_millis_ update are atomic together
if (now < last && (last - now) > HALF_MAX_UINT32) {
// Potential rollover - need lock for atomic rollover detection + update
LockGuard guard{this->lock_};
// Re-read with lock held; mutex already provides ordering
last = this->last_millis_.load(std::memory_order_relaxed);
if (now < last && (last - now) > HALF_MAX_UINT32) {
// True rollover detected (happens every ~49.7 days)
this->millis_major_.fetch_add(1, std::memory_order_relaxed);
major++;
#ifdef ESPHOME_DEBUG_SCHEDULER
ESP_LOGD(TAG, "Detected true 32-bit rollover at %" PRIu32 "ms (was %" PRIu32 ")", now, last);
#endif /* ESPHOME_DEBUG_SCHEDULER */
}
/*
* Update last_millis_ while holding the lock to prevent races
* Publish the new low-word *after* bumping `millis_major_` (done above)
* so readers never see a mismatched pair.
*/
this->last_millis_.store(now, std::memory_order_release);
} else {
// Normal case: Try lock-free update, but only allow forward movement within same epoch
// This prevents accidentally moving backwards across a rollover boundary
while (now > last && (now - last) < HALF_MAX_UINT32) {
if (this->last_millis_.compare_exchange_weak(last, now,
std::memory_order_release, // success
std::memory_order_relaxed)) { // failure
break;
}
// CAS failure means no data was published; relaxed is fine
// last is automatically updated by compare_exchange_weak if it fails
}
}
uint16_t major_end = this->millis_major_.load(std::memory_order_relaxed);
if (major_end == major)
return now + (static_cast<uint64_t>(major) << 32);
}
// Unreachable - the loop always returns when major_end == major
__builtin_unreachable();
#else
#error \
"No platform threading model defined. One of ESPHOME_THREAD_SINGLE, ESPHOME_THREAD_MULTI_NO_ATOMICS, or ESPHOME_THREAD_MULTI_ATOMICS must be defined."
#endif
}
#endif // !USE_ESP32 && !USE_HOST && !USE_ZEPHYR && !USE_RP2040
bool HOT Scheduler::SchedulerItem::cmp(const SchedulerItemPtr &a, const SchedulerItemPtr &b) {
// High bits are almost always equal (change only on 32-bit rollover ~49 days)
// Optimize for common case: check low bits first when high bits are equal
+5 -44
View File
@@ -12,6 +12,7 @@
#include "esphome/core/component.h"
#include "esphome/core/hal.h"
#include "esphome/core/helpers.h"
#include "esphome/core/time_64.h"
namespace esphome {
@@ -283,23 +284,16 @@ class Scheduler {
bool cancel_retry_(Component *component, NameType name_type, const char *static_name, uint32_t hash_or_id);
// Extend a 32-bit millis() value to 64-bit. Use when the caller already has a fresh now.
// On ESP32, ignores now and uses esp_timer_get_time() directly (native 64-bit).
// On non-ESP32, extends now to 64-bit using rollover tracking.
// On platforms with native 64-bit time, ignores now and uses millis_64() directly.
// On other platforms, extends now to 64-bit using rollover tracking.
uint64_t millis_64_from_(uint32_t now) {
#if defined(USE_ESP32) || defined(USE_HOST) || defined(USE_ZEPHYR) || defined(USE_RP2040)
#ifdef USE_NATIVE_64BIT_TIME
(void) now;
return millis_64();
#else
return this->millis_64_impl_(now);
return Millis64Impl::compute(now);
#endif
}
#if !defined(USE_ESP32) && !defined(USE_HOST) && !defined(USE_ZEPHYR) && !defined(USE_RP2040)
// On platforms without native 64-bit time, millis_64() HAL function delegates to this
// method which tracks 32-bit millis() rollover using millis_major_ and last_millis_.
friend uint64_t millis_64();
uint64_t millis_64_impl_(uint32_t now);
#endif
// Cleanup logically deleted items from the scheduler
// Returns the number of items remaining after cleanup
// IMPORTANT: This method should only be called from the main thread (loop task).
@@ -565,39 +559,6 @@ class Scheduler {
// can stall the entire system, causing timing issues and dropped events for any components that need
// to synchronize between tasks (see https://github.com/esphome/backlog/issues/52)
std::vector<SchedulerItemPtr> scheduler_item_pool_;
#if !defined(USE_ESP32) && !defined(USE_HOST) && !defined(USE_ZEPHYR) && !defined(USE_RP2040)
// On platforms with native 64-bit time (ESP32, Host, Zephyr, RP2040), no rollover tracking needed.
// On other platforms, these fields track 32-bit millis() rollover for millis_64_impl_().
#ifdef ESPHOME_THREAD_MULTI_ATOMICS
/*
* Multi-threaded platforms with atomic support: last_millis_ needs atomic for lock-free updates
*
* MEMORY-ORDERING NOTE
* --------------------
* `last_millis_` and `millis_major_` form a single 64-bit timestamp split in half.
* Writers publish `last_millis_` with memory_order_release and readers use
* memory_order_acquire. This ensures that once a reader sees the new low word,
* it also observes the corresponding increment of `millis_major_`.
*/
std::atomic<uint32_t> last_millis_{0};
#else /* not ESPHOME_THREAD_MULTI_ATOMICS */
// Platforms without atomic support or single-threaded platforms
uint32_t last_millis_{0};
#endif /* else ESPHOME_THREAD_MULTI_ATOMICS */
/*
* Upper 16 bits of the 64-bit millis counter. Incremented only while holding
* `lock_`; read concurrently. Atomic (relaxed) avoids a formal data race.
* Ordering relative to `last_millis_` is provided by its release store and the
* corresponding acquire loads.
*/
#ifdef ESPHOME_THREAD_MULTI_ATOMICS
std::atomic<uint16_t> millis_major_{0};
#else /* not ESPHOME_THREAD_MULTI_ATOMICS */
uint16_t millis_major_{0};
#endif /* else ESPHOME_THREAD_MULTI_ATOMICS */
#endif /* !USE_ESP32 && !USE_HOST && !USE_ZEPHYR && !USE_RP2040 */
};
} // namespace esphome
+207
View File
@@ -0,0 +1,207 @@
#include "esphome/core/defines.h"
#ifndef USE_NATIVE_64BIT_TIME
#include "time_64.h"
#include "esphome/core/helpers.h"
#ifdef ESPHOME_DEBUG_SCHEDULER
#include "esphome/core/log.h"
#include <cinttypes>
#endif
#ifdef ESPHOME_THREAD_MULTI_ATOMICS
#include <atomic>
#endif
#include <limits>
namespace esphome {
#ifdef ESPHOME_DEBUG_SCHEDULER
static const char *const TAG = "time_64";
#endif
uint64_t Millis64Impl::compute(uint32_t now) {
// Half the 32-bit range - used to detect rollovers vs normal time progression
static constexpr uint32_t HALF_MAX_UINT32 = std::numeric_limits<uint32_t>::max() / 2;
// State variables for rollover tracking - static to persist across calls
#ifdef ESPHOME_THREAD_MULTI_ATOMICS
// Mutex for rollover serialization (taken only every ~49.7 days).
// A spinlock would be smaller (~1 byte vs ~80-100 bytes) but is unsafe on
// preemptive single-core RTOS platforms due to priority inversion: a high-priority
// task spinning would prevent the lock holder from running to release it.
static Mutex lock;
/*
* Multi-threaded platforms with atomic support: last_millis needs atomic for lock-free updates.
* Writers publish last_millis with memory_order_release and readers use memory_order_acquire.
* This ensures that once a reader sees the new low word, it also observes the corresponding
* increment of millis_major.
*/
static std::atomic<uint32_t> last_millis{0};
/*
* Upper 16 bits of the 64-bit millis counter. Incremented only while holding lock;
* read concurrently. Atomic (relaxed) avoids a formal data race. Ordering relative
* to last_millis is provided by its release store and the corresponding acquire loads.
*/
static std::atomic<uint16_t> millis_major{0};
#elif !defined(ESPHOME_THREAD_SINGLE) /* ESPHOME_THREAD_MULTI_NO_ATOMICS */
static Mutex lock;
static uint32_t last_millis{0};
static uint16_t millis_major{0};
#else /* ESPHOME_THREAD_SINGLE */
static uint32_t last_millis{0};
static uint16_t millis_major{0};
#endif
// THREAD SAFETY NOTE:
// This function has three implementations, based on the precompiler flags
// - ESPHOME_THREAD_SINGLE - Runs on single-threaded platforms (ESP8266, etc.)
// - ESPHOME_THREAD_MULTI_NO_ATOMICS - Runs on multi-threaded platforms without atomics (LibreTiny BK72xx)
// - ESPHOME_THREAD_MULTI_ATOMICS - Runs on multi-threaded platforms with atomics (LibreTiny RTL87xx/LN882x, etc.)
//
// Make sure all changes are synchronized if you edit this function.
//
// IMPORTANT: Always pass fresh millis() values to this function. The implementation
// handles out-of-order timestamps between threads, but minimizing time differences
// helps maintain accuracy.
#ifdef ESPHOME_THREAD_SINGLE
// Single-core platforms have no concurrency, so this is a simple implementation
// that just tracks 32-bit rollover (every 49.7 days) without any locking or atomics.
uint16_t major = millis_major;
uint32_t last = last_millis;
// Check for rollover
if (now < last && (last - now) > HALF_MAX_UINT32) {
millis_major++;
major++;
last_millis = now;
#ifdef ESPHOME_DEBUG_SCHEDULER
ESP_LOGD(TAG, "Detected true 32-bit rollover at %" PRIu32 "ms (was %" PRIu32 ")", now, last);
#endif /* ESPHOME_DEBUG_SCHEDULER */
} else if (now > last) {
// Only update if time moved forward
last_millis = now;
}
// Combine major (high 32 bits) and now (low 32 bits) into 64-bit time
return now + (static_cast<uint64_t>(major) << 32);
#elif defined(ESPHOME_THREAD_MULTI_NO_ATOMICS)
// Without atomics, this implementation uses locks more aggressively:
// 1. Always locks when near the rollover boundary (within 10 seconds)
// 2. Always locks when detecting a large backwards jump
// 3. Updates without lock in normal forward progression (accepting minor races)
// This is less efficient but necessary without atomic operations.
uint16_t major = millis_major;
uint32_t last = last_millis;
// Define a safe window around the rollover point (10 seconds)
// This covers any reasonable scheduler delays or thread preemption
static constexpr uint32_t ROLLOVER_WINDOW = 10000; // 10 seconds in milliseconds
// Check if we're near the rollover boundary (close to std::numeric_limits<uint32_t>::max() or just past 0)
bool near_rollover = (last > (std::numeric_limits<uint32_t>::max() - ROLLOVER_WINDOW)) || (now < ROLLOVER_WINDOW);
if (near_rollover || (now < last && (last - now) > HALF_MAX_UINT32)) {
// Near rollover or detected a rollover - need lock for safety
LockGuard guard{lock};
// Re-read with lock held
last = last_millis;
if (now < last && (last - now) > HALF_MAX_UINT32) {
// True rollover detected (happens every ~49.7 days)
millis_major++;
major++;
#ifdef ESPHOME_DEBUG_SCHEDULER
ESP_LOGD(TAG, "Detected true 32-bit rollover at %" PRIu32 "ms (was %" PRIu32 ")", now, last);
#endif /* ESPHOME_DEBUG_SCHEDULER */
}
// Update last_millis while holding lock
last_millis = now;
} else if (now > last) {
// Normal case: Not near rollover and time moved forward
// Update without lock. While this may cause minor races (microseconds of
// backwards time movement), they're acceptable because:
// 1. The scheduler operates at millisecond resolution, not microsecond
// 2. We've already prevented the critical rollover race condition
// 3. Any backwards movement is orders of magnitude smaller than scheduler delays
last_millis = now;
}
// If now <= last and we're not near rollover, don't update
// This minimizes backwards time movement
// Combine major (high 32 bits) and now (low 32 bits) into 64-bit time
return now + (static_cast<uint64_t>(major) << 32);
#elif defined(ESPHOME_THREAD_MULTI_ATOMICS)
// Uses atomic operations with acquire/release semantics to ensure coherent
// reads of millis_major and last_millis across cores. Features:
// 1. Epoch-coherency retry loop to handle concurrent updates
// 2. Lock only taken for actual rollover detection and update
// 3. Lock-free CAS updates for normal forward time progression
// 4. Memory ordering ensures cores see consistent time values
for (;;) {
uint16_t major = millis_major.load(std::memory_order_acquire);
/*
* Acquire so that if we later decide **not** to take the lock we still
* observe a millis_major value coherent with the loaded last_millis.
* The acquire load ensures any later read of millis_major sees its
* corresponding increment.
*/
uint32_t last = last_millis.load(std::memory_order_acquire);
// If we might be near a rollover (large backwards jump), take the lock
// This ensures rollover detection and last_millis update are atomic together
if (now < last && (last - now) > HALF_MAX_UINT32) {
// Potential rollover - need lock for atomic rollover detection + update
LockGuard guard{lock};
// Re-read with lock held; mutex already provides ordering
last = last_millis.load(std::memory_order_relaxed);
if (now < last && (last - now) > HALF_MAX_UINT32) {
// True rollover detected (happens every ~49.7 days)
millis_major.fetch_add(1, std::memory_order_relaxed);
major++;
#ifdef ESPHOME_DEBUG_SCHEDULER
ESP_LOGD(TAG, "Detected true 32-bit rollover at %" PRIu32 "ms (was %" PRIu32 ")", now, last);
#endif /* ESPHOME_DEBUG_SCHEDULER */
}
/*
* Update last_millis while holding the lock to prevent races.
* Publish the new low-word *after* bumping millis_major (done above)
* so readers never see a mismatched pair.
*/
last_millis.store(now, std::memory_order_release);
} else {
// Normal case: Try lock-free update, but only allow forward movement within same epoch
// This prevents accidentally moving backwards across a rollover boundary
while (now > last && (now - last) < HALF_MAX_UINT32) {
if (last_millis.compare_exchange_weak(last, now,
std::memory_order_release, // success
std::memory_order_relaxed)) { // failure
break;
}
// CAS failure means no data was published; relaxed is fine
// last is automatically updated by compare_exchange_weak if it fails
}
}
uint16_t major_end = millis_major.load(std::memory_order_relaxed);
if (major_end == major)
return now + (static_cast<uint64_t>(major) << 32);
}
// Unreachable - the loop always returns when major_end == major
__builtin_unreachable();
#else
#error \
"No platform threading model defined. One of ESPHOME_THREAD_SINGLE, ESPHOME_THREAD_MULTI_NO_ATOMICS, or ESPHOME_THREAD_MULTI_ATOMICS must be defined."
#endif
}
} // namespace esphome
#endif // !USE_NATIVE_64BIT_TIME
+24
View File
@@ -0,0 +1,24 @@
#pragma once
#include "esphome/core/defines.h"
#ifndef USE_NATIVE_64BIT_TIME
#include <cstdint>
namespace esphome {
class Scheduler;
/// Extends 32-bit millis() to 64-bit using rollover tracking.
/// Access restricted to platform HAL (millis_64()) and Scheduler.
/// All other code should call millis_64() from hal.h instead.
class Millis64Impl {
friend uint64_t millis_64();
friend class Scheduler;
static uint64_t compute(uint32_t now);
};
} // namespace esphome
#endif // !USE_NATIVE_64BIT_TIME
+1 -1
View File
@@ -1,6 +1,6 @@
pylint==4.0.5
flake8==7.3.0 # also change in .pre-commit-config.yaml when updating
ruff==0.15.3 # also change in .pre-commit-config.yaml when updating
ruff==0.15.4 # also change in .pre-commit-config.yaml when updating
pyupgrade==3.21.2 # also change in .pre-commit-config.yaml when updating
pre-commit
@@ -0,0 +1,10 @@
esphome:
name: test
esp32:
variant: esp32s3
framework:
type: esp-idf
psram:
mode: octal
@@ -0,0 +1,11 @@
esphome:
name: test
esp32:
variant: esp32p4
framework:
type: esp-idf
advanced:
execute_from_psram: true
psram:
@@ -0,0 +1,12 @@
esphome:
name: test
esp32:
variant: esp32s3
framework:
type: esp-idf
advanced:
execute_from_psram: true
psram:
mode: octal
+53 -2
View File
@@ -2,13 +2,17 @@
Test ESP32 configuration
"""
from collections.abc import Callable
from pathlib import Path
from typing import Any
import pytest
from esphome.components.esp32 import VARIANTS
from esphome.components.esp32.const import KEY_ESP32, KEY_SDKCONFIG_OPTIONS
import esphome.config_validation as cv
from esphome.const import CONF_ESPHOME, PlatformFramework
from esphome.core import CORE
from tests.component_tests.types import SetCoreConfigCallable
@@ -70,7 +74,7 @@ def test_esp32_config(
"advanced": {"execute_from_psram": True},
},
},
r"'execute_from_psram' is only supported on ESP32S3 variant @ data\['framework'\]\['advanced'\]\['execute_from_psram'\]",
r"'execute_from_psram' is not available on this esp32 variant @ data\['framework'\]\['advanced'\]\['execute_from_psram'\]",
id="execute_from_psram_invalid_for_variant_config",
),
pytest.param(
@@ -82,7 +86,18 @@ def test_esp32_config(
},
},
r"'execute_from_psram' requires PSRAM to be configured @ data\['framework'\]\['advanced'\]\['execute_from_psram'\]",
id="execute_from_psram_requires_psram_config",
id="execute_from_psram_requires_psram_s3_config",
),
pytest.param(
{
"variant": "esp32p4",
"framework": {
"type": "esp-idf",
"advanced": {"execute_from_psram": True},
},
},
r"'execute_from_psram' requires PSRAM to be configured @ data\['framework'\]\['advanced'\]\['execute_from_psram'\]",
id="execute_from_psram_requires_psram_p4_config",
),
pytest.param(
{
@@ -108,3 +123,39 @@ def test_esp32_configuration_errors(
with pytest.raises(cv.Invalid, match=error_match):
FINAL_VALIDATE_SCHEMA(CONFIG_SCHEMA(config))
def test_execute_from_psram_s3_sdkconfig(
generate_main: Callable[[str | Path], str],
component_config_path: Callable[[str], Path],
) -> None:
"""Test that execute_from_psram on ESP32-S3 sets the correct sdkconfig options."""
generate_main(component_config_path("execute_from_psram_s3.yaml"))
sdkconfig = CORE.data[KEY_ESP32][KEY_SDKCONFIG_OPTIONS]
assert sdkconfig.get("CONFIG_SPIRAM_FETCH_INSTRUCTIONS") is True
assert sdkconfig.get("CONFIG_SPIRAM_RODATA") is True
assert "CONFIG_SPIRAM_XIP_FROM_PSRAM" not in sdkconfig
def test_execute_from_psram_p4_sdkconfig(
generate_main: Callable[[str | Path], str],
component_config_path: Callable[[str], Path],
) -> None:
"""Test that execute_from_psram on ESP32-P4 sets the correct sdkconfig options."""
generate_main(component_config_path("execute_from_psram_p4.yaml"))
sdkconfig = CORE.data[KEY_ESP32][KEY_SDKCONFIG_OPTIONS]
assert sdkconfig.get("CONFIG_SPIRAM_XIP_FROM_PSRAM") is True
assert "CONFIG_SPIRAM_FETCH_INSTRUCTIONS" not in sdkconfig
assert "CONFIG_SPIRAM_RODATA" not in sdkconfig
def test_execute_from_psram_disabled_sdkconfig(
generate_main: Callable[[str | Path], str],
component_config_path: Callable[[str], Path],
) -> None:
"""Test that without execute_from_psram, no XIP sdkconfig options are set."""
generate_main(component_config_path("execute_from_psram_disabled.yaml"))
sdkconfig = CORE.data[KEY_ESP32][KEY_SDKCONFIG_OPTIONS]
assert "CONFIG_SPIRAM_FETCH_INSTRUCTIONS" not in sdkconfig
assert "CONFIG_SPIRAM_RODATA" not in sdkconfig
assert "CONFIG_SPIRAM_XIP_FROM_PSRAM" not in sdkconfig