From c470bbb698a969d99719ee8181be0b83e692ffa0 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 13 Oct 2026 10:21:30 -1000 Subject: [PATCH 1/4] [core] Let PROGMEM_STRING_TABLE hold more than 255 bytes --- esphome/core/progmem.h | 26 +++++++++--- .../core/test_progmem_string_table.cpp | 41 +++++++++++++++++++ 2 files changed, 62 insertions(+), 5 deletions(-) create mode 100644 tests/components/core/test_progmem_string_table.cpp diff --git a/esphome/core/progmem.h b/esphome/core/progmem.h index 54fc42dde16..64f3eee8aba 100644 --- a/esphome/core/progmem.h +++ b/esphome/core/progmem.h @@ -1,10 +1,12 @@ #pragma once +#include #include #include #include #include #include +#include #include "esphome/core/hal.h" // For PROGMEM definition @@ -78,6 +80,10 @@ template struct FixedString { template struct ProgmemStringTable { static constexpr size_t COUNT = sizeof...(Strs); static constexpr size_t BLOB_SIZE = (0 + ... + (Strs.size() + 1)); + /// Length of the longest string, for sizing a RAM copy on ESP8266 + static constexpr size_t MAX_LENGTH = std::max({Strs.size()...}); + /// Offsets are one byte while the blob fits in 255 bytes and two bytes beyond that. + using offset_t = std::conditional_t<(BLOB_SIZE <= UINT8_MAX), uint8_t, uint16_t>; /// Generate packed string blob at compile time static constexpr auto make_blob() { @@ -91,16 +97,25 @@ template struct ProgmemStringTable { return result; } - /// Generate offset table at compile time (uint8_t limits blob to 255 bytes) + /// Generate offset table at compile time static constexpr auto make_offsets() { static_assert(COUNT > 0, "PROGMEM_STRING_TABLE must contain at least one string"); static_assert(COUNT <= 255, "PROGMEM_STRING_TABLE supports at most 255 strings with uint8_t indices"); - static_assert(BLOB_SIZE <= 255, "PROGMEM_STRING_TABLE blob exceeds 255 bytes; use fewer/shorter strings"); - std::array result{}; + static_assert(BLOB_SIZE <= UINT16_MAX, "PROGMEM_STRING_TABLE blob exceeds 65535 bytes; use fewer/shorter strings"); + std::array result{}; size_t pos = 0, idx = 0; - ((result[idx++] = static_cast(pos), pos += Strs.size() + 1), ...); + ((result[idx++] = static_cast(pos), pos += Strs.size() + 1), ...); return result; } + + /// Read one offset from the table, which may live in flash + static offset_t read_offset(const offset_t *offset) { + if constexpr (sizeof(offset_t) == 1) { + return progmem_read_byte(offset); + } else { + return progmem_read_uint16(offset); + } + } }; // Forward declaration for LogString (defined in log.h) @@ -115,6 +130,7 @@ struct LogString; static constexpr size_t COUNT = Table::COUNT; \ static constexpr uint8_t LAST_INDEX = COUNT - 1; \ static constexpr size_t BLOB_SIZE = Table::BLOB_SIZE; \ + static constexpr size_t MAX_LENGTH = Table::MAX_LENGTH; \ static constexpr auto BLOB ESPHOME_PROGMEM_STRING_TABLE_STORAGE = Table::make_blob(); \ static constexpr auto OFFSETS ESPHOME_PROGMEM_STRING_TABLE_STORAGE = Table::make_offsets(); \ static const char *get_(uint8_t idx, uint8_t fallback) { \ @@ -122,7 +138,7 @@ struct LogString; idx = fallback; \ /* std::launder is used here to prevent the inter-procedural analysis that */ \ /* causes the false positive that the string is not null terminated */ \ - return std::launder(&BLOB[::esphome::progmem_read_byte(&OFFSETS[idx])]); \ + return std::launder(&BLOB[Table::read_offset(&OFFSETS[idx])]); \ } \ static ::ProgmemStr get_progmem_str(uint8_t idx, uint8_t fallback) { \ return reinterpret_cast<::ProgmemStr>(get_(idx, fallback)); \ diff --git a/tests/components/core/test_progmem_string_table.cpp b/tests/components/core/test_progmem_string_table.cpp new file mode 100644 index 00000000000..e0d8da56468 --- /dev/null +++ b/tests/components/core/test_progmem_string_table.cpp @@ -0,0 +1,41 @@ +#include + +#include + +#include "esphome/core/log.h" +#include "esphome/core/progmem.h" + +namespace esphome::core::testing { + +PROGMEM_STRING_TABLE(SmallStrings, "OFF", "HEAT", "COOL", "UNKNOWN"); + +// Eight strings of 40 characters: a 328-byte blob, past what one-byte offsets can address. +PROGMEM_STRING_TABLE(LargeStrings, "0123456789012345678901234567890123456789", + "1123456789012345678901234567890123456789", "2123456789012345678901234567890123456789", + "3123456789012345678901234567890123456789", "4123456789012345678901234567890123456789", + "5123456789012345678901234567890123456789", "6123456789012345678901234567890123456789", + "7123456789012345678901234567890123456789"); + +TEST(ProgmemStringTable, SmallTableKeepsOneByteOffsets) { + static_assert(SmallStrings::BLOB_SIZE <= 255); + static_assert(sizeof(SmallStrings::OFFSETS[0]) == 1); + static_assert(SmallStrings::MAX_LENGTH == 7); + EXPECT_STREQ(reinterpret_cast(SmallStrings::get_log_str(0, SmallStrings::LAST_INDEX)), "OFF"); + EXPECT_STREQ(reinterpret_cast(SmallStrings::get_log_str(2, SmallStrings::LAST_INDEX)), "COOL"); + EXPECT_STREQ(reinterpret_cast(SmallStrings::get_log_str(200, SmallStrings::LAST_INDEX)), "UNKNOWN"); +} + +TEST(ProgmemStringTable, LargeTableUsesTwoByteOffsets) { + static_assert(LargeStrings::BLOB_SIZE > 255); + static_assert(sizeof(LargeStrings::OFFSETS[0]) == 2); + static_assert(LargeStrings::MAX_LENGTH == 40); + EXPECT_STREQ(reinterpret_cast(LargeStrings::get_progmem_str(0, LargeStrings::LAST_INDEX)), + "0123456789012345678901234567890123456789"); + EXPECT_STREQ(reinterpret_cast(LargeStrings::get_progmem_str(7, LargeStrings::LAST_INDEX)), + "7123456789012345678901234567890123456789"); + EXPECT_STREQ(reinterpret_cast(LargeStrings::get_progmem_str(8, LargeStrings::LAST_INDEX)), + "7123456789012345678901234567890123456789"); + EXPECT_EQ(LargeStrings::OFFSETS[7], 7 * 41); +} + +} // namespace esphome::core::testing From 962ba06ae951f724ff00b7becb31dc817b505cc7 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 13 Oct 2026 10:31:56 -1000 Subject: [PATCH 2/4] Pin the 255 and 256 byte switch point in the tests --- .../core/test_progmem_string_table.cpp | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/tests/components/core/test_progmem_string_table.cpp b/tests/components/core/test_progmem_string_table.cpp index e0d8da56468..72a2c507507 100644 --- a/tests/components/core/test_progmem_string_table.cpp +++ b/tests/components/core/test_progmem_string_table.cpp @@ -16,6 +16,29 @@ PROGMEM_STRING_TABLE(LargeStrings, "0123456789012345678901234567890123456789", "5123456789012345678901234567890123456789", "6123456789012345678901234567890123456789", "7123456789012345678901234567890123456789"); +// Five 50-character strings pack into exactly 255 bytes; one more byte needs two-byte offsets. +PROGMEM_STRING_TABLE(Exactly255, "01234567890123456789012345678901234567890123456789", + "01234567890123456789012345678901234567890123456789", + "01234567890123456789012345678901234567890123456789", + "01234567890123456789012345678901234567890123456789", + "01234567890123456789012345678901234567890123456789"); +PROGMEM_STRING_TABLE(Exactly256, "01234567890123456789012345678901234567890123456789", + "01234567890123456789012345678901234567890123456789", + "01234567890123456789012345678901234567890123456789", + "01234567890123456789012345678901234567890123456789", + "01234567890123456789012345678901234567890123456789x"); + +TEST(ProgmemStringTable, OffsetWidthSwitchesAfter255Bytes) { + static_assert(Exactly255::BLOB_SIZE == 255); + static_assert(sizeof(Exactly255::OFFSETS[0]) == 1); + static_assert(Exactly256::BLOB_SIZE == 256); + static_assert(sizeof(Exactly256::OFFSETS[0]) == 2); + EXPECT_EQ(Exactly255::OFFSETS[4], 204); + EXPECT_EQ(Exactly256::OFFSETS[4], 204); + EXPECT_STREQ(reinterpret_cast(Exactly256::get_progmem_str(4, 0)), + "01234567890123456789012345678901234567890123456789x"); +} + TEST(ProgmemStringTable, SmallTableKeepsOneByteOffsets) { static_assert(SmallStrings::BLOB_SIZE <= 255); static_assert(sizeof(SmallStrings::OFFSETS[0]) == 1); From 28268bcd3763c72d72990dd61fbee2fc7ea0b806 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 13 Oct 2026 10:36:32 -1000 Subject: [PATCH 3/4] Scan large tables instead of storing two-byte offsets --- esphome/core/progmem.h | 36 ++++++++++--------- .../core/test_progmem_string_table.cpp | 21 +++++------ 2 files changed, 31 insertions(+), 26 deletions(-) diff --git a/esphome/core/progmem.h b/esphome/core/progmem.h index 64f3eee8aba..b87783745fd 100644 --- a/esphome/core/progmem.h +++ b/esphome/core/progmem.h @@ -6,7 +6,6 @@ #include #include #include -#include #include "esphome/core/hal.h" // For PROGMEM definition @@ -82,8 +81,9 @@ template struct ProgmemStringTable { static constexpr size_t BLOB_SIZE = (0 + ... + (Strs.size() + 1)); /// Length of the longest string, for sizing a RAM copy on ESP8266 static constexpr size_t MAX_LENGTH = std::max({Strs.size()...}); - /// Offsets are one byte while the blob fits in 255 bytes and two bytes beyond that. - using offset_t = std::conditional_t<(BLOB_SIZE <= UINT8_MAX), uint8_t, uint16_t>; + /// One-byte offsets address up to 255 bytes; a larger blob has no offset table and is scanned instead, + /// which costs less flash than wider offsets for the rarely read tables that get this big. + static constexpr bool INDEXED = BLOB_SIZE <= UINT8_MAX; /// Generate packed string blob at compile time static constexpr auto make_blob() { @@ -101,23 +101,24 @@ template struct ProgmemStringTable { static constexpr auto make_offsets() { static_assert(COUNT > 0, "PROGMEM_STRING_TABLE must contain at least one string"); static_assert(COUNT <= 255, "PROGMEM_STRING_TABLE supports at most 255 strings with uint8_t indices"); - static_assert(BLOB_SIZE <= UINT16_MAX, "PROGMEM_STRING_TABLE blob exceeds 65535 bytes; use fewer/shorter strings"); - std::array result{}; - size_t pos = 0, idx = 0; - ((result[idx++] = static_cast(pos), pos += Strs.size() + 1), ...); - return result; - } - - /// Read one offset from the table, which may live in flash - static offset_t read_offset(const offset_t *offset) { - if constexpr (sizeof(offset_t) == 1) { - return progmem_read_byte(offset); + if constexpr (INDEXED) { + std::array result{}; + size_t pos = 0, idx = 0; + ((result[idx++] = static_cast(pos), pos += Strs.size() + 1), ...); + return result; } else { - return progmem_read_uint16(offset); + return std::array{}; } } }; +/// Skip idx strings in a packed NUL-separated blob that may live in flash. +inline const char *progmem_skip_strings(const char *blob, uint8_t idx) { + while (idx-- > 0) + blob += ESPHOME_strlen_P(blob) + 1; + return blob; +} + // Forward declaration for LogString (defined in log.h) struct LogString; @@ -138,7 +139,10 @@ struct LogString; idx = fallback; \ /* std::launder is used here to prevent the inter-procedural analysis that */ \ /* causes the false positive that the string is not null terminated */ \ - return std::launder(&BLOB[Table::read_offset(&OFFSETS[idx])]); \ + if constexpr (Table::INDEXED) \ + return std::launder(&BLOB[::esphome::progmem_read_byte(&OFFSETS[idx])]); \ + else \ + return std::launder(::esphome::progmem_skip_strings(BLOB.data(), idx)); \ } \ static ::ProgmemStr get_progmem_str(uint8_t idx, uint8_t fallback) { \ return reinterpret_cast<::ProgmemStr>(get_(idx, fallback)); \ diff --git a/tests/components/core/test_progmem_string_table.cpp b/tests/components/core/test_progmem_string_table.cpp index 72a2c507507..4069bd4d2ab 100644 --- a/tests/components/core/test_progmem_string_table.cpp +++ b/tests/components/core/test_progmem_string_table.cpp @@ -9,14 +9,14 @@ namespace esphome::core::testing { PROGMEM_STRING_TABLE(SmallStrings, "OFF", "HEAT", "COOL", "UNKNOWN"); -// Eight strings of 40 characters: a 328-byte blob, past what one-byte offsets can address. +// Eight strings of 40 characters: a 328-byte blob, past what one-byte offsets can address, so it is scanned. PROGMEM_STRING_TABLE(LargeStrings, "0123456789012345678901234567890123456789", "1123456789012345678901234567890123456789", "2123456789012345678901234567890123456789", "3123456789012345678901234567890123456789", "4123456789012345678901234567890123456789", "5123456789012345678901234567890123456789", "6123456789012345678901234567890123456789", "7123456789012345678901234567890123456789"); -// Five 50-character strings pack into exactly 255 bytes; one more byte needs two-byte offsets. +// Five 50-character strings pack into exactly 255 bytes; one more byte switches to scanning. PROGMEM_STRING_TABLE(Exactly255, "01234567890123456789012345678901234567890123456789", "01234567890123456789012345678901234567890123456789", "01234567890123456789012345678901234567890123456789", @@ -28,29 +28,29 @@ PROGMEM_STRING_TABLE(Exactly256, "0123456789012345678901234567890123456789012345 "01234567890123456789012345678901234567890123456789", "01234567890123456789012345678901234567890123456789x"); -TEST(ProgmemStringTable, OffsetWidthSwitchesAfter255Bytes) { +TEST(ProgmemStringTable, ScansInsteadOfIndexingAfter255Bytes) { static_assert(Exactly255::BLOB_SIZE == 255); - static_assert(sizeof(Exactly255::OFFSETS[0]) == 1); + static_assert(Exactly255::Table::INDEXED); static_assert(Exactly256::BLOB_SIZE == 256); - static_assert(sizeof(Exactly256::OFFSETS[0]) == 2); + static_assert(!Exactly256::Table::INDEXED); + static_assert(Exactly256::OFFSETS.empty()); EXPECT_EQ(Exactly255::OFFSETS[4], 204); - EXPECT_EQ(Exactly256::OFFSETS[4], 204); EXPECT_STREQ(reinterpret_cast(Exactly256::get_progmem_str(4, 0)), "01234567890123456789012345678901234567890123456789x"); } TEST(ProgmemStringTable, SmallTableKeepsOneByteOffsets) { static_assert(SmallStrings::BLOB_SIZE <= 255); - static_assert(sizeof(SmallStrings::OFFSETS[0]) == 1); + static_assert(SmallStrings::Table::INDEXED); static_assert(SmallStrings::MAX_LENGTH == 7); EXPECT_STREQ(reinterpret_cast(SmallStrings::get_log_str(0, SmallStrings::LAST_INDEX)), "OFF"); EXPECT_STREQ(reinterpret_cast(SmallStrings::get_log_str(2, SmallStrings::LAST_INDEX)), "COOL"); EXPECT_STREQ(reinterpret_cast(SmallStrings::get_log_str(200, SmallStrings::LAST_INDEX)), "UNKNOWN"); } -TEST(ProgmemStringTable, LargeTableUsesTwoByteOffsets) { +TEST(ProgmemStringTable, LargeTableIsScanned) { static_assert(LargeStrings::BLOB_SIZE > 255); - static_assert(sizeof(LargeStrings::OFFSETS[0]) == 2); + static_assert(!LargeStrings::Table::INDEXED); static_assert(LargeStrings::MAX_LENGTH == 40); EXPECT_STREQ(reinterpret_cast(LargeStrings::get_progmem_str(0, LargeStrings::LAST_INDEX)), "0123456789012345678901234567890123456789"); @@ -58,7 +58,8 @@ TEST(ProgmemStringTable, LargeTableUsesTwoByteOffsets) { "7123456789012345678901234567890123456789"); EXPECT_STREQ(reinterpret_cast(LargeStrings::get_progmem_str(8, LargeStrings::LAST_INDEX)), "7123456789012345678901234567890123456789"); - EXPECT_EQ(LargeStrings::OFFSETS[7], 7 * 41); + EXPECT_STREQ(reinterpret_cast(LargeStrings::get_progmem_str(3, LargeStrings::LAST_INDEX)), + "3123456789012345678901234567890123456789"); } } // namespace esphome::core::testing From e509fb4fbae205cd14ea4b480d18c97a0738e907 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 13 Oct 2026 10:42:21 -1000 Subject: [PATCH 4/4] Note in the table comment that large blobs are scanned --- esphome/core/progmem.h | 1 + 1 file changed, 1 insertion(+) diff --git a/esphome/core/progmem.h b/esphome/core/progmem.h index b87783745fd..677ef0e2afb 100644 --- a/esphome/core/progmem.h +++ b/esphome/core/progmem.h @@ -69,6 +69,7 @@ template struct FixedString { }; /// Compile-time string table that packs strings into a single blob with offset lookup. +/// Blobs over 255 bytes have no offsets and are scanned, so keep those for rarely read strings. /// Use PROGMEM_STRING_TABLE macro to instantiate with proper flash placement on ESP8266. /// /// Example: