From 88fdf822cdbc47d127ed686029dc038e17ae0ac3 Mon Sep 17 00:00:00 2001 From: jcianci12 Date: Tue, 13 Oct 2026 11:41:30 +1000 Subject: [PATCH] [runtime_image] Speed up RGB565 JPEG decoding (#19454) Co-authored-by: J. Nick Koston Co-authored-by: J. Nick Koston --- .../runtime_image/image_decoder.cpp | 3 + .../components/runtime_image/image_decoder.h | 4 + .../components/runtime_image/jpeg_decoder.cpp | 90 +++++++++++++++++-- .../components/runtime_image/jpeg_decoder.h | 12 ++- .../components/runtime_image/runtime_image.h | 2 + .../runtime_image/test_decoder_reuse.cpp | 84 +++++++++++++++++ 6 files changed, 188 insertions(+), 7 deletions(-) diff --git a/esphome/components/runtime_image/image_decoder.cpp b/esphome/components/runtime_image/image_decoder.cpp index 7056ce35aa..55e25d76b7 100644 --- a/esphome/components/runtime_image/image_decoder.cpp +++ b/esphome/components/runtime_image/image_decoder.cpp @@ -29,4 +29,7 @@ void ImageDecoder::draw(int x, int y, int w, int h, const Color &color) { } } +uint8_t *ImageDecoder::image_buffer_() const { return this->image_->buffer_; } +bool ImageDecoder::image_is_big_endian_() const { return this->image_->is_big_endian_; } + } // namespace esphome::runtime_image diff --git a/esphome/components/runtime_image/image_decoder.h b/esphome/components/runtime_image/image_decoder.h index 2a8b393888..8e0399ac35 100644 --- a/esphome/components/runtime_image/image_decoder.h +++ b/esphome/components/runtime_image/image_decoder.h @@ -127,6 +127,10 @@ class ImageDecoder { } protected: + /// The image's pixel buffer, for decoders that write their output straight into it + uint8_t *image_buffer_() const; + bool image_is_big_endian_() const; + double x_scale_ = 1.0; double y_scale_ = 1.0; RuntimeImage *image_; diff --git a/esphome/components/runtime_image/jpeg_decoder.cpp b/esphome/components/runtime_image/jpeg_decoder.cpp index 74ced63a4e..d4783701c4 100644 --- a/esphome/components/runtime_image/jpeg_decoder.cpp +++ b/esphome/components/runtime_image/jpeg_decoder.cpp @@ -2,6 +2,7 @@ #ifdef USE_RUNTIME_IMAGE_JPEG #include "esphome/components/display/display_buffer.h" +#include "esphome/components/image/image.h" #include "esphome/core/application.h" #include "esphome/core/defines.h" #include "esphome/core/helpers.h" @@ -15,6 +16,67 @@ ESPHOME_LOG_TAG(TAG, "image_decoder.jpeg"); namespace esphome::runtime_image { +bool JpegDecoder::prepare_rgb565_scaling_(int width, int height) { + if (this->x_boundaries_.capacity() < static_cast(width + 1) && !this->x_boundaries_.try_init(width + 1)) { + return false; + } + if (this->y_boundaries_.capacity() < static_cast(height + 1) && !this->y_boundaries_.try_init(height + 1)) { + return false; + } + + this->source_width_ = width; + this->source_height_ = height; + this->x_boundaries_.clear(); + this->y_boundaries_.clear(); + const uint32_t target_width = this->image_->get_buffer_width(); + const uint32_t target_height = this->image_->get_buffer_height(); + // Boundary n is the first target pixel source pixel n covers, rounded up. When shrinking, the source + // pixel whose span lands on a target pixel paints it, so an 8 to 4 scale keeps columns 0, 2, 4 and 6; + // the generic draw() path rounds down and keeps the odd ones instead + for (uint32_t x = 0; x <= static_cast(width); x++) { + this->x_boundaries_.push_back(static_cast((x * target_width + width - 1) / width)); + } + for (uint32_t y = 0; y <= static_cast(height); y++) { + this->y_boundaries_.push_back(static_cast((y * target_height + height - 1) / height)); + } + return true; +} + +void JpegDecoder::draw_rgb565(int x, int y, int w, int h, const uint8_t *pixels) { + if (pixels == nullptr) { + return; + } + uint8_t *const buffer = this->image_buffer_(); + const uint32_t buffer_width = this->image_->get_buffer_width(); + // JPEGDEC gives little-endian words; a big-endian image stores the high byte first + const uint8_t first = this->image_is_big_endian_() ? 1 : 0; + for (int source_y = 0; source_y < h; source_y++) { + const int absolute_y = y + source_y; + if (absolute_y < 0 || absolute_y >= this->source_height_) { + continue; + } + const uint16_t y1 = this->y_boundaries_[absolute_y]; + const uint16_t y2 = this->y_boundaries_[absolute_y + 1]; + for (int source_x = 0; source_x < w; source_x++) { + const int absolute_x = x + source_x; + if (absolute_x < 0 || absolute_x >= this->source_width_) { + continue; + } + const uint16_t x1 = this->x_boundaries_[absolute_x]; + const uint16_t x2 = this->x_boundaries_[absolute_x + 1]; + const uint8_t *pixel = pixels + (source_y * w + source_x) * 2; + for (uint16_t destination_y = y1; destination_y < y2; destination_y++) { + uint8_t *destination = buffer + (destination_y * buffer_width + x1) * 2; + for (uint16_t destination_x = x1; destination_x < x2; destination_x++) { + destination[0] = pixel[first]; + destination[1] = pixel[1 - first]; + destination += 2; + } + } + } + } +} + /** * @brief Callback method that will be called by the JPEGDEC engine when a chunk * of the image is decoded. @@ -22,7 +84,12 @@ namespace esphome::runtime_image { * @param jpeg The JPEGDRAW object, including the context data. */ static int draw_callback(JPEGDRAW *jpeg) { - ImageDecoder *decoder = (ImageDecoder *) jpeg->pUser; + JpegDecoder *decoder = (JpegDecoder *) jpeg->pUser; + + if (decoder == nullptr) { + ESP_LOGE(TAG, "Decoder pointer is null!"); + return 0; + } // Some very big images take too long to decode, so feed the watchdog on each callback // to avoid crashing if the executing task has a watchdog enabled. @@ -33,6 +100,11 @@ static int draw_callback(JPEGDRAW *jpeg) { #ifdef USE_ESP_IDF } #endif + if (decoder->uses_direct_rgb565()) { + decoder->draw_rgb565(jpeg->x, jpeg->y, jpeg->iWidth, jpeg->iHeight, + reinterpret_cast(jpeg->pPixels)); + return 1; + } size_t position = 0; size_t height = static_cast(jpeg->iHeight); size_t width = static_cast(jpeg->iWidth); @@ -42,10 +114,6 @@ static int draw_callback(JPEGDRAW *jpeg) { auto ba = decode_value(jpeg->pPixels[position++]); Color color(rg[1], rg[0], ba[1], ba[0]); - if (!decoder) { - ESP_LOGE(TAG, "Decoder pointer is null!"); - return 0; - } decoder->draw(jpeg->x + x, jpeg->y + y, 1, 1, color); } } @@ -78,10 +146,20 @@ int HOT JpegDecoder::decode(uint8_t *buffer, size_t size) { ESP_LOGD(TAG, "Image size: %d x %d, bpp: %d", this->jpeg_.getWidth(), this->jpeg_.getHeight(), this->jpeg_.getBpp()); this->jpeg_.setUserPointer(this); - this->jpeg_.setPixelType(RGB8888); + this->direct_rgb565_ = this->image_->get_type() == image::IMAGE_TYPE_RGB565 && !this->image_->has_transparency(); + if (this->direct_rgb565_) { + // JPEGDEC's RGB565_BIG_ENDIAN output is not byte swapped on every build (the x86 host build + // returns little-endian words for it), so the byte order is applied in draw_rgb565 instead + this->jpeg_.setPixelType(RGB565_LITTLE_ENDIAN); + } else { + this->jpeg_.setPixelType(RGB8888); + } if (!this->set_size(this->jpeg_.getWidth(), this->jpeg_.getHeight())) { return DECODE_ERROR_OUT_OF_MEMORY; } + if (this->direct_rgb565_ && !this->prepare_rgb565_scaling_(this->jpeg_.getWidth(), this->jpeg_.getHeight())) { + return DECODE_ERROR_OUT_OF_MEMORY; + } if (!this->jpeg_.decode(0, 0, 0)) { auto error = this->jpeg_.getLastError(); ESP_LOGE(TAG, "Error while decoding: %d", error); diff --git a/esphome/components/runtime_image/jpeg_decoder.h b/esphome/components/runtime_image/jpeg_decoder.h index 67c9b77f4d..f72f69422d 100644 --- a/esphome/components/runtime_image/jpeg_decoder.h +++ b/esphome/components/runtime_image/jpeg_decoder.h @@ -1,8 +1,8 @@ #pragma once - #include "image_decoder.h" #include "runtime_image.h" #include "esphome/core/defines.h" +#include "esphome/core/helpers.h" #ifdef USE_RUNTIME_IMAGE_JPEG #include @@ -23,8 +23,18 @@ class JpegDecoder : public ImageDecoder { int HOT decode(uint8_t *buffer, size_t size) override; + bool uses_direct_rgb565() const { return this->direct_rgb565_; } + void draw_rgb565(int x, int y, int w, int h, const uint8_t *pixels); + protected: + bool prepare_rgb565_scaling_(int width, int height); + JPEGDEC jpeg_{}; + FixedVector x_boundaries_{}; + FixedVector y_boundaries_{}; + int source_width_{0}; + int source_height_{0}; + bool direct_rgb565_{false}; }; } // namespace esphome::runtime_image diff --git a/esphome/components/runtime_image/runtime_image.h b/esphome/components/runtime_image/runtime_image.h index 55d5c0ae86..33bad52acf 100644 --- a/esphome/components/runtime_image/runtime_image.h +++ b/esphome/components/runtime_image/runtime_image.h @@ -234,6 +234,8 @@ class RuntimeImage : public image::Image { bool is_big_endian_{false}; /** Whether buffer_ belongs to the caller, so it must not be freed or resized here. */ bool external_buffer_{false}; + + friend class ImageDecoder; }; } // namespace esphome::runtime_image diff --git a/tests/components/runtime_image/test_decoder_reuse.cpp b/tests/components/runtime_image/test_decoder_reuse.cpp index e62e2a98d9..9f7190980b 100644 --- a/tests/components/runtime_image/test_decoder_reuse.cpp +++ b/tests/components/runtime_image/test_decoder_reuse.cpp @@ -101,6 +101,9 @@ class TestableRuntimeImage : public RuntimeImage { explicit TestableRuntimeImage(ImageFormat format, image::Transparency transparency = image::TRANSPARENCY_OPAQUE) : RuntimeImage(format, image::IMAGE_TYPE_RGB, transparency, nullptr, false, 0, 0) {} + TestableRuntimeImage(ImageFormat format, image::ImageType type, bool is_big_endian, int fixed_width, int fixed_height) + : RuntimeImage(format, type, image::TRANSPARENCY_OPAQUE, nullptr, is_big_endian, fixed_width, fixed_height) {} + ImageDecoder *decoder() { return this->decoder_.get(); } }; @@ -373,6 +376,87 @@ TEST(RuntimeImageDecoder, JpegDecoderStaysWarmAcrossDecodes) { EXPECT_EQ(img.decoder(), first); EXPECT_EQ(pixel_bytes(img), first_pixels) << "reused decoder must reproduce identical pixels"; } + +// JPEGDEC's colour conversion differs by one step between its SIMD and scalar builds, so each +// channel is checked within one step; the byte order is still exact +static void expect_rgb565_near(uint16_t actual, uint16_t expected, const char *order, size_t pixel) { + EXPECT_NEAR(actual >> 11, expected >> 11, 1) << order << " pixel " << pixel << " red"; + EXPECT_NEAR((actual >> 5) & 0x3F, (expected >> 5) & 0x3F, 1) << order << " pixel " << pixel << " green"; + EXPECT_NEAR(actual & 0x1F, expected & 0x1F, 1) << order << " pixel " << pixel << " blue"; +} + +TEST(RuntimeImageDecoder, JpegDecodesDirectlyToScaledRgb565WithBothByteOrders) { + // Halving the 8x8 gradient keeps its even rows and columns. The words are JPEGDEC's output for those + // pixels, red in the top five bits: a decode of the fixture with PIL gives (51, 7, 30) for source + // pixel (2, 0), which is word 0x3803 here. + static const uint16_t EXPECTED[4 * 4] = { + 0x0000, 0x3803, 0x8008, 0xB80B, 0x0203, 0x3A07, 0x81EC, 0xBA0F, + 0x03E7, 0x3C0B, 0x83F0, 0xBC13, 0x05EC, 0x3E0F, 0x85F4, 0xBE17, + }; + TestableRuntimeImage rgb565_little_endian(JPEG, image::IMAGE_TYPE_RGB565, false, 4, 4); + TestableRuntimeImage rgb565_big_endian(JPEG, image::IMAGE_TYPE_RGB565, true, 4, 4); + + ASSERT_TRUE(decode_all(rgb565_little_endian, JPEG_GRADIENT, sizeof(JPEG_GRADIENT))); + ASSERT_TRUE(decode_all(rgb565_big_endian, JPEG_GRADIENT, sizeof(JPEG_GRADIENT))); + ASSERT_EQ(rgb565_little_endian.get_width(), 4); + ASSERT_EQ(rgb565_little_endian.get_height(), 4); + ASSERT_EQ(rgb565_big_endian.get_width(), 4); + ASSERT_EQ(rgb565_big_endian.get_height(), 4); + + const uint8_t *little = rgb565_little_endian.get_data_start(); + const uint8_t *big = rgb565_big_endian.get_data_start(); + for (size_t i = 0; i < 4 * 4; i++) { + expect_rgb565_near(little[i * 2] | (little[i * 2 + 1] << 8), EXPECTED[i], "little-endian", i); + expect_rgb565_near((big[i * 2] << 8) | big[i * 2 + 1], EXPECTED[i], "big-endian", i); + } +} + +// Target pixel index each source pixel paints when `source` pixels map onto `target`, the way +// prepare_rgb565_scaling_ rounds: source n covers targets ceil(n * target / source) up to the next start +static uint16_t scaled_boundary(int n, int source, int target) { return (n * target + source - 1) / source; } + +// The RGB565 word at (x, y) of a decoded image, in its own byte order +static uint16_t rgb565_word(const TestableRuntimeImage &img, int x, int y) { + const uint8_t *p = img.get_data_start() + (y * img.get_width() + x) * 2; + return p[0] | (p[1] << 8); +} + +// Every scaled direct decode must equal the unscaled one sampled through the boundary table, +// for shrinking, growing and a ratio that is not a whole number +TEST(RuntimeImageDecoder, JpegDirectRgb565ScalesBySourcePixelSpans) { + TestableRuntimeImage full(JPEG, image::IMAGE_TYPE_RGB565, false, 8, 8); + ASSERT_TRUE(decode_all(full, JPEG_GRADIENT, sizeof(JPEG_GRADIENT))); + + for (int size : {4, 6, 16}) { + TestableRuntimeImage scaled(JPEG, image::IMAGE_TYPE_RGB565, false, size, size); + ASSERT_TRUE(decode_all(scaled, JPEG_GRADIENT, sizeof(JPEG_GRADIENT))); + ASSERT_EQ(scaled.get_width(), size); + ASSERT_EQ(scaled.get_height(), size); + for (int n = 0; n < 8; n++) { + for (int m = 0; m < 8; m++) { + for (int y = scaled_boundary(m, 8, size); y < scaled_boundary(m + 1, 8, size); y++) { + for (int x = scaled_boundary(n, 8, size); x < scaled_boundary(n + 1, 8, size); x++) { + EXPECT_EQ(rgb565_word(scaled, x, y), rgb565_word(full, n, m)) + << size << "x" << size << " pixel " << x << "," << y; + } + } + } + } + } +} + +// A second decode into the same image reuses the boundary tables and gives the same pixels +TEST(RuntimeImageDecoder, JpegDirectRgb565RepeatsWithReusedTables) { + TestableRuntimeImage img(JPEG, image::IMAGE_TYPE_RGB565, false, 6, 6); + ASSERT_TRUE(decode_all(img, JPEG_GRADIENT, sizeof(JPEG_GRADIENT))); + std::vector first_pixels = pixel_bytes(img); + ImageDecoder *first = img.decoder(); + ASSERT_NE(first, nullptr); + + ASSERT_TRUE(decode_all(img, JPEG_GRADIENT, sizeof(JPEG_GRADIENT))); + EXPECT_EQ(img.decoder(), first); + EXPECT_EQ(pixel_bytes(img), first_pixels); +} #endif // USE_RUNTIME_IMAGE_JPEG TEST(RuntimeImageDecoder, QoiDecoderStaysWarmAcrossDecodes) {