[runtime_image] Speed up RGB565 JPEG decoding (#19454)

Co-authored-by: J. Nick Koston <nick@koston.org>
Co-authored-by: J. Nick Koston <nick@home-assistant.io>

esphome/components/runtime_image/image_decoder.cpp

@@ -29,4 +29,7 @@ void ImageDecoder::draw(int x, int y, int w, int h, const Color &color) {
   }
 }
 
+uint8_t *ImageDecoder::image_buffer_() const { return this->image_->buffer_; }
+bool ImageDecoder::image_is_big_endian_() const { return this->image_->is_big_endian_; }
+
 }  // namespace esphome::runtime_image

esphome/components/runtime_image/image_decoder.h

@@ -127,6 +127,10 @@ class ImageDecoder {
   }
 
  protected:
+  /// The image's pixel buffer, for decoders that write their output straight into it
+  uint8_t *image_buffer_() const;
+  bool image_is_big_endian_() const;
+
   double x_scale_ = 1.0;
   double y_scale_ = 1.0;
   RuntimeImage *image_;

esphome/components/runtime_image/jpeg_decoder.cpp

@@ -2,6 +2,7 @@
 #ifdef USE_RUNTIME_IMAGE_JPEG
 
 #include "esphome/components/display/display_buffer.h"
+#include "esphome/components/image/image.h"
 #include "esphome/core/application.h"
 #include "esphome/core/defines.h"
 #include "esphome/core/helpers.h"
@@ -15,6 +16,67 @@ ESPHOME_LOG_TAG(TAG, "image_decoder.jpeg");
 
 namespace esphome::runtime_image {
 
+bool JpegDecoder::prepare_rgb565_scaling_(int width, int height) {
+  if (this->x_boundaries_.capacity() < static_cast<size_t>(width + 1) && !this->x_boundaries_.try_init(width + 1)) {
+    return false;
+  }
+  if (this->y_boundaries_.capacity() < static_cast<size_t>(height + 1) && !this->y_boundaries_.try_init(height + 1)) {
+    return false;
+  }
+
+  this->source_width_ = width;
+  this->source_height_ = height;
+  this->x_boundaries_.clear();
+  this->y_boundaries_.clear();
+  const uint32_t target_width = this->image_->get_buffer_width();
+  const uint32_t target_height = this->image_->get_buffer_height();
+  // Boundary n is the first target pixel source pixel n covers, rounded up. When shrinking, the source
+  // pixel whose span lands on a target pixel paints it, so an 8 to 4 scale keeps columns 0, 2, 4 and 6;
+  // the generic draw() path rounds down and keeps the odd ones instead
+  for (uint32_t x = 0; x <= static_cast<uint32_t>(width); x++) {
+    this->x_boundaries_.push_back(static_cast<uint16_t>((x * target_width + width - 1) / width));
+  }
+  for (uint32_t y = 0; y <= static_cast<uint32_t>(height); y++) {
+    this->y_boundaries_.push_back(static_cast<uint16_t>((y * target_height + height - 1) / height));
+  }
+  return true;
+}
+
+void JpegDecoder::draw_rgb565(int x, int y, int w, int h, const uint8_t *pixels) {
+  if (pixels == nullptr) {
+    return;
+  }
+  uint8_t *const buffer = this->image_buffer_();
+  const uint32_t buffer_width = this->image_->get_buffer_width();
+  // JPEGDEC gives little-endian words; a big-endian image stores the high byte first
+  const uint8_t first = this->image_is_big_endian_() ? 1 : 0;
+  for (int source_y = 0; source_y < h; source_y++) {
+    const int absolute_y = y + source_y;
+    if (absolute_y < 0 || absolute_y >= this->source_height_) {
+      continue;
+    }
+    const uint16_t y1 = this->y_boundaries_[absolute_y];
+    const uint16_t y2 = this->y_boundaries_[absolute_y + 1];
+    for (int source_x = 0; source_x < w; source_x++) {
+      const int absolute_x = x + source_x;
+      if (absolute_x < 0 || absolute_x >= this->source_width_) {
+        continue;
+      }
+      const uint16_t x1 = this->x_boundaries_[absolute_x];
+      const uint16_t x2 = this->x_boundaries_[absolute_x + 1];
+      const uint8_t *pixel = pixels + (source_y * w + source_x) * 2;
+      for (uint16_t destination_y = y1; destination_y < y2; destination_y++) {
+        uint8_t *destination = buffer + (destination_y * buffer_width + x1) * 2;
+        for (uint16_t destination_x = x1; destination_x < x2; destination_x++) {
+          destination[0] = pixel[first];
+          destination[1] = pixel[1 - first];
+          destination += 2;
+        }
+      }
+    }
+  }
+}
+
 /**
  * @brief Callback method that will be called by the JPEGDEC engine when a chunk
  * of the image is decoded.
@@ -22,7 +84,12 @@ namespace esphome::runtime_image {
  * @param jpeg  The JPEGDRAW object, including the context data.
  */
 static int draw_callback(JPEGDRAW *jpeg) {
-  ImageDecoder *decoder = (ImageDecoder *) jpeg->pUser;
+  JpegDecoder *decoder = (JpegDecoder *) jpeg->pUser;
+
+  if (decoder == nullptr) {
+    ESP_LOGE(TAG, "Decoder pointer is null!");
+    return 0;
+  }
 
   // Some very big images take too long to decode, so feed the watchdog on each callback
   // to avoid crashing if the executing task has a watchdog enabled.
@@ -33,6 +100,11 @@ static int draw_callback(JPEGDRAW *jpeg) {
 #ifdef USE_ESP_IDF
   }
 #endif
+  if (decoder->uses_direct_rgb565()) {
+    decoder->draw_rgb565(jpeg->x, jpeg->y, jpeg->iWidth, jpeg->iHeight,
+                         reinterpret_cast<const uint8_t *>(jpeg->pPixels));
+    return 1;
+  }
   size_t position = 0;
   size_t height = static_cast<size_t>(jpeg->iHeight);
   size_t width = static_cast<size_t>(jpeg->iWidth);
@@ -42,10 +114,6 @@ static int draw_callback(JPEGDRAW *jpeg) {
       auto ba = decode_value(jpeg->pPixels[position++]);
       Color color(rg[1], rg[0], ba[1], ba[0]);
 
-      if (!decoder) {
-        ESP_LOGE(TAG, "Decoder pointer is null!");
-        return 0;
-      }
       decoder->draw(jpeg->x + x, jpeg->y + y, 1, 1, color);
     }
   }
@@ -78,10 +146,20 @@ int HOT JpegDecoder::decode(uint8_t *buffer, size_t size) {
   ESP_LOGD(TAG, "Image size: %d x %d, bpp: %d", this->jpeg_.getWidth(), this->jpeg_.getHeight(), this->jpeg_.getBpp());
 
   this->jpeg_.setUserPointer(this);
-  this->jpeg_.setPixelType(RGB8888);
+  this->direct_rgb565_ = this->image_->get_type() == image::IMAGE_TYPE_RGB565 && !this->image_->has_transparency();
+  if (this->direct_rgb565_) {
+    // JPEGDEC's RGB565_BIG_ENDIAN output is not byte swapped on every build (the x86 host build
+    // returns little-endian words for it), so the byte order is applied in draw_rgb565 instead
+    this->jpeg_.setPixelType(RGB565_LITTLE_ENDIAN);
+  } else {
+    this->jpeg_.setPixelType(RGB8888);
+  }
   if (!this->set_size(this->jpeg_.getWidth(), this->jpeg_.getHeight())) {
     return DECODE_ERROR_OUT_OF_MEMORY;
   }
+  if (this->direct_rgb565_ && !this->prepare_rgb565_scaling_(this->jpeg_.getWidth(), this->jpeg_.getHeight())) {
+    return DECODE_ERROR_OUT_OF_MEMORY;
+  }
   if (!this->jpeg_.decode(0, 0, 0)) {
     auto error = this->jpeg_.getLastError();
     ESP_LOGE(TAG, "Error while decoding: %d", error);

esphome/components/runtime_image/jpeg_decoder.h

@@ -1,8 +1,8 @@
 #pragma once
-
 #include "image_decoder.h"
 #include "runtime_image.h"
 #include "esphome/core/defines.h"
+#include "esphome/core/helpers.h"
 #ifdef USE_RUNTIME_IMAGE_JPEG
 #include <JPEGDEC.h>
 
@@ -23,8 +23,18 @@ class JpegDecoder : public ImageDecoder {
 
   int HOT decode(uint8_t *buffer, size_t size) override;
 
+  bool uses_direct_rgb565() const { return this->direct_rgb565_; }
+  void draw_rgb565(int x, int y, int w, int h, const uint8_t *pixels);
+
  protected:
+  bool prepare_rgb565_scaling_(int width, int height);
+
   JPEGDEC jpeg_{};
+  FixedVector<uint16_t> x_boundaries_{};
+  FixedVector<uint16_t> y_boundaries_{};
+  int source_width_{0};
+  int source_height_{0};
+  bool direct_rgb565_{false};
 };
 
 }  // namespace esphome::runtime_image

esphome/components/runtime_image/runtime_image.h

@@ -234,6 +234,8 @@ class RuntimeImage : public image::Image {
   bool is_big_endian_{false};
   /** Whether buffer_ belongs to the caller, so it must not be freed or resized here. */
   bool external_buffer_{false};
+
+  friend class ImageDecoder;
 };
 
 }  // namespace esphome::runtime_image

tests/components/runtime_image/test_decoder_reuse.cpp

@@ -101,6 +101,9 @@ class TestableRuntimeImage : public RuntimeImage {
   explicit TestableRuntimeImage(ImageFormat format, image::Transparency transparency = image::TRANSPARENCY_OPAQUE)
       : RuntimeImage(format, image::IMAGE_TYPE_RGB, transparency, nullptr, false, 0, 0) {}
 
+  TestableRuntimeImage(ImageFormat format, image::ImageType type, bool is_big_endian, int fixed_width, int fixed_height)
+      : RuntimeImage(format, type, image::TRANSPARENCY_OPAQUE, nullptr, is_big_endian, fixed_width, fixed_height) {}
+
   ImageDecoder *decoder() { return this->decoder_.get(); }
 };
 
@@ -373,6 +376,87 @@ TEST(RuntimeImageDecoder, JpegDecoderStaysWarmAcrossDecodes) {
   EXPECT_EQ(img.decoder(), first);
   EXPECT_EQ(pixel_bytes(img), first_pixels) << "reused decoder must reproduce identical pixels";
 }
+
+// JPEGDEC's colour conversion differs by one step between its SIMD and scalar builds, so each
+// channel is checked within one step; the byte order is still exact
+static void expect_rgb565_near(uint16_t actual, uint16_t expected, const char *order, size_t pixel) {
+  EXPECT_NEAR(actual >> 11, expected >> 11, 1) << order << " pixel " << pixel << " red";
+  EXPECT_NEAR((actual >> 5) & 0x3F, (expected >> 5) & 0x3F, 1) << order << " pixel " << pixel << " green";
+  EXPECT_NEAR(actual & 0x1F, expected & 0x1F, 1) << order << " pixel " << pixel << " blue";
+}
+
+TEST(RuntimeImageDecoder, JpegDecodesDirectlyToScaledRgb565WithBothByteOrders) {
+  // Halving the 8x8 gradient keeps its even rows and columns. The words are JPEGDEC's output for those
+  // pixels, red in the top five bits: a decode of the fixture with PIL gives (51, 7, 30) for source
+  // pixel (2, 0), which is word 0x3803 here.
+  static const uint16_t EXPECTED[4 * 4] = {
+      0x0000, 0x3803, 0x8008, 0xB80B, 0x0203, 0x3A07, 0x81EC, 0xBA0F,
+      0x03E7, 0x3C0B, 0x83F0, 0xBC13, 0x05EC, 0x3E0F, 0x85F4, 0xBE17,
+  };
+  TestableRuntimeImage rgb565_little_endian(JPEG, image::IMAGE_TYPE_RGB565, false, 4, 4);
+  TestableRuntimeImage rgb565_big_endian(JPEG, image::IMAGE_TYPE_RGB565, true, 4, 4);
+
+  ASSERT_TRUE(decode_all(rgb565_little_endian, JPEG_GRADIENT, sizeof(JPEG_GRADIENT)));
+  ASSERT_TRUE(decode_all(rgb565_big_endian, JPEG_GRADIENT, sizeof(JPEG_GRADIENT)));
+  ASSERT_EQ(rgb565_little_endian.get_width(), 4);
+  ASSERT_EQ(rgb565_little_endian.get_height(), 4);
+  ASSERT_EQ(rgb565_big_endian.get_width(), 4);
+  ASSERT_EQ(rgb565_big_endian.get_height(), 4);
+
+  const uint8_t *little = rgb565_little_endian.get_data_start();
+  const uint8_t *big = rgb565_big_endian.get_data_start();
+  for (size_t i = 0; i < 4 * 4; i++) {
+    expect_rgb565_near(little[i * 2] | (little[i * 2 + 1] << 8), EXPECTED[i], "little-endian", i);
+    expect_rgb565_near((big[i * 2] << 8) | big[i * 2 + 1], EXPECTED[i], "big-endian", i);
+  }
+}
+
+// Target pixel index each source pixel paints when `source` pixels map onto `target`, the way
+// prepare_rgb565_scaling_ rounds: source n covers targets ceil(n * target / source) up to the next start
+static uint16_t scaled_boundary(int n, int source, int target) { return (n * target + source - 1) / source; }
+
+// The RGB565 word at (x, y) of a decoded image, in its own byte order
+static uint16_t rgb565_word(const TestableRuntimeImage &img, int x, int y) {
+  const uint8_t *p = img.get_data_start() + (y * img.get_width() + x) * 2;
+  return p[0] | (p[1] << 8);
+}
+
+// Every scaled direct decode must equal the unscaled one sampled through the boundary table,
+// for shrinking, growing and a ratio that is not a whole number
+TEST(RuntimeImageDecoder, JpegDirectRgb565ScalesBySourcePixelSpans) {
+  TestableRuntimeImage full(JPEG, image::IMAGE_TYPE_RGB565, false, 8, 8);
+  ASSERT_TRUE(decode_all(full, JPEG_GRADIENT, sizeof(JPEG_GRADIENT)));
+
+  for (int size : {4, 6, 16}) {
+    TestableRuntimeImage scaled(JPEG, image::IMAGE_TYPE_RGB565, false, size, size);
+    ASSERT_TRUE(decode_all(scaled, JPEG_GRADIENT, sizeof(JPEG_GRADIENT)));
+    ASSERT_EQ(scaled.get_width(), size);
+    ASSERT_EQ(scaled.get_height(), size);
+    for (int n = 0; n < 8; n++) {
+      for (int m = 0; m < 8; m++) {
+        for (int y = scaled_boundary(m, 8, size); y < scaled_boundary(m + 1, 8, size); y++) {
+          for (int x = scaled_boundary(n, 8, size); x < scaled_boundary(n + 1, 8, size); x++) {
+            EXPECT_EQ(rgb565_word(scaled, x, y), rgb565_word(full, n, m))
+                << size << "x" << size << " pixel " << x << "," << y;
+          }
+        }
+      }
+    }
+  }
+}
+
+// A second decode into the same image reuses the boundary tables and gives the same pixels
+TEST(RuntimeImageDecoder, JpegDirectRgb565RepeatsWithReusedTables) {
+  TestableRuntimeImage img(JPEG, image::IMAGE_TYPE_RGB565, false, 6, 6);
+  ASSERT_TRUE(decode_all(img, JPEG_GRADIENT, sizeof(JPEG_GRADIENT)));
+  std::vector<uint8_t> first_pixels = pixel_bytes(img);
+  ImageDecoder *first = img.decoder();
+  ASSERT_NE(first, nullptr);
+
+  ASSERT_TRUE(decode_all(img, JPEG_GRADIENT, sizeof(JPEG_GRADIENT)));
+  EXPECT_EQ(img.decoder(), first);
+  EXPECT_EQ(pixel_bytes(img), first_pixels);
+}
 #endif  // USE_RUNTIME_IMAGE_JPEG
 
 TEST(RuntimeImageDecoder, QoiDecoderStaysWarmAcrossDecodes) {