diff --git a/esphome/components/api/__init__.py b/esphome/components/api/__init__.py
index dd99862cc27..4591b5dddac 100644
--- a/esphome/components/api/__init__.py
+++ b/esphome/components/api/__init__.py
@@ -453,6 +453,18 @@ async def to_code(config: ConfigType) -> None:
             # and plaintext disabled. Only a factory reset can remove it.
             cg.add_define("USE_API_PLAINTEXT")
         cg.add_define("USE_API_NOISE")
+        # libsodium uses NATIVE_LITTLE_ENDIAN to enable optimized 32-bit
+        # load/store via memcpy instead of byte-at-a-time operations in
+        # poly1305 and chacha20.
+        if (
+            CORE.is_esp32
+            or CORE.is_esp8266
+            or CORE.is_rp2040
+            or CORE.is_bk72xx
+            or CORE.is_rtl87xx
+            or CORE.is_ln882x
+        ):
+            cg.add_build_flag("-DNATIVE_LITTLE_ENDIAN")
         cg.add_library("esphome/noise-c", "0.1.10")
     else:
         cg.add_define("USE_API_PLAINTEXT")