diff --git a/esphome/components/api/__init__.py b/esphome/components/api/__init__.py
index 5f1f0f884b5..dd99862cc27 100644
--- a/esphome/components/api/__init__.py
+++ b/esphome/components/api/__init__.py
@@ -453,10 +453,6 @@ async def to_code(config: ConfigType) -> None:
             # and plaintext disabled. Only a factory reset can remove it.
             cg.add_define("USE_API_PLAINTEXT")
         cg.add_define("USE_API_NOISE")
-        # libsodium uses NATIVE_LITTLE_ENDIAN to enable optimized 32-bit
-        # load/store via memcpy instead of byte-at-a-time operations in
-        # poly1305 and chacha20. All ESPHome targets are little-endian.
-        cg.add_build_flag("-DNATIVE_LITTLE_ENDIAN")
         cg.add_library("esphome/noise-c", "0.1.10")
     else:
         cg.add_define("USE_API_PLAINTEXT")