diff --git a/esphome/writer.py b/esphome/writer.py
index c75c45daa4..d614204603 100644
--- a/esphome/writer.py
+++ b/esphome/writer.py
@@ -348,15 +348,17 @@ def copy_src_tree():
             sources_changed = True
         except (ValueError, OSError) as err:
             # ValueError covers both JSONDecodeError and UnicodeDecodeError;
-            # unlink so the regenerating write never re-reads the damage
-            _LOGGER.warning("Replacing damaged build_info.json: %s", err)
+            # unlink so the regenerating write never re-reads the bad copy.
+            # "Unreadable" not "damaged": EACCES/EISDIR land here too
+            _LOGGER.warning("Regenerating unreadable build_info.json: %s", err)
             try:
-                build_info_json_path.unlink()
+                # missing_ok: a concurrent clean may have removed it already
+                build_info_json_path.unlink(missing_ok=True)
             except OSError as unlink_err:
-                # The later write re-reads the file, so a kept damaged copy
+                # The later write re-reads the file, so a kept unreadable copy
                 # fails again with a misattributed error; name the real cause
                 _LOGGER.warning(
-                    "Could not remove damaged build_info.json: %s", unlink_err
+                    "Could not remove unreadable build_info.json: %s", unlink_err
                 )
             sources_changed = True
 
diff --git a/tests/unit_tests/test_writer.py b/tests/unit_tests/test_writer.py
index 6bf20e20c2..c73a5c5789 100644
--- a/tests/unit_tests/test_writer.py
+++ b/tests/unit_tests/test_writer.py
@@ -2161,7 +2161,7 @@ def test_copy_src_tree_missing_build_info_rebuilds_quietly(
     build_info_json_path.unlink()
     _run_copy_src_tree()
     assert json.loads(build_info_json_path.read_text())["config_hash"] == 0xDEADBEEF
-    assert "damaged" not in caplog.text
+    assert "unreadable" not in caplog.text
 
 
 @patch("esphome.writer.CORE")
@@ -2189,7 +2189,7 @@ def test_copy_src_tree_unremovable_damaged_build_info_is_logged(
 
     with patch.object(Path, "unlink", fail_on_build_info):
         _run_copy_src_tree()
-    assert "Could not remove damaged build_info.json" in caplog.text
+    assert "Could not remove unreadable build_info.json" in caplog.text
     assert json.loads(build_info_json_path.read_text())["config_hash"] == 0xDEADBEEF