[espidf] Read the ESP-IDF version in process instead of starting idf_tools (#19998)

esphome/espidf/framework.py

@@ -336,15 +336,42 @@ def _raise_script_failure(what: str, root: PathType, stderr: str | None) -> NoRe
     )
 
 
-def _get_idf_version(
-    idf_framework_root: PathType, env: dict[str, str] | None = None
-) -> str:
+# What idf_tools.get_idf_version() matches: ``version.txt`` first, then the
+# version header. Both give major.minor only.
+_IDF_VERSION_TXT_RE = re.compile(r"^v(\d+\.\d+)")
+_IDF_VERSION_HEADER_RE = re.compile(
+    r"^#define\s+ESP_IDF_VERSION_MAJOR\s+(\d+).+?^#define\s+ESP_IDF_VERSION_MINOR\s+(\d+)",
+    re.DOTALL | re.MULTILINE,
+)
+
+
+def _read_text(path: Path) -> str:
+    return path.read_text(encoding="utf-8") if path.is_file() else ""
+
+
+def read_idf_version_txt(idf_framework_root: Path) -> str | None:
+    """major.minor from ``version.txt``, as idf_tools reads it."""
+    if match := _IDF_VERSION_TXT_RE.match(
+        _read_text(idf_framework_root / "version.txt")
+    ):
+        return match.group(1)
+    return None
+
+
+def read_idf_version_header(idf_framework_root: Path) -> str | None:
+    """major.minor from ``esp_idf_version.h``, as idf_tools reads it."""
+    header = idf_framework_root / "components" / "esp_common" / "include"
+    if match := _IDF_VERSION_HEADER_RE.search(_read_text(header / "esp_idf_version.h")):
+        return f"{match.group(1)}.{match.group(2)}"
+    return None
+
+
+def _get_idf_version(idf_framework_root: PathType) -> str:
     """
     Get the ESP-IDF version from the specified framework root.
 
     Args:
         idf_framework_root: Path to the ESP-IDF framework root directory
-        env: Optional dictionary of environment variables to set
 
     Returns:
         String containing ESP-IDF version
@@ -352,9 +379,20 @@ def _get_idf_version(
     Raises:
         RuntimeError: If ESP-IDF version cannot be determined
     """
+    root = Path(idf_framework_root)
+    try:
+        version = read_idf_version_txt(root) or read_idf_version_header(root)
+    except (OSError, UnicodeError) as e:
+        raise RuntimeError(f"Can't get ESP-IDF version of {root}: {e}") from e
+    if version is None:
+        raise RuntimeError(f"Can't get ESP-IDF version of {root}")
+    return version
 
+
+def idf_tools_version(idf_framework_root: PathType) -> str:
+    """The version from the framework's own ``idf_tools``, for the CI drift guard."""
     success, stdout, stderr = _run_idf_tools_script(
-        idf_framework_root, "get_idf_version.py", "ESP-IDF version", env=env
+        idf_framework_root, "get_idf_version.py", "ESP-IDF version"
     )
     if stdout:
         stdout = stdout.strip()
@@ -1070,7 +1108,7 @@ def _check_esp_idf_python_env_install(
 
         create_venv(python_env_path, msg=f"ESP-IDF {version}")
 
-        esp_idf_version = _get_idf_version(framework_path, env=env)
+        esp_idf_version = _get_idf_version(framework_path)
         constraint_file_path = (
             get_idf_tools_path() / f"espidf.constraints.v{esp_idf_version}.txt"
         )
@@ -1311,7 +1349,7 @@ def get_framework_env(
 
     # 4. Set framework-specific environment variables
     env["IDF_PATH"] = str(framework_path)
-    env["ESP_IDF_VERSION"] = _get_idf_version(framework_path, env)
+    env["ESP_IDF_VERSION"] = _get_idf_version(framework_path)
 
     # 5. Get and add tool paths and environment variables
     paths_to_export, export_vars = _get_idf_tool_paths(framework_path, env)

script/check_idf_py_equivalence.py

@@ -55,6 +55,10 @@ MACRO_CHANGED = (
     "IDF changed __build_process_project_includes; update "
     "IDF_BOOTLOADER_OVERRIDE in esphome/build_gen/espidf.py"
 )
+VERSION_DRIFT = (
+    "ESPHome reads ESP-IDF version {ours!r} from {source} but idf_tools reports "
+    "{theirs!r}; update read_idf_version_{source} in esphome/espidf/framework.py"
+)
 WORK_SUFFIXES = (".obj", ".o", ".a", ".elf", ".map", ".bin", ".ld")
 DEFAULT_GLOB = "tests/test_build_components/build/.esphome/build/*"
 
@@ -127,7 +131,7 @@ def check(build_path: Path) -> list[str]:
     # pylint: disable=protected-access
     from esphome.build_gen.espidf import idf_macro_matches
     from esphome.core import CORE
-    from esphome.espidf import toolchain
+    from esphome.espidf import framework, toolchain
 
     description = json.loads(
         (build_path / "build" / "project_description.json").read_text(encoding="utf-8")
@@ -139,8 +143,18 @@ def check(build_path: Path) -> list[str]:
     CORE.skip_bootloader = skip_bootloader
     # A prior tree's memoized decision must not leak into this one.
     toolchain._cache().skip_bootloader = None
-    if not idf_macro_matches(toolchain._get_idf_path(version)):
+    idf_path = toolchain._get_idf_path(version)
+    if not idf_macro_matches(idf_path):
         return [MACRO_CHANGED]
+    # A managed tree always has version.txt, so the header branch is
+    # compared on its own or it would never be exercised here.
+    theirs = framework.idf_tools_version(idf_path)
+    for source, read in (
+        ("txt", framework.read_idf_version_txt),
+        ("header", framework.read_idf_version_header),
+    ):
+        if (ours := read(idf_path)) != theirs:
+            return [VERSION_DRIFT.format(ours=ours, source=source, theirs=theirs)]
     # ESP-IDF's openthread stamps the configure time into its compile flags;
     # pin it before the env is cached so both configures get the same value.
     os.environ["SOURCE_DATE_EPOCH"] = "0"

tests/script/test_check_idf_py_equivalence.py

@@ -16,7 +16,7 @@ import check_idf_py_equivalence as guard  # noqa: E402
 
 from esphome.build_gen import espidf as build_gen  # noqa: E402
 from esphome.core import CORE  # noqa: E402
-from esphome.espidf import toolchain  # noqa: E402
+from esphome.espidf import framework, toolchain  # noqa: E402
 
 
 @pytest.fixture(autouse=True)
@@ -74,10 +74,12 @@ def _run_check(
     esphome_rcs: tuple[int, int] = (0, 0),
     macro_matches: bool = True,
     envs: list[dict[str, str]] | None = None,
+    versions: tuple[str | None, str | None, str] = ("5.5", "5.5", "5.5"),
 ) -> tuple[list[str], list[list[str]]]:
     """Run check() with idf.py replaced by ``side_effect``; return problems, calls.
 
-    ``envs`` collects the env each idf.py call receives.
+    ``envs`` collects the env each idf.py call receives. ``versions`` is what
+    version.txt, the version header and idf_tools report for the framework.
     """
     calls: list[list[str]] = []
 
@@ -98,6 +100,9 @@ def _run_check(
         patch.object(toolchain, "run_reconfigure", return_value=esphome_rcs[0]),
         patch.object(toolchain, "_run_ninja", return_value=esphome_rcs[1]),
         patch.object(build_gen, "idf_macro_matches", return_value=macro_matches),
+        patch.object(framework, "read_idf_version_txt", return_value=versions[0]),
+        patch.object(framework, "read_idf_version_header", return_value=versions[1]),
+        patch.object(framework, "idf_tools_version", return_value=versions[2]),
         patch.object(guard.subprocess, "run", side_effect=run),
         patch.dict(os.environ),
     ):
@@ -285,6 +290,24 @@ def test_check_fails_loudly_when_the_idf_macro_changed(tmp_path: Path) -> None:
     assert calls == []
 
 
+@pytest.mark.parametrize(
+    ("versions", "source"),
+    [(("5.4", "5.5", "5.5"), "txt"), (("5.5", None, "5.5"), "header")],
+)
+def test_check_fails_loudly_when_the_version_read_drifts(
+    tmp_path: Path, versions: tuple[str | None, str | None, str], source: str
+) -> None:
+    """An IDF bump that changes how idf_tools reads its version must fail CI;
+    both sources are checked since a managed tree never reaches the header."""
+    tree = _make_tree(tmp_path)
+    problems, calls = _run_check(tree, versions=versions)
+    ours = versions[0] if source == "txt" else versions[1]
+    assert problems == [
+        guard.VERSION_DRIFT.format(ours=ours, source=source, theirs="5.5")
+    ]
+    assert calls == []
+
+
 def test_main_without_build_trees(
     tmp_path: Path, capsys: pytest.CaptureFixture[str]
 ) -> None:

tests/unit_tests/test_espidf_framework.py

@@ -44,6 +44,7 @@ from esphome.espidf.framework import (
     check_esp_idf_install,
     get_framework_env,
     get_idf_tools_path,
+    idf_tools_version,
 )
 from esphome.framework_helpers import _tar_extract_all, get_python_env_executable_path
 
@@ -1468,19 +1469,59 @@ def test_demote_unused_tools_already_patched_is_noop(tmp_path: Path) -> None:
 # ---------------------------------------------------------------------------
 
 
-def test_get_idf_version_parses_stdout(tmp_path: Path) -> None:
+def _write_idf_version_header(root: Path, major: int, minor: int) -> None:
+    include = root / "components" / "esp_common" / "include"
+    include.mkdir(parents=True)
+    (include / "esp_idf_version.h").write_text(
+        f"#define ESP_IDF_VERSION_MAJOR   {major}\n"
+        "/** Minor version number (x.X.x) */\n"
+        f"#define ESP_IDF_VERSION_MINOR   {minor}\n"
+        "#define ESP_IDF_VERSION_PATCH   0\n",
+        encoding="utf-8",
+    )
+
+
+def test_get_idf_version_reads_version_txt(tmp_path: Path) -> None:
+    """version.txt wins and gives major.minor, as idf_tools returns it."""
+    (tmp_path / "version.txt").write_text("v5.5.5\n", encoding="utf-8")
+    _write_idf_version_header(tmp_path, 6, 1)
+    assert _get_idf_version(tmp_path) == "5.5"
+
+
+def test_get_idf_version_falls_back_to_the_header(tmp_path: Path) -> None:
+    """A version.txt that does not match (a git ref) defers to the header."""
+    (tmp_path / "version.txt").write_text("vrelease/v6.1\n", encoding="utf-8")
+    _write_idf_version_header(tmp_path, 6, 1)
+    assert _get_idf_version(tmp_path) == "6.1"
+
+
+def test_get_idf_version_raises_without_a_source(tmp_path: Path) -> None:
+    with pytest.raises(RuntimeError, match="Can't get ESP-IDF version"):
+        _get_idf_version(tmp_path)
+
+
+def test_get_idf_version_wraps_an_unreadable_source(tmp_path: Path) -> None:
+    """A source that cannot be decoded keeps the RuntimeError contract."""
+    (tmp_path / "version.txt").write_bytes(b"\xff\xfev")
+    with pytest.raises(RuntimeError, match="Can't get ESP-IDF version") as info:
+        _get_idf_version(tmp_path)
+    assert isinstance(info.value.__cause__, UnicodeError)
+
+
+def test_idf_tools_version_runs_the_framework_script(tmp_path: Path) -> None:
     with patch(
-        "esphome.espidf.framework.run_command", return_value=(True, "5.1.2\n", "")
-    ):
-        assert _get_idf_version(tmp_path) == "5.1.2"
+        "esphome.espidf.framework.run_command", return_value=(True, "5.5\n", "")
+    ) as run:
+        assert idf_tools_version(tmp_path) == "5.5"
+    assert run.call_args.args[0][1].endswith("get_idf_version.py")
 
 
-def test_get_idf_version_raises_on_failure(tmp_path: Path) -> None:
+def test_idf_tools_version_raises_on_failure(tmp_path: Path) -> None:
     with (
         patch("esphome.espidf.framework.run_command", return_value=(False, "", "boom")),
         pytest.raises(RuntimeError, match="Can't get ESP-IDF version"),
     ):
-        _get_idf_version(tmp_path)
+        idf_tools_version(tmp_path)
 
 
 def test_get_idf_tool_paths_parses_json(tmp_path: Path) -> None: