From b8690c8e31600201439a6ffb8325c436cde8a46e Mon Sep 17 00:00:00 2001 From: Jonathan Swoboda <154711427+swoboda1337@users.noreply.github.com> Date: Mon, 29 Jun 2026 12:32:28 -0400 Subject: [PATCH] [core] Drop Python 3.11 support (#17280) --- .github/workflows/ci-api-proto.yml | 2 +- .github/workflows/ci-docker.yml | 4 +- .../workflows/ci-memory-impact-comment.yml | 2 +- .github/workflows/ci.yml | 6 +- .github/workflows/release.yml | 2 +- .pre-commit-config.yaml | 2 +- AGENTS.md | 2 +- esphome/async_thread.py | 9 +- esphome/components/nrf52/framework.py | 17 +-- esphome/framework_helpers.py | 124 ++++-------------- esphome/helpers.py | 10 +- esphome/platformio/library.py | 7 +- pyproject.toml | 6 +- script/lint-python | 2 +- tests/integration/state_utils.py | 5 +- tests/unit_tests/test_framework_helpers.py | 10 -- 16 files changed, 58 insertions(+), 152 deletions(-) diff --git a/.github/workflows/ci-api-proto.yml b/.github/workflows/ci-api-proto.yml index 17234e811a..4c0c330a19 100644 --- a/.github/workflows/ci-api-proto.yml +++ b/.github/workflows/ci-api-proto.yml @@ -25,7 +25,7 @@ jobs: - name: Set up Python uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 with: - python-version: "3.11" + python-version: "3.12" - name: Set up uv # ``--system`` (below) installs into the setup-python interpreter; # no venv is created or restored by this workflow. diff --git a/.github/workflows/ci-docker.yml b/.github/workflows/ci-docker.yml index 9678831b50..d6ad28dffe 100644 --- a/.github/workflows/ci-docker.yml +++ b/.github/workflows/ci-docker.yml @@ -65,7 +65,7 @@ jobs: - name: Set up Python uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 with: - python-version: "3.11" + python-version: "3.12" - name: Set up Docker Buildx uses: docker/setup-buildx-action@d7f5e7f509e45cec5c76c4d5afdd7de93d0b3df5 # v4.1.0 @@ -149,7 +149,7 @@ jobs: - name: Set up Python uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 with: - python-version: "3.11" + python-version: "3.12" - name: Set up Docker Buildx uses: docker/setup-buildx-action@d7f5e7f509e45cec5c76c4d5afdd7de93d0b3df5 # v4.1.0 diff --git a/.github/workflows/ci-memory-impact-comment.yml b/.github/workflows/ci-memory-impact-comment.yml index 4bef082aab..ac0322e2fa 100644 --- a/.github/workflows/ci-memory-impact-comment.yml +++ b/.github/workflows/ci-memory-impact-comment.yml @@ -60,7 +60,7 @@ jobs: if: steps.pr.outputs.skip != 'true' uses: ./.github/actions/restore-python with: - python-version: "3.11" + python-version: "3.12" cache-key: ${{ hashFiles('.cache-key') }} - name: Download memory analysis artifacts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 72519e421a..751241f563 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,8 +12,8 @@ permissions: contents: read # actions/checkout for all jobs; individual jobs add their own scopes when they need to write env: - DEFAULT_PYTHON: "3.11" - PYUPGRADE_TARGET: "--py311-plus" + DEFAULT_PYTHON: "3.12" + PYUPGRADE_TARGET: "--py312-plus" concurrency: # yamllint disable-line rule:line-length @@ -203,7 +203,7 @@ jobs: fail-fast: false matrix: python-version: - - "3.11" + - "3.12" - "3.13" - "3.14" os: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 2b23b561bd..20a77b152d 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -96,7 +96,7 @@ jobs: - name: Set up Python uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 with: - python-version: "3.11" + python-version: "3.12" - name: Set up Docker Buildx uses: docker/setup-buildx-action@d7f5e7f509e45cec5c76c4d5afdd7de93d0b3df5 # v4.1.0 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index ba74aff07c..da424f516f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -40,7 +40,7 @@ repos: rev: v3.21.2 hooks: - id: pyupgrade - args: [--py311-plus] + args: [--py312-plus] - repo: https://github.com/adrienverge/yamllint.git rev: v1.37.1 hooks: diff --git a/AGENTS.md b/AGENTS.md index 21905ea356..46caea3aec 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -9,7 +9,7 @@ This document provides essential context for AI models interacting with this pro ## 2. Core Technologies & Stack -* **Languages:** Python (>=3.11), C++ (gnu++20) +* **Languages:** Python (>=3.12), C++ (gnu++20) * **Frameworks & Runtimes:** PlatformIO, Arduino, ESP-IDF. * **Build Systems:** PlatformIO is the primary build system. CMake is used as an alternative. * **Configuration:** YAML. diff --git a/esphome/async_thread.py b/esphome/async_thread.py index c5225a7a14..3972d735f5 100644 --- a/esphome/async_thread.py +++ b/esphome/async_thread.py @@ -12,12 +12,9 @@ from __future__ import annotations import asyncio from collections.abc import Awaitable, Callable import threading -from typing import Generic, TypeVar - -_T = TypeVar("_T") -class AsyncThreadRunner(threading.Thread, Generic[_T]): +class AsyncThreadRunner[T](threading.Thread): """Run an async coroutine in a daemon thread and expose its result. The runner catches all exceptions from the coroutine and stores them in @@ -35,10 +32,10 @@ class AsyncThreadRunner(threading.Thread, Generic[_T]): result = runner.result """ - def __init__(self, coro_factory: Callable[[], Awaitable[_T]]) -> None: + def __init__(self, coro_factory: Callable[[], Awaitable[T]]) -> None: super().__init__(daemon=True) self._coro_factory = coro_factory - self.result: _T | None = None + self.result: T | None = None self.exception: BaseException | None = None self.event = threading.Event() diff --git a/esphome/components/nrf52/framework.py b/esphome/components/nrf52/framework.py index 05feadb001..7aec6b088e 100644 --- a/esphome/components/nrf52/framework.py +++ b/esphome/components/nrf52/framework.py @@ -54,22 +54,15 @@ def _get_toolchain_path(version: str) -> Path: return _get_tools_path() / "toolchains" / version -# onexc/dir_fd were added to shutil.rmtree in 3.12; the 3.11 branch uses onerror. _SITECUSTOMIZE = """\ -import os, stat, shutil, sys +import os, stat, shutil _orig = shutil.rmtree def _handler(func, path, exc): os.chmod(path, stat.S_IWRITE); func(path) -if sys.version_info >= (3, 12): - def _rmtree(path, ignore_errors=False, onerror=None, *, onexc=None, dir_fd=None): - if onerror is None and onexc is None: - onexc = _handler - return _orig(path, ignore_errors=ignore_errors, onerror=onerror, onexc=onexc, dir_fd=dir_fd) -else: - def _rmtree(path, ignore_errors=False, onerror=None): - if onerror is None: - onerror = _handler - return _orig(path, ignore_errors=ignore_errors, onerror=onerror) +def _rmtree(path, ignore_errors=False, onerror=None, *, onexc=None, dir_fd=None): + if onerror is None and onexc is None: + onexc = _handler + return _orig(path, ignore_errors=ignore_errors, onerror=onerror, onexc=onexc, dir_fd=dir_fd) shutil.rmtree = _rmtree """ diff --git a/esphome/framework_helpers.py b/esphome/framework_helpers.py index a8e5cf75a8..69cecc58e2 100644 --- a/esphome/framework_helpers.py +++ b/esphome/framework_helpers.py @@ -239,22 +239,19 @@ def _tar_extract_all( """ Extract a TAR archive to the specified directory. - Implementation is inspired by Python 3.12's tarfile data filtering logic. - This can be replaced with the standard library implementation once - support for Python 3.11 is no longer required. + Path-traversal, link, permission and ownership sanitization is delegated to + the stdlib ``tarfile.data_filter`` (PEP 706). We keep the wrapper-directory + stripping (no stdlib equivalent) and the absolute-path reject (data_filter's + check is os.path-dependent and would miss a Windows drive path when + extracting on POSIX). Args: data: File-like object containing the TAR archive extract_dir: Directory to extract contents to progress_header: If set, show a progress bar with this header """ - import stat import tarfile - # Tar extraction safety: os.path.realpath / commonpath / normpath have no - # pathlib equivalents and Path.resolve() would follow symlinks unsafely. - # Use os.path for the security-sensitive parts; the simple checks move to - # Path. extract_dir = os.fspath(extract_dir) abs_dest = os.path.abspath(extract_dir) # noqa: PTH100 @@ -269,18 +266,14 @@ def _tar_extract_all( safe_members = [] for member in all_members: - name = member.name - - # 1. Strip leading slashes - name = name.lstrip("/" + os.sep) - - # 2. Reject absolute paths (incl. Windows drive) + # Strip leading slashes, then reject absolute / Windows-drive paths + name = member.name.lstrip("/" + os.sep) if Path(name).is_absolute() or ( os.name == "nt" and ":" in name.split(os.sep)[0] # noqa: PTH206 ): continue - # 3. Strip wrapper directory if one was detected + # Strip wrapper directory if one was detected if strip_prefix is not None: norm = name.replace("\\", "/") if norm in (strip_root, strip_prefix): @@ -288,88 +281,29 @@ def _tar_extract_all( if not norm.startswith(strip_prefix): continue name = norm[len(strip_prefix) :] - - # 4. Compute final path - target_path = os.path.realpath(os.path.join(abs_dest, name)) # noqa: PTH118 - if os.path.commonpath([abs_dest, target_path]) != abs_dest: - continue - - # 5. Validate links properly - if member.issym() or member.islnk(): - linkname = member.linkname - - # Reject absolute link targets - if Path(linkname).is_absolute(): - continue - - if member.islnk() and strip_prefix is not None: - # Hard-link linknames reference another archive member - # by its archive name. We've stripped the wrapper prefix - # from member.name above (step 3); strip it here too so - # tarfile._find_link_target can resolve the target during - # extraction. Symlink linknames are filesystem-relative - # paths, not archive-member references, so they don't - # need this treatment. - norm_link = linkname.replace("\\", "/") - if norm_link in (strip_root, strip_prefix): - continue - if not norm_link.startswith(strip_prefix): - continue - linkname = norm_link[len(strip_prefix) :] - - # Strip leading slashes - linkname = os.path.normpath(linkname) - - if member.issym(): - link_target = os.path.join( # noqa: PTH118 - abs_dest, - os.path.dirname(name), # noqa: PTH120 - linkname, - ) - else: - link_target = os.path.join(abs_dest, linkname) # noqa: PTH118 - link_target = os.path.realpath(link_target) - - if os.path.commonpath([abs_dest, link_target]) != abs_dest: - continue - - # write back normalized linkname - member.linkname = linkname - - # 6. Sanitize permissions - mode = member.mode - if mode is not None: - # Strip high bits & group/other write bits - mode &= ( - stat.S_IRWXU - | stat.S_IRGRP - | stat.S_IXGRP - | stat.S_IROTH - | stat.S_IXOTH - ) - if member.isfile() or member.islnk(): - # remove exec bits unless explicitly user-executable - if not (mode & stat.S_IXUSR): - mode &= ~(stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH) - mode |= stat.S_IRUSR | stat.S_IWUSR - elif not (member.isdir() or member.issym()): - # Block special files. Directories and symlinks keep - # their masked-original mode — passing None here would - # crash tarfile.extract on Python <3.12 (its chmod - # path calls os.chmod unconditionally). - continue - - member.mode = mode - - # 7. Strip ownership - member.uid = None - member.gid = None - member.uname = None - member.gname = None - - # 8. Assign sanitized name back member.name = name + # Hard-link linknames reference another archive member by its + # archive name; strip the wrapper prefix here too so + # tarfile._find_link_target can resolve the target during + # extraction. Symlink linknames are filesystem-relative paths, + # not archive-member references, so they don't need this. + if member.islnk() and strip_prefix is not None: + norm_link = member.linkname.replace("\\", "/") + if norm_link in (strip_root, strip_prefix): + continue + if not norm_link.startswith(strip_prefix): + continue + member.linkname = norm_link[len(strip_prefix) :] + + # Delegate traversal, link, permission and ownership sanitization + # to the stdlib data filter; it raises FilterError for unsafe + # members (path traversal, links outside dest, special files). + try: + member = tarfile.data_filter(member, abs_dest) + except tarfile.FilterError: + continue + safe_members.append(member) total = len(safe_members) diff --git a/esphome/helpers.py b/esphome/helpers.py index 62dfd0fb09..631bcb6f39 100644 --- a/esphome/helpers.py +++ b/esphome/helpers.py @@ -397,17 +397,13 @@ def rmtree(path: Path | str) -> None: read-only flag and retrying. """ - def _onerror(func, path, exc_info): + def _onexc(func, path, exc): if os.access(path, os.W_OK): - raise exc_info[1].with_traceback(exc_info[2]) + raise exc Path(path).chmod(stat.S_IWUSR | stat.S_IRUSR) func(path) - # ``onerror`` is deprecated in 3.12 in favour of ``onexc`` (different - # callable signature); keep the existing handler shape for now and - # silence the lint locally so this PR doesn't bundle an unrelated - # migration. - shutil.rmtree(path, onerror=_onerror) # pylint: disable=deprecated-argument + shutil.rmtree(path, onexc=_onexc) def walk_files(path: Path): diff --git a/esphome/platformio/library.py b/esphome/platformio/library.py index 43282c7aa0..c2d783ecbe 100644 --- a/esphome/platformio/library.py +++ b/esphome/platformio/library.py @@ -24,7 +24,7 @@ import os from pathlib import Path import re import tempfile -from typing import Any, TypeVar +from typing import Any from urllib.parse import urlparse, urlsplit, urlunsplit from esphome import git @@ -195,10 +195,7 @@ class LibraryBackend: emit: Callable[["ConvertedLibrary"], None] -T = TypeVar("T") - - -def ensure_list(obj: T | list[T]) -> list[T]: +def ensure_list[T](obj: T | list[T]) -> list[T]: """ Convert an object to a list if it isn't already a list. diff --git a/pyproject.toml b/pyproject.toml index a292377835..e959578553 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,7 +20,7 @@ classifiers = [ "Topic :: Home Automation", ] -requires-python = ">=3.11.0,<3.15" +requires-python = ">=3.12.0,<3.15" dynamic = ["dependencies", "optional-dependencies", "version"] @@ -62,7 +62,7 @@ addopts = [ ] [tool.pylint.MAIN] -py-version = "3.11" +py-version = "3.12" ignore = [ "api_pb2.py", ] @@ -106,7 +106,7 @@ expected-line-ending-format = "LF" [tool.ruff] required-version = ">=0.5.0" -target-version = "py311" +target-version = "py312" exclude = ['generated'] [tool.ruff.lint] diff --git a/script/lint-python b/script/lint-python index e4b3314d2a..6bd95778fa 100755 --- a/script/lint-python +++ b/script/lint-python @@ -139,7 +139,7 @@ def main(): print() print("Running pyupgrade...") print() - PYUPGRADE_TARGET = "--py311-plus" + PYUPGRADE_TARGET = "--py312-plus" for files in filesets: cmd = ["pyupgrade", PYUPGRADE_TARGET] + files log = get_err(*cmd) diff --git a/tests/integration/state_utils.py b/tests/integration/state_utils.py index c8517aff09..65af57b944 100644 --- a/tests/integration/state_utils.py +++ b/tests/integration/state_utils.py @@ -19,7 +19,6 @@ from aioesphomeapi import ( _LOGGER = logging.getLogger(__name__) -T = TypeVar("T", bound=EntityInfo) S = TypeVar("S", bound=EntityState) @@ -58,7 +57,7 @@ async def wait_for_state( return await asyncio.wait_for(future, timeout=timeout) -def find_entity( +def find_entity[T: EntityInfo]( entities: list[EntityInfo], object_id_substring: str, entity_type: type[T] | None = None, @@ -86,7 +85,7 @@ def find_entity( return None -def require_entity( +def require_entity[T: EntityInfo]( entities: list[EntityInfo], object_id_substring: str, entity_type: type[T] | None = None, diff --git a/tests/unit_tests/test_framework_helpers.py b/tests/unit_tests/test_framework_helpers.py index fd807ed05d..6fe62dcc8c 100644 --- a/tests/unit_tests/test_framework_helpers.py +++ b/tests/unit_tests/test_framework_helpers.py @@ -658,11 +658,6 @@ def test_get_python_env_executable_path_nt() -> None: class TestTarExtractAllBranches: - @pytest.mark.skipif( - sys.version_info < (3, 12), - reason="patching os.name makes pathlib build a WindowsPath, which only " - "instantiates on POSIX in 3.12+", - ) def test_windows_drive_path_skipped(self, tmp_path: Path) -> None: """Windows-style drive path (C:/...) is skipped when os.name == 'nt'.""" info = tarfile.TarInfo(name="C:/secret.txt") @@ -755,11 +750,6 @@ class TestTarExtractAllBranches: class TestZipExtractAllBranches: - @pytest.mark.skipif( - sys.version_info < (3, 12), - reason="patching os.name makes pathlib build a WindowsPath, which only " - "instantiates on POSIX in 3.12+", - ) def test_windows_drive_path_skipped(self, tmp_path: Path) -> None: """Windows-style drive path (C:/...) is skipped when os.name == 'nt'.""" buf = _make_zip([("C:/secret.txt", "bad")])