github/esphome
2
0
Fork 0
mirror of https://github.com/esphome/esphome.git synced 2026-09-14 08:38:39 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/cv-sensitive-redact-sentinel' into integration

Browse Source
This commit is contained in:
J. Nick Koston
2026-05-26 23:51:13 -05:00
parent 49fe864d12 4c6437b825
commit 3ce95682ca
211 changed files with 4484 additions and 1257 deletions
Download Patch File Download Diff File Expand all files Collapse all files
script/api_protobuf/api_protobuf.py
+14 -19
View File
@@ -84,12 +84,7 @@ def indent_list(text: str, padding: str = " ") -> list[str]:
"""Indent each line of the given text with the specified padding."""
lines = []
for line in text.splitlines():
if (
line == ""
or line.startswith("#ifdef")
or line.startswith("#if ")
or line.startswith("#endif")
):
if line == "" or line.startswith(("#ifdef", "#if ", "#endif")):
p = ""
else:
p = padding
@@ -1283,11 +1278,11 @@ class PackedBufferTypeInfo(TypeInfo):
"""Dump shows buffer info but not decoded values."""
return (
f'out.append(2, \' \').append_p(ESPHOME_PSTR("{self.name}")).append(": ");\n'
+ 'out.append_p(ESPHOME_PSTR("packed buffer ["));\n'
+ f"append_uint(out, this->{self.field_name}_count_);\n"
+ 'out.append_p(ESPHOME_PSTR(" values, "));\n'
+ f"append_uint(out, this->{self.field_name}_length_);\n"
+ 'out.append_p(ESPHOME_PSTR(" bytes]\\n"));'
'out.append_p(ESPHOME_PSTR("packed buffer ["));\n'
f"append_uint(out, this->{self.field_name}_count_);\n"
'out.append_p(ESPHOME_PSTR(" values, "));\n'
f"append_uint(out, this->{self.field_name}_length_);\n"
'out.append_p(ESPHOME_PSTR(" bytes]\\n"));'
)
def dump(self, name: str) -> str:
@@ -3163,7 +3158,7 @@ def main() -> None:
defines_content += "\n"
defines_content += "\nnamespace esphome::api {} // namespace esphome::api\n"
with open(root / "api_pb2_defines.h", "w", encoding="utf-8") as f:
with (root / "api_pb2_defines.h").open("w", encoding="utf-8") as f:
f.write(defines_content)
content = FILE_HEADER
@@ -3448,13 +3443,13 @@ static void dump_bytes_field(DumpBuffer &out, const char *field_name, const uint
#endif // HAS_PROTO_MESSAGE_DUMP
"""
with open(root / "api_pb2.h", "w", encoding="utf-8") as f:
with (root / "api_pb2.h").open("w", encoding="utf-8") as f:
f.write(content)
with open(root / "api_pb2.cpp", "w", encoding="utf-8") as f:
with (root / "api_pb2.cpp").open("w", encoding="utf-8") as f:
f.write(cpp)
with open(root / "api_pb2_dump.cpp", "w", encoding="utf-8") as f:
with (root / "api_pb2_dump.cpp").open("w", encoding="utf-8") as f:
f.write(dump_cpp)
hpp = FILE_HEADER
@@ -3551,7 +3546,7 @@ static const char *const TAG = "api.service";
if id_ is not None and not mt.options.deprecated:
id_to_msg_name[id_] = mt.name
for id_, (_, _, case_label) in cases:
for id_, (_, _, _case_label) in cases:
msg_name = id_to_msg_name.get(id_, "")
if msg_name in message_auth_map:
needs_auth = message_auth_map[msg_name]
@@ -3614,7 +3609,7 @@ static const char *const TAG = "api.service";
# Dispatch switch
out += " switch (msg_type) {\n"
for i, (case, ifdef, case_label) in cases:
for _i, (case, ifdef, case_label) in cases:
if ifdef is not None:
out += _make_ifdef_line(ifdef) + "\n"
@@ -3641,10 +3636,10 @@ static const char *const TAG = "api.service";
} // namespace esphome::api
"""
with open(root / "api_pb2_service.h", "w", encoding="utf-8") as f:
with (root / "api_pb2_service.h").open("w", encoding="utf-8") as f:
f.write(hpp)
with open(root / "api_pb2_service.cpp", "w", encoding="utf-8") as f:
with (root / "api_pb2_service.cpp").open("w", encoding="utf-8") as f:
f.write(cpp)
prot_file.unlink()
script/build_helpers.py
+1 -1
View File
@@ -195,7 +195,7 @@ def load_component_yaml_configs(components: list[str], tests_dir: Path) -> dict:
yaml_path = tests_dir / component / BENCHMARK_YAML_FILENAME
if not yaml_path.is_file():
continue
with open(yaml_path) as f:
with yaml_path.open() as f:
component_config = yaml.safe_load(f)
if component_config and isinstance(component_config, dict):
for key, value in component_config.items():
script/build_language_schema.py
+38 -3
View File
@@ -39,7 +39,11 @@ parser.add_argument(
)
parser.add_argument("--check", action="store_true", help="Check only for CI")
args = parser.parse_args()
# Module-level ``Namespace`` so helper functions can reference ``args``
# without threading it through every call. ``main()`` fills it via
# ``parser.parse_args(namespace=args)``; tests import this module without
# invoking ``main()`` and rely on the defaults below.
args = argparse.Namespace(output_path=".", check=False)
DUMP_RAW = False
DUMP_UNKNOWN = False
@@ -850,6 +854,12 @@ def convert(schema, config_var, path):
convert(ext, config_var, f"{path}/ext{idx}")
return
if isinstance(schema, cv.SensitiveValidator):
config_var["sensitive"] = True
config_var["sensitive_source"] = "explicit"
convert(schema.inner, config_var, f"{path}/sensitive")
return
if isinstance(schema, cv.All):
i = 0
for inner in schema.validators:
@@ -972,7 +982,7 @@ def convert(schema, config_var, path):
}
elif schema_type == "use_id":
if inspect.ismodule(data):
m_attr_obj = getattr(data, "CONFIG_SCHEMA")
m_attr_obj = data.CONFIG_SCHEMA
use_schema = known_schemas.get(repr(m_attr_obj))
if use_schema:
[output_module, output_name] = use_schema[0][1].split(".")
@@ -1125,6 +1135,25 @@ def convert_keys(converted, schema, path):
# Do value
convert(v, result, path + f"/{str(k)}")
# Heuristic fallback when the field's validator wasn't explicitly
# wrapped in ``cv.sensitive``. Only applies to string-typed leaves so
# we don't mark unrelated nested schemas. ``sensitive_source`` lets
# consumers distinguish explicit markers from heuristic matches. Pull
# the field name from ``k.schema`` (voluptuous's stored key) rather
# than ``str(k)`` so we don't depend on the marker's ``__str__``
# representation.
if (
"sensitive" not in result
and result.get(S_TYPE) == "string"
and isinstance(k, (cv.Required, cv.Optional, cv.Inclusive, cv.Exclusive))
and isinstance(k.schema, str)
):
key_lower = k.schema.lower()
if any(frag in key_lower for frag in cv.SENSITIVE_KEY_FRAGMENTS):
result["sensitive"] = True
result["sensitive_source"] = "heuristic"
if "schema" not in converted:
converted[S_TYPE] = "schema"
converted["schema"] = {S_CONFIG_VARS: {}}
@@ -1142,4 +1171,10 @@ def convert_keys(converted, schema, path):
config_vars["string"] = config_vars.pop(key)
build_schema()
def main() -> None:
parser.parse_args(namespace=args)
build_schema()
if __name__ == "__main__":
main()
script/bump-version.py
+3 -2
View File
@@ -2,6 +2,7 @@
import argparse
from dataclasses import dataclass
from pathlib import Path
import re
import sys
@@ -39,12 +40,12 @@ class Version:
def sub(path, pattern, repl, expected_count=1):
with open(path, encoding="utf-8") as fh:
with Path(path).open(encoding="utf-8") as fh:
content = fh.read()
content, count = re.subn(pattern, repl, content, flags=re.MULTILINE)
if expected_count is not None:
assert count == expected_count, f"Pattern {pattern} replacement failed!"
with open(path, "w", encoding="utf-8") as fh:
with Path(path).open("w", encoding="utf-8") as fh:
fh.write(content)
script/ci-custom.py
+5 -18
View File
@@ -14,7 +14,7 @@ import time
import colorama
from helpers import filter_changed, git_ls_files, print_error_for_file, styled
sys.path.append(os.path.dirname(__file__))
sys.path.append(str(Path(__file__).parent))
def find_all(a_str, sub):
@@ -341,9 +341,9 @@ def lint_const_ordered(fname, content):
matching = [
(i + 1, line) for i, line in enumerate(lines) if line.startswith(start)
]
ordered = list(sorted(matching, key=lambda x: x[1].replace("_", " ")))
ordered = [(mi, ol) for (mi, _), (_, ol) in zip(matching, ordered)]
for (mi, mline), (_, ol) in zip(matching, ordered):
ordered = sorted(matching, key=lambda x: x[1].replace("_", " "))
ordered = [(mi, ol) for (mi, _), (_, ol) in zip(matching, ordered, strict=True)]
for (mi, mline), (_, ol) in zip(matching, ordered, strict=True):
if mline == ol:
continue
target = next(i for i, line in ordered if line == mline)
@@ -562,7 +562,7 @@ def lint_constants_usage():
# Maximum allowed CONF_ constants in esphome/const.py.
# This file is frozen — new constants go in esphome/components/const/__init__.py.
# Decrease this number when constants are moved out of const.py.
CONST_PY_MAX_CONF = 1012
CONST_PY_MAX_CONF = 1013
@lint_content_check(include=["esphome/const.py"])
@@ -693,19 +693,6 @@ def lint_esphome_h(fname, line, col, content):
)
@lint_content_find_check(
"CORE.using_esp_idf",
include=py_include,
exclude=["esphome/core/__init__.py", "script/ci-custom.py"],
)
def lint_using_esp_idf_deprecated(fname, line, col, content):
return (
f"{highlight('CORE.using_esp_idf')} is deprecated and will change behavior in 2026.6. "
"ESP32 Arduino builds on top of ESP-IDF, so ESP-IDF features are available in both frameworks. "
f"Please use {highlight('CORE.is_esp32')} and/or {highlight('CORE.using_arduino')} instead."
)
@lint_content_check(include=["*.h"], exclude=["esphome/core/entity_types.h"])
def lint_pragma_once(fname, content):
if "#pragma once" not in content:
script/ci_add_metadata_to_json.py
+2 -2
View File
@@ -44,7 +44,7 @@ def main() -> int:
return 1
try:
with open(json_path, encoding="utf-8") as f:
with Path(json_path).open(encoding="utf-8") as f:
data = json.load(f)
except (json.JSONDecodeError, OSError) as e:
print(f"Error loading JSON: {e}", file=sys.stderr)
@@ -74,7 +74,7 @@ def main() -> int:
# Write back
try:
with open(json_path, "w", encoding="utf-8") as f:
with Path(json_path).open("w", encoding="utf-8") as f:
json.dump(data, f, indent=2)
print(f"Added metadata to {args.json_file}", file=sys.stderr)
except OSError as e:
script/ci_helpers.py
Executable → Regular
+2 -1
View File
@@ -3,6 +3,7 @@
from __future__ import annotations
import os
from pathlib import Path
def write_github_output(outputs: dict[str, str | int]) -> None:
@@ -16,7 +17,7 @@ def write_github_output(outputs: dict[str, str | int]) -> None:
"""
github_output = os.environ.get("GITHUB_OUTPUT")
if github_output:
with open(github_output, "a", encoding="utf-8") as f:
with Path(github_output).open("a", encoding="utf-8") as f:
f.writelines(f"{key}={value}\n" for key, value in outputs.items())
else:
for key, value in outputs.items():
script/ci_memory_impact_comment.py
+1 -1
View File
@@ -91,7 +91,7 @@ def load_analysis_json(json_path: str) -> dict | None:
return None
try:
with open(json_file, encoding="utf-8") as f:
with Path(json_file).open(encoding="utf-8") as f:
return json.load(f)
except (json.JSONDecodeError, OSError) as e:
print(f"Failed to load analysis JSON: {e}", file=sys.stderr)
script/ci_memory_impact_extract.py
+2 -2
View File
@@ -127,7 +127,7 @@ def run_detailed_analysis(build_dir: str) -> dict | None:
if not idedata_path.exists():
continue
try:
with open(idedata_path, encoding="utf-8") as f:
with idedata_path.open(encoding="utf-8") as f:
raw_data = json.load(f)
idedata = IDEData(raw_data)
print(f"Loaded idedata from: {idedata_path}", file=sys.stderr)
@@ -264,7 +264,7 @@ def main() -> int:
output_path = Path(args.output_json)
output_path.parent.mkdir(parents=True, exist_ok=True)
with open(output_path, "w", encoding="utf-8") as f:
with output_path.open("w", encoding="utf-8") as f:
json.dump(output_data, f, indent=2)
print(f"Saved analysis to {args.output_json}", file=sys.stderr)
script/clang-format
+2 -1
View File
@@ -2,6 +2,7 @@
import argparse
import os
from pathlib import Path
import queue
import re
import subprocess
@@ -70,7 +71,7 @@ def main():
)
args = parser.parse_args()
cwd = os.getcwd()
cwd = Path.cwd()
files = [
os.path.relpath(path, cwd) for path in git_ls_files(["*.cpp", "*.h", "*.tcc"])
]
script/clang-tidy
+5 -4
View File
@@ -2,6 +2,7 @@
import argparse
import os
from pathlib import Path
import queue
import re
import shutil
@@ -32,7 +33,7 @@ def clang_options(idedata):
cmd = []
# extract target architecture from triplet in g++ filename
triplet = os.path.basename(idedata["cxx_path"])[:-4]
triplet = Path(idedata["cxx_path"]).name[:-4]
if triplet.startswith("xtensa-"):
# clang doesn't support Xtensa (yet?), so compile in 32-bit mode and pretend we're the Xtensa compiler
cmd.append("-m32")
@@ -153,8 +154,8 @@ def run_tidy(executable, args, options, tmpdir, path_queue, lock, failed_files):
if sys.stdout.isatty():
invocation.append("--use-color")
invocation.append(f"--header-filter={os.path.abspath(basepath)}/.*")
invocation.append(os.path.abspath(path))
invocation.append(f"--header-filter={Path(basepath).resolve()}/.*")
invocation.append(str(Path(path).resolve()))
invocation.append("--")
invocation.extend(options)
@@ -229,7 +230,7 @@ def main():
)
args = parser.parse_args()
cwd = os.getcwd()
cwd = Path.cwd()
files = [os.path.relpath(path, cwd) for path in git_ls_files(["*.cpp"])]
# Exclude benchmark files — they require google benchmark headers not
# available in the ESP32 toolchain and use different naming conventions.
script/clang_tidy_hash.py
+3 -3
View File
@@ -16,7 +16,7 @@ sys.path.insert(0, str(script_dir))
def read_file_lines(path: Path) -> list[str]:
"""Read lines from a file."""
with open(path) as f:
with path.open() as f:
return f.readlines()
@@ -65,7 +65,7 @@ def get_clang_tidy_version_from_requirements(repo_root: Path | None = None) -> s
def read_file_bytes(path: Path) -> bytes:
"""Read bytes from a file."""
with open(path, "rb") as f:
with path.open("rb") as f:
return f.read()
@@ -120,7 +120,7 @@ def read_stored_hash(repo_root: Path | None = None) -> str | None:
def write_file_content(path: Path, content: str) -> None:
"""Write content to a file."""
with open(path, "w") as f:
with path.open("w") as f:
f.write(content)
script/determine-jobs.py
+5 -9
View File
@@ -306,7 +306,7 @@ def _is_clang_tidy_full_scan() -> bool:
"""
try:
result = subprocess.run(
[os.path.join(root_path, "script", "clang_tidy_hash.py"), "--check"],
[str(Path(root_path) / "script" / "clang_tidy_hash.py"), "--check"],
capture_output=True,
check=False,
)
@@ -483,9 +483,7 @@ def should_run_device_builder(branch: str | None = None) -> bool:
True if the device-builder downstream tests should run, False otherwise.
"""
target_branch = get_target_branch()
if target_branch and (
target_branch.startswith("release") or target_branch.startswith("beta")
):
if target_branch and (target_branch.startswith(("release", "beta"))):
return False
for file in changed_files(branch):
@@ -955,9 +953,7 @@ def detect_memory_impact_config(
# all components at once would produce nonsensical memory impact results.
# Memory impact analysis is most useful for focused PRs targeting dev.
target_branch = get_target_branch()
if target_branch and (
target_branch.startswith("release") or target_branch.startswith("beta")
):
if target_branch and (target_branch.startswith(("release", "beta"))):
print(
f"Memory impact: Skipping analysis for target branch {target_branch} "
f"(would try to build all components at once, giving nonsensical results)",
@@ -1047,7 +1043,7 @@ def detect_memory_impact_config(
# Find common platforms supported by ALL components
# This ensures we can build all components together in a merged config
common_platforms = set(MEMORY_IMPACT_PLATFORM_PREFERENCE)
for component, platforms in component_platforms_map.items():
for platforms in component_platforms_map.values():
common_platforms &= platforms
# Select the most preferred platform from the common set
@@ -1311,7 +1307,7 @@ def main() -> None:
# (no isolation, all components are groupable)
target_branch = get_target_branch()
is_release_branch = target_branch and (
target_branch.startswith("release") or target_branch.startswith("beta")
target_branch.startswith(("release", "beta"))
)
if is_release_branch:
script/extract_automations.py
+3 -3
View File
@@ -12,9 +12,9 @@ if __name__ == "__main__":
components = get_components_with_dependencies(files, True)
dump = {
"actions": sorted(list(ACTION_REGISTRY.keys())),
"conditions": sorted(list(CONDITION_REGISTRY.keys())),
"pin_providers": sorted(list(PIN_SCHEMA_REGISTRY.keys())),
"actions": sorted(ACTION_REGISTRY.keys()),
"conditions": sorted(CONDITION_REGISTRY.keys()),
"pin_providers": sorted(PIN_SCHEMA_REGISTRY.keys()),
}
print(json.dumps(dump, indent=2))
script/helpers.py
+12 -13
View File
@@ -17,10 +17,10 @@ from typing import Any
import colorama
root_path = os.path.abspath(os.path.normpath(os.path.join(__file__, "..", "..")))
basepath = os.path.join(root_path, "esphome")
temp_folder = os.path.join(root_path, ".temp")
temp_header_file = os.path.join(temp_folder, "all-include.cpp")
root_path = str(Path(__file__).resolve().parent.parent)
basepath = str(Path(root_path) / "esphome")
temp_folder = str(Path(root_path) / ".temp")
temp_header_file = str(Path(temp_folder) / "all-include.cpp")
# C++ file extensions used for clang-tidy and clang-format checks
CPP_FILE_EXTENSIONS = (".cpp", ".h", ".hpp", ".cc", ".cxx", ".c", ".tcc")
@@ -103,9 +103,7 @@ def get_component_from_path(file_path: str) -> str | None:
Returns:
Component name if path is in components or tests directory, None otherwise
"""
if file_path.startswith(ESPHOME_COMPONENTS_PATH) or file_path.startswith(
ESPHOME_TESTS_COMPONENTS_PATH
):
if file_path.startswith((ESPHOME_COMPONENTS_PATH, ESPHOME_TESTS_COMPONENTS_PATH)):
parts = file_path.split("/")
if len(parts) >= 3 and parts[2]:
# Verify that parts[2] is actually a component directory, not a file
@@ -160,7 +158,7 @@ def is_validate_only_file(test_file: Path) -> bool:
``esphome config`` only and skipped during compile.
"""
name = test_file.name
return name.startswith("validate.") or name.startswith("validate-")
return name.startswith(("validate.", "validate-"))
@dataclass(frozen=True)
@@ -345,8 +343,8 @@ def _get_github_event_data() -> dict | None:
Parsed event data dictionary, or None if not available
"""
github_event_path = os.environ.get("GITHUB_EVENT_PATH")
if github_event_path and os.path.exists(github_event_path):
with open(github_event_path) as f:
if github_event_path and Path(github_event_path).exists():
with Path(github_event_path).open() as f:
return json.load(f)
return None
@@ -470,7 +468,8 @@ def _get_changed_files_from_command(command: list[str]) -> list[str]:
raise Exception(f"Command failed: {' '.join(command)}\nstderr: {proc.stderr}")
changed_files = splitlines_no_ends(proc.stdout)
changed_files = [os.path.relpath(f, os.getcwd()) for f in changed_files if f]
cwd = Path.cwd()
changed_files = [os.path.relpath(f, cwd) for f in changed_files if f] # noqa: PTH109
changed_files.sort()
return changed_files
@@ -505,7 +504,7 @@ def get_changed_components() -> list[str] | None:
return None
# Use list-components.py to get changed components
script_path = os.path.join(root_path, "script", "list-components.py")
script_path = str(Path(root_path) / "script" / "list-components.py")
cmd = [script_path, "--changed"]
try:
@@ -625,7 +624,7 @@ def filter_changed(files: list[str]) -> list[str]:
def filter_grep(files: list[str], value: list[str]) -> list[str]:
matched = []
for file in files:
with open(file, encoding="utf-8") as handle:
with Path(file).open(encoding="utf-8") as handle:
contents = handle.read()
if any(v in contents for v in value):
matched.append(file)
script/lint-python
+4 -2
View File
@@ -2,6 +2,7 @@
import argparse
import os
from pathlib import Path
import re
import sys
@@ -66,11 +67,12 @@ def main():
args = parser.parse_args()
files = []
cwd = Path.cwd()
for path in git_ls_files():
filetypes = (".py",)
ext = os.path.splitext(path)[1]
ext = Path(path).suffix
if ext in filetypes and path.startswith("esphome"):
path = os.path.relpath(path, os.getcwd())
path = os.path.relpath(path, cwd)
files.append(path)
# Match against re
file_name_re = re.compile("|".join(args.files))
script/split_components_for_ci.py
+1 -1
View File
@@ -295,7 +295,7 @@ def main() -> int:
# Sort groups by signature for readability
groupable_groups = []
isolated_groups = []
for (platform, signature), group_comps in sorted(signature_groups.items()):
for (_platform, signature), group_comps in sorted(signature_groups.items()):
if signature.startswith(ISOLATED_SIGNATURE_PREFIX):
isolated_groups.append((signature, group_comps))
else:
script/sync-device_class.py
+3 -2
View File
@@ -1,5 +1,6 @@
#!/usr/bin/env python3
from pathlib import Path
import re
# pylint: disable=import-error
@@ -34,10 +35,10 @@ DOMAINS = {
def sub(path, pattern, repl):
with open(path, encoding="utf-8") as handle:
with Path(path).open(encoding="utf-8") as handle:
content = handle.read()
content = re.sub(pattern, repl, content, flags=re.MULTILINE)
with open(path, "w", encoding="utf-8") as handle:
with Path(path).open("w", encoding="utf-8") as handle:
handle.write(content)
script/test_build_components.py
+3 -3
View File
@@ -297,7 +297,7 @@ def write_github_summary(
test_results: List of all test results
"""
summary_content = format_github_summary(test_results, toolchain)
with open(os.environ["GITHUB_STEP_SUMMARY"], "a", encoding="utf-8") as f:
with Path(os.environ["GITHUB_STEP_SUMMARY"]).open("a", encoding="utf-8") as f:
f.write(summary_content)
@@ -890,7 +890,7 @@ def run_grouped_component_tests(
print("=" * 80 + "\n")
# Execute grouped tests
for (platform, signature), components in grouped_components.items():
for (platform, _signature), components in grouped_components.items():
# Only group if we have multiple components with same signature
if len(components) <= 1:
continue
@@ -1055,7 +1055,7 @@ def test_components(
# Create empty test files for each platform (or filtered platform)
reference_tests: list[Path] = []
for platform_name, base_file in platform_bases.items():
for platform_name in platform_bases:
if platform_filter and not platform_name.startswith(platform_filter):
continue
# Create an empty test file named to match the platform