Merge remote-tracking branch 'upstream/dev' into logstr-literal-lint

# Conflicts:
#	tests/script/test_ci_custom.py

script/ci-custom.py

@@ -3,6 +3,7 @@
 import argparse
 import codecs
 import collections
+from collections.abc import Iterator
 import fnmatch
 import functools
 import os.path
@@ -1120,7 +1121,56 @@ def lint_no_std_bind(fname, match):
     )
 
 
-LOG_MULTILINE_RE = re.compile(r"ESP_LOG\w+\s*\(.*?;", re.DOTALL)
+LOG_CALL_START_RE = re.compile(r"ESP_LOG\w+\s*\(")
+# Comments, raw/plain string literals and single char literals are consumed whole so ; ( ) ? :
+# inside them are never seen. A char literal is exactly one (escaped) char so a digit separator
+# like 1'000'000 cannot open one.
+CPP_COMMENT_RE = r"//[^\n]*|/\*.*?\*/"
+CPP_SKIP_RE = (
+    CPP_COMMENT_RE
+    + r'|R"(?P<raw_delim>[^(\s]*)\(.*?\)(?P=raw_delim)"|"(?:[^"\\]|\\.)*"|\'(?:[^\'\\\n]|\\.)\''
+)
+LOG_CALL_TOKEN_RE = re.compile(CPP_SKIP_RE + r"|[()]", re.DOTALL)
+# The last alternative matches a ? or : followed (after spaces or comments) by an opening quote,
+# i.e. a string literal used as a ternary branch.
+LOG_TERNARY_LITERAL_RE = re.compile(
+    CPP_SKIP_RE + r"|[?:](?:\s|" + CPP_COMMENT_RE + r')*(?=")', re.DOTALL
+)
+# A bare NOLINT; a clang-tidy NOLINT(check-name) is aimed at a different tool.
+NOLINT_RE = re.compile(r"\bNOLINT\b(?!\()")
+
+
+def _line_col(content: str, pos: int) -> tuple[int, int]:
+    """1-based line and column of an offset in content."""
+    return content.count("\n", 0, pos) + 1, pos - content.rfind("\n", 0, pos)
+
+
+def _iter_log_calls(content: str) -> Iterator[tuple[int, str | None]]:
+    """Yield (start, text) for every ESP_LOG*(...) call, text running to the matching close paren.
+    text is None when no matching paren exists so callers can report the call instead of skipping it."""
+    for head in LOG_CALL_START_RE.finditer(content):
+        depth = 1
+        for tok in LOG_CALL_TOKEN_RE.finditer(content, head.end()):
+            if tok.group(0) == "(":
+                depth += 1
+            elif tok.group(0) == ")":
+                depth -= 1
+                if depth == 0:
+                    yield head.start(), content[head.start() : tok.end()]
+                    break
+        else:
+            yield head.start(), None
+
+
+def _unbalanced_log_call_error(content: str, pos: int) -> tuple[int, int, str]:
+    lineno, col = _line_col(content, pos)
+    return (
+        lineno,
+        col,
+        "ESP_LOG call has no matching closing parenthesis, so it cannot be checked.",
+    )
+
+
 LOG_BAD_CONTINUATION_RE = re.compile(r'\\n(?:[^ \\"\r\n\t]|"\s*\n\s*"[^ \\])')
 LOG_PERCENT_S_CONTINUATION_RE = re.compile(r'\\n(?:%s|"\s*\n\s*"%s)')
 
@@ -1128,16 +1178,16 @@ LOG_PERCENT_S_CONTINUATION_RE = re.compile(r'\\n(?:%s|"\s*\n\s*"%s)')
 @lint_content_check(include=cpp_include)
 def lint_log_multiline_continuation(fname, content):
     errs = []
-    for log_match in LOG_MULTILINE_RE.finditer(content):
-        log_text = log_match.group(0)
+    for log_start, log_text in _iter_log_calls(content):
+        if log_text is None:
+            errs.append(_unbalanced_log_call_error(content, log_start))
+            continue
         for bad_match in LOG_BAD_CONTINUATION_RE.finditer(log_text):
             # %s may expand to a whitespace prefix at runtime, skip those
             if LOG_PERCENT_S_CONTINUATION_RE.match(log_text, bad_match.start()):
                 continue
             # Calculate line number from position in full content
-            abs_pos = log_match.start() + bad_match.start()
-            lineno = content.count("\n", 0, abs_pos) + 1
-            col = abs_pos - content.rfind("\n", 0, abs_pos)
+            lineno, col = _line_col(content, log_start + bad_match.start())
             errs.append(
                 (
                     lineno,
@@ -1155,6 +1205,90 @@ def lint_log_multiline_continuation(fname, content):
     return errs
 
 
+def _find_ternary_literals(text: str) -> Iterator[tuple[int, str]]:
+    """Yield (offset, literal) for every string literal used as a ternary branch."""
+    branch = False
+    for m in LOG_TERNARY_LITERAL_RE.finditer(text):
+        tok = m.group(0)
+        # An empty literal is merged with every other string's terminator, so it costs no RAM,
+        # while a PSTR("") would add its own flash array; leave it alone.
+        if branch and tok[0] == '"' and tok != '""':
+            yield m.start(), tok
+        branch = tok[0] in "?:"
+
+
+# LOG_STR_LITERAL is a no op everywhere except ESP8266, so code that never builds there is skipped
+# to avoid churn: platform specific sources and components for ESP32, LibreTiny, RP2 and Zephyr only.
+# A component belongs here only if it has no tests/components/<name>/test.esp8266-ard.yaml.
+LOG_LITERAL_LINT_EXCLUDE = [
+    "*_esp32.cpp",
+    "*_esp32_*.cpp",
+    "*_esp_idf.cpp",
+    "*_rmt.cpp",
+    "*_zephyr.cpp",
+    "*_bk72xx.cpp",
+    "*_libretiny.cpp",
+    "*_pico_w.cpp",
+    "*_host.cpp",
+    "esphome/components/esp32*/*",
+    "esphome/components/bk72xx*/*",
+    "esphome/components/ln882h*/*",
+    "esphome/components/ln882x*/*",
+    "esphome/components/rp2*/*",
+    "esphome/components/zephyr*/*",
+    "esphome/components/host/*",
+    "esphome/components/libretiny*/*",
+    "esphome/components/bluetooth_proxy/*",
+    "esphome/components/bluetooth_connection/*",
+    "esphome/components/ble_client/*",
+    "esphome/components/bedjet/*",
+    "esphome/components/anova/*",
+    "esphome/components/xiaomi_ble/*",
+    "esphome/components/bthome_mithermometer/*",
+    "esphome/components/usb_host/*",
+    "esphome/components/zigbee/*",
+    "esphome/components/lvgl/*",
+    # Test fixtures and host only unit tests - not production embedded code
+    "tests/integration/fixtures/*",
+    "tests/components/*",
+]
+
+
+@lint_content_check(include=cpp_include, exclude=LOG_LITERAL_LINT_EXCLUDE)
+def lint_log_no_bare_literal_ternary(
+    fname: Path, content: str
+) -> list[tuple[int, int, str]]:
+    errs = []
+    for log_start, log_text in _iter_log_calls(content):
+        if log_text is None:
+            continue  # reported by lint_log_multiline_continuation, which sees every file
+        # A NOLINT anywhere on the lines the call spans silences every branch in it
+        first_line = content.rfind("\n", 0, log_start) + 1
+        last_line = content.find("\n", log_start + len(log_text))
+        if NOLINT_RE.search(
+            content[first_line : last_line if last_line != -1 else None]
+        ):
+            continue
+        for offset, literal in _find_ternary_literals(log_text):
+            lineno, col = _line_col(content, log_start + offset)
+            errs.append(
+                (
+                    lineno,
+                    col,
+                    (
+                        "String literal used as a ternary branch in a log call. On ESP8266 the "
+                        "log macro moves the format string to flash, but bare literal arguments "
+                        "stay in RAM. Wrap each branch passed straight to the log call in "
+                        f"{highlight('LOG_STR_LITERAL(...)')}:\n"
+                        f"  Before: {highlight(literal)}\n"
+                        f"  After:  {highlight(f'LOG_STR_LITERAL({literal})')}\n"
+                        f"(If strictly necessary, add `{highlight('// NOLINT')}` to the end of the line)"
+                    ),
+                )
+            )
+    return errs
+
+
 @lint_content_find_check(
     "ESP_LOG",
     include=["*.h", "*.tcc"],

tests/script/test_ci_custom.py

@@ -4,12 +4,16 @@ The rule flags an if/else/for/while whose only body is an unbraced ESP_LOG*() ca
 empty statement -- and a -Wempty-body warning -- once the log level compiles the macro out). These
 tests pin the comment/string/raw-string masker, the accepted control-statement shapes, and the
 NOLINT escape hatch at both placements a contributor would try.
+
+Also covers the ESP_LOG call scanner (_iter_log_calls) and the bare-literal-ternary lint.
 """
 
 import importlib.util
 from pathlib import Path
 import sys
 
+import pytest
+
 SCRIPT_DIR = (Path(__file__).parent / ".." / ".." / "script").resolve()
 sys.path.insert(0, str(SCRIPT_DIR))
 _spec = importlib.util.spec_from_file_location("ci_custom", SCRIPT_DIR / "ci-custom.py")
@@ -145,3 +149,125 @@ def test_nolint_at_end_of_log_line_suppresses() -> None:
 
 def test_nolint_on_control_line_suppresses() -> None:
     assert not _lint("if (x)  // NOLINT\n  ESP_LOGD(t);\n")
+
+
+# --- ESP_LOG call scanner and bare-literal-ternary lint ---
+
+
+def _calls(content: str) -> list[str | None]:
+    return [text for _, text in ci_custom._iter_log_calls(content)]
+
+
+def _ternary_errors(content: str) -> list[tuple[int, int]]:
+    errs = ci_custom.lint_log_no_bare_literal_ternary(Path("x.cpp"), content)
+    return [(line, col) for line, col, _ in errs]
+
+
+@pytest.mark.parametrize(
+    "content",
+    [
+        'ESP_LOGD(TAG, "a ) b ( c; d")',
+        'ESP_LOGD(TAG, "quote \\" inside")',
+        "ESP_LOGD(TAG, \"%s\", format_hex_pretty(x, '-', false).c_str())",
+        "ESP_LOGD(TAG, \"%c%c\", '(', ')')",
+        "ESP_LOGD(TAG, \"%d\", 1'000'000)",
+        'ESP_LOGD(TAG,  // it\'s a comment with ) and (\n         "x")',
+        'ESP_LOGD(TAG, /* :) */ "x")',
+        'ESP_LOGD(TAG, "%s", R"(say "hi" :) )")',
+        'ESP_LOGD(TAG, "%s", R"x(a)"b)x")',
+    ],
+)
+def test_iter_log_calls_spans_whole_call(content: str) -> None:
+    calls = _calls(content + ";\nint other = (1);")
+    assert calls == [content]
+
+
+def test_iter_log_calls_reports_unbalanced_call_once() -> None:
+    content = 'ESP_LOGD(TAG, "x";\nvoid f();'
+    assert _calls(content) == [None]
+    errs = ci_custom.lint_log_multiline_continuation(Path("x.cpp"), content)
+    assert len(errs) == 1
+    assert errs[0][:2] == (1, 1)
+    assert "no matching closing parenthesis" in errs[0][2]
+    assert _ternary_errors(content) == []
+
+
+@pytest.mark.parametrize(
+    ("content", "expected"),
+    [
+        # A ; inside the format string no longer cuts the call short
+        ('ESP_LOGD(TAG, "a; b\\nc %s", x);', [(1, 20)]),
+        # A \n%s continuation is exempt since %s may expand to leading whitespace
+        ('ESP_LOGD(TAG, "a\\n%s", x);', []),
+        ('ESP_LOGD(TAG, "a\\n  b");', []),
+    ],
+)
+def test_multiline_continuation_detection(
+    content: str, expected: list[tuple[int, int]]
+) -> None:
+    errs = ci_custom.lint_log_multiline_continuation(Path("x.cpp"), content)
+    assert [(line, col) for line, col, _ in errs] == expected
+
+
+def test_exclusion_list_only_names_components_without_esp8266_tests() -> None:
+    root = Path(__file__).parent / ".." / ".."
+    for pattern in ci_custom.LOG_LITERAL_LINT_EXCLUDE:
+        if not pattern.startswith("esphome/components/"):
+            continue
+        prefix = pattern.removeprefix("esphome/components/").split("/")[0]
+        comps = list((root / "esphome" / "components").glob(prefix))
+        assert comps, f"{pattern!r} matches no component"
+        for comp in comps:
+            test = root / "tests" / "components" / comp.name / "test.esp8266-ard.yaml"
+            assert not test.exists(), (
+                f"{comp.name} builds for ESP8266, drop {pattern!r}"
+            )
+
+
+def test_unbalanced_calls_are_reported_by_a_check_that_sees_every_file() -> None:
+    # lint_log_no_bare_literal_ternary skips unbalanced calls and relies on this
+    checks = {c["func"].__name__: c for c in ci_custom.LINT_CONTENT_CHECKS}
+    continuation = checks["lint_log_multiline_continuation"]
+    ternary = checks["lint_log_no_bare_literal_ternary"]
+    assert continuation["exclude"] == []
+    assert continuation["include"] == ternary["include"]
+
+
+@pytest.mark.parametrize(
+    ("content", "expected"),
+    [
+        ('ESP_LOGD(TAG, "%s", x ? "on" : "off");', [(1, 25), (1, 32)]),
+        (
+            'ESP_LOGD(TAG, "%s", x ? LOG_STR_LITERAL("on") : LOG_STR_LITERAL("off"));',
+            [],
+        ),
+        ('ESP_LOGD(TAG, "%s", x ? LOG_STR_LITERAL("on") : "off");', [(1, 49)]),
+        ('ESP_LOGD(TAG, "%s", x ? "on" : "");', [(1, 25)]),
+        (
+            'ESP_LOGD(TAG, "%s",\n         x ? "yes"\n           : "no");',
+            [(2, 14), (3, 14)],
+        ),
+        ("ESP_LOGD(TAG, \"%c\", x ? '1' : '0');", []),
+        ('ESP_LOGD(TAG, "a ? b : c %s", x ? "on" : "off");', [(1, 35), (1, 42)]),
+        ('ESP_LOGD(TAG, "x:" "y %s", p);', []),
+        ('ESP_LOGD(TAG, "%s", x ? "on" : "off");  // NOLINT', []),
+        ('ESP_LOGD(TAG, "%s",\n         x ? "yes"\n           : "no");  // NOLINT', []),
+        ('ESP_LOGD(TAG, "%s", x ? /* c */ "on" : "off");', [(1, 33), (1, 40)]),
+        (
+            'ESP_LOGD(TAG, "%s",\n         x ? "on"  // NOLINT(some-clang-check)\n           : "off");',
+            [(2, 14), (3, 14)],
+        ),
+    ],
+)
+def test_ternary_literal_detection(
+    content: str, expected: list[tuple[int, int]]
+) -> None:
+    assert _ternary_errors(content) == expected
+
+
+def test_ternary_error_message_names_the_literal() -> None:
+    errs = ci_custom.lint_log_no_bare_literal_ternary(
+        Path("x.cpp"), 'ESP_LOGD(TAG, "%s", x ? "enabled" : LOG_STR_LITERAL("off"));'
+    )
+    assert len(errs) == 1
+    assert 'LOG_STR_LITERAL("enabled")' in errs[0][2]