[core] Retry gh CLI calls on transient network errors in CI scripts (#18292)

script/ci_memory_impact_comment.py

@@ -20,17 +20,21 @@ from jinja2 import Environment, FileSystemLoader
 sys.path.insert(0, str(Path(__file__).parent.parent))
 
 # pylint: disable=wrong-import-position
+from helpers import run_gh_command  # noqa: E402
 
 # Comment marker to identify our memory impact comments
 COMMENT_MARKER = "<!-- esphome-memory-impact-analysis -->"
 
 
-def run_gh_command(args: list[str], operation: str) -> subprocess.CompletedProcess:
-    """Run a gh CLI command with error handling.
+def run_gh_command_logged(
+    args: list[str], operation: str, *, retry: bool = True
+) -> subprocess.CompletedProcess:
+    """Run a gh CLI command with retries and error reporting.
 
     Args:
         args: Command arguments (including 'gh')
         operation: Description of the operation for error messages
+        retry: Pass False for non-idempotent commands (see run_gh_command)
 
     Returns:
         CompletedProcess result
@@ -39,12 +43,7 @@ def run_gh_command(args: list[str], operation: str) -> subprocess.CompletedProce
         subprocess.CalledProcessError: If command fails (with detailed error output)
     """
     try:
-        return subprocess.run(
-            args,
-            check=True,
-            capture_output=True,
-            text=True,
-        )
+        return run_gh_command(args, retry=retry)
     except subprocess.CalledProcessError as e:
         print(
             f"ERROR: {operation} failed with exit code {e.returncode}", file=sys.stderr
@@ -472,7 +471,7 @@ def find_existing_comment(pr_number: str) -> str | None:
     print(f"DEBUG: Looking for existing comment on PR #{pr_number}", file=sys.stderr)
 
     # Use gh api to get comments directly - this returns the numeric id field
-    result = run_gh_command(
+    result = run_gh_command_logged(
         [
             "gh",
             "api",
@@ -535,7 +534,7 @@ def update_existing_comment(comment_id: str, comment_body: str) -> None:
     """
     print(f"DEBUG: Updating existing comment {comment_id}", file=sys.stderr)
     print(f"DEBUG: Comment body length: {len(comment_body)} bytes", file=sys.stderr)
-    result = run_gh_command(
+    result = run_gh_command_logged(
         [
             "gh",
             "api",
@@ -562,9 +561,12 @@ def create_new_comment(pr_number: str, comment_body: str) -> None:
     """
     print(f"DEBUG: Posting new comment on PR #{pr_number}", file=sys.stderr)
     print(f"DEBUG: Comment body length: {len(comment_body)} bytes", file=sys.stderr)
-    result = run_gh_command(
+    # Creating a comment is not idempotent: a retry after a dropped response
+    # could post the same comment twice, so fail on the first error instead.
+    result = run_gh_command_logged(
         ["gh", "pr", "comment", pr_number, "--body", comment_body],
         operation="Create PR comment",
+        retry=False,
     )
     print(f"DEBUG: Post response: {result.stdout}", file=sys.stderr)
 

script/helpers.py

@@ -469,6 +469,77 @@ def get_target_branch() -> str | None:
     return None
 
 
+# Substrings (matched case-insensitively against gh's stderr) that identify
+# transient failures worth retrying: server errors (HTTP 5xx) and dropped or
+# failed connections. Permanent failures (bad auth, missing PR, the 300-file
+# diff limit) never match so callers see them immediately. Phrases are
+# anchored so gh's GraphQL "Could not resolve to a PullRequest" (a missing
+# PR) never classifies as a DNS failure.
+_TRANSIENT_GH_ERROR_RE = re.compile(
+    r"http 5\d\d"
+    r"|timed out|timeout"
+    r"|connection (?:reset|refused|closed)"
+    r"|no such host|could not resolve host"
+    # gh intercepts DNS errors and prints its own "error connecting to
+    # <host>" text; the Go phrases above are kept as a hedge in case a
+    # future gh stops swallowing the underlying error
+    r"|error connecting to"
+    r"|failed to verify certificate"
+    # Go reports a server-closed connection as 'Post "<url>": EOF'; the
+    # quote-and-colon anchor keeps a URL or message body containing the
+    # letters from matching
+    r"|unexpected eof"
+    r'|": eof'
+    r"|network is unreachable"
+    r"|temporary failure"
+)
+
+# Same retry policy as git network commands in esphome/git.py: 3 attempts
+# with 2s/4s backoff.
+_GH_MAX_ATTEMPTS = 3
+
+
+def run_gh_command(
+    args: list[str], *, retry: bool = True
+) -> subprocess.CompletedProcess[str]:
+    """Run a gh CLI command, retrying transient network and server failures.
+
+    Args:
+        args: Full command line, including the leading "gh".
+        retry: Pass False for commands that are not idempotent (e.g. posting
+            a comment), where a retry after a dropped response could repeat
+            a write that already succeeded server-side.
+
+    Returns:
+        CompletedProcess with captured text output.
+
+    Raises:
+        subprocess.CalledProcessError: If the command fails with a permanent
+            error, or is still failing after the retries are exhausted.
+    """
+    attempts = _GH_MAX_ATTEMPTS if retry else 1
+    attempt = 0
+    while True:
+        try:
+            return subprocess.run(
+                args, check=True, capture_output=True, text=True, close_fds=False
+            )
+        except subprocess.CalledProcessError as err:
+            attempt += 1
+            stderr = err.stderr or ""
+            if attempt >= attempts or not _TRANSIENT_GH_ERROR_RE.search(stderr.lower()):
+                raise
+            delay = 2**attempt
+            # Only the leading arguments: comment-update calls carry the
+            # whole multi-KB comment body in the argument list
+            print(
+                f"WARNING: {' '.join(args[:3])} failed: {stderr.strip()}; "
+                f"retrying in {delay}s (attempt {attempt}/{attempts})",
+                file=sys.stderr,
+            )
+            time.sleep(delay)
+
+
 @cache
 def _get_changed_files_github_actions() -> list[str] | None:
     """Get changed files in GitHub Actions environment.
@@ -542,10 +613,22 @@ def changed_files(branch: str | None = None) -> list[str]:
 
 
 def _get_changed_files_from_command(command: list[str]) -> list[str]:
-    """Run a git command to get changed files and return them as a list."""
-    proc = subprocess.run(command, capture_output=True, text=True, check=False)
-    if proc.returncode != 0:
-        raise Exception(f"Command failed: {' '.join(command)}\nstderr: {proc.stderr}")
+    """Run a git or gh command to get changed files and return them as a list."""
+    if command[0] == "gh":
+        try:
+            proc = run_gh_command(command)
+        except subprocess.CalledProcessError as e:
+            raise Exception(
+                f"Command failed: {' '.join(command)}\nstderr: {e.stderr}"
+            ) from e
+    else:
+        proc = subprocess.run(
+            command, capture_output=True, text=True, check=False, close_fds=False
+        )
+        if proc.returncode != 0:
+            raise Exception(
+                f"Command failed: {' '.join(command)}\nstderr: {proc.stderr}"
+            )
 
     changed_files = splitlines_no_ends(proc.stdout)
     cwd = Path.cwd()