Tighten the WISP, AP restart, Xiaomi advertiser and OTA timing checks

.agents/skills/hardware-test/scripts/ota_timing/summ.py

@@ -22,7 +22,7 @@ for r in rows:
 rounds = int(sys.argv[2]) if len(sys.argv) > 2 else 0
 failed = unparsed = short = 0
 for (board, mode), v in sorted(by.items()):
-    ok = sorted(r["up"] for r in v if r["rc"] == 0 and r["up"])
+    ok = sorted(r["up"] for r in v if r["rc"] == 0 and r["up"] is not None)
     fails = sum(r["rc"] != 0 for r in v)
     failed += fails
     unparsed += sum(r["rc"] == 0 and r["up"] is None for r in v)

.agents/skills/hardware-test/scripts/wifi_restart/an.py

@@ -2,8 +2,8 @@
 Usage: an.py DIR BUILD [RECOVER_S]   reads BUILD-marks.txt, BUILD-syslog.txt, BUILD-bk.txt, BUILD-cb.txt (optional)
 and BUILD-up.txt (the bridge's API logs)
 Exits 1 when a DUT rebooted, crashed, sent nothing, or had no syslog line within RECOVER_S (default 120) of an
-AP start, when the BK7238 capture is missing or empty, or when the bridge did not switch its AP to WPA2/WPA3 mixed
-after every AP start."""
+AP start, when the BK7238 capture (or a CB3S capture that was started) is empty, or when the bridge did not switch
+its AP to WPA2/WPA3 mixed after each AP start."""
 
 from itertools import pairwise
 from pathlib import Path
@@ -60,9 +60,10 @@ for dev in ("rebk", "recb"):
         if first is None or first > RECOVER:
             bad.append(f"{dev} not back {RECOVER} s after an AP start")
 for dev, suffix in (("rebk", "bk"), ("recb", "cb")):
-    if not (s := read(suffix)):
-        if dev == "rebk":
-            bad.append("BK7238 serial capture missing or empty")
+    if (s := read(suffix)) is None and dev == "recb":
+        continue  # the CB3S capture is optional
+    if not s:
+        bad.append(f"{dev} serial capture missing or empty")
         continue
     crashes = len(re.findall(r"(?i)hard fault|exception|abort|watchdog", s))
     connected = len(re.findall(r"\[wifi:\d+\]: Connected", s))
@@ -73,12 +74,22 @@ for dev, suffix in (("rebk", "bk"), ("recb", "cb")):
     if crashes:
         bad.append(f"{dev} crashed")
 up = read("up") or ""
+switches = [
+    secs(m[1]) for m in re.finditer(r"\[(\d+:\d+:\d+)[.\d]*\].*PMF capable: 0", up)
+]
+# each AP start needs its own switch between that mark (less a few seconds of clock skew) and the next mark
+ends = [t for t, _ in marks[1:]] + [float("inf")]
+missed = [
+    t
+    for (t, st), end in zip(marks, ends, strict=True)
+    if st == "on" and not any(t - 5 <= s < end for s in switches)
+]
 starts = sum(1 for _, st in marks if st == "on")
-switched = up.count("PMF capable: 0")
 print(
-    f"-- bridge: WPA2/WPA3 switches {switched} for {starts} AP starts, failures {up.count('AP CONFIG FAILED')}"
+    f"-- bridge: WPA2/WPA3 switches {len(switches)}, AP starts {starts} ({len(missed)} without a switch), "
+    f"failures {up.count('AP CONFIG FAILED')}"
 )
-if "AP CONFIG FAILED" in up or switched < starts:
+if "AP CONFIG FAILED" in up or missed:
     bad.append("the bridge did not switch its AP to WPA2/WPA3 mixed after every start")
 print("PASS" if not bad else "FAIL: " + ", ".join(bad))
 sys.exit(1 if bad else 0)

.agents/skills/hardware-test/scripts/wisp_ap/sum.py

@@ -2,7 +2,7 @@
 Usage: sum.py DIR NAME   reads NAME-marks.txt, NAME-cli.txt and NAME-dut.txt
 Phase N runs from mark N (the first mark is the start state). Exits 1 when the DUT crashed, the DUT capture does not
 hold exactly one boot (empty or rebooted), the client had no successful result at all, or a phase with the upstream
-on had no successful name lookup."""
+on had no successful name lookup (an answer equal to the gateway is the captive portal and does not count)."""
 
 from collections import defaultdict
 from pathlib import Path
@@ -38,9 +38,11 @@ for line in cli:
     s = m.group(1)
     if s.startswith("NET"):
         dns = re.search(r"dns (\S+)", s).group(1)
+        gw = re.search(r"gw (\S+)", s).group(1)
         res = s.split("-> ")[1].strip()
-        s = f"dns={dns} resolve={res}"
-        if res != "FAIL":
+        # the captive portal answers every name with the AP's own address; that is not a working upstream
+        s = f"dns={dns} resolve={'CAPTIVE' if res == gw else res}"
+        if res not in ("FAIL", gw):
             good[phase(line[1:9])] += 1
             any_ok += 1
     elif re.match(r"(NAME|GW|MAC) 200\b", s):

.agents/skills/hardware-test/scripts/xiaomi_ble/xbadv.yaml

@@ -47,6 +47,8 @@ button:
       - lambda: |-
           id(af_idx) = 0;
           af_failed = 0;
+          // an unfinished sequence reads as unknown, not as the 0 from an earlier run
+          id(failed_sends).publish_state(NAN);
           ESP_LOGI("af", "sequence start");
 
 interval: