#!/usr/bin/env python3 """Reproduce: the bench's hidden_think_ms reads 0 where the adapter recorded hidden reasoning. Usage: python3 -I repro.py FILE.traces.jsonl [FILE2 ...] Input: trace files of the public dataset huggingface.co/datasets/r4topunk/boar-traces (read only). The function visible_timing below is a line-by-line transcription of scorers/timing.py lines 16-42 of github.com/r4topunk/boar-bench at commit b41a25acfabf89f1d015103ca1329554f985a257. No repository code is run. """ import json, sys SILENT_MS = 10_000 # harness/gate.py line 45 NONE_MS = 1_000 # harness/gate.py line 46; also the report's hidden-thinking flag (harness/report.py line 54) def _text(e): if e.get("type") == "token": return e.get("text") or "" if e.get("type") == "instant": s = e.get("snippet") return (s.get("text") if isinstance(s, dict) else s) or "" return "" def visible_timing(t): ev = t.get("events") or [] toks = [e for e in ev if e.get("type") == "token"] first_tok = toks[0]["tMs"] if toks else None vis_tok = next((e["tMs"] for e in toks if _text(e).strip()), None) vis = next((e["tMs"] for e in ev if e.get("type") in ("token", "instant") and _text(e).strip()), None) source = "event" if vis is not None else None if vis is None: out_toks = (t.get("timing") or {}).get("outputTokens") or 0 truncated = bool(toks) and out_toks > len(toks) done = next((e for e in reversed(ev) if e.get("type") == "done"), None) shown_text = (done or {}).get("finalText") or ((t.get("answer") or {}).get("text") if not toks else "") if done and (shown_text or "").strip() and not truncated: vis, source = done["tMs"], "done" hidden = None if first_tok is not None: hidden = max(0.0, vis_tok - first_tok) if vis_tok is not None else None return {"ttft_visible_ms": vis, "hidden_think_ms": hidden} def p50(xs): xs = sorted(xs) return xs[len(xs) // 2] if xs else None def main(paths): for path in paths: n = 0 scorer, adapter, zero, over_silent = [], [], 0, 0 models = set() with open(path, encoding="utf-8") as fh: for line in fh: line = line.strip() if not line: continue t = json.loads(line) if "events" not in t: continue n += 1 rm = (t.get("timing") or {}).get("reasoningMs") if not isinstance(rm, (int, float)) or rm <= NONE_MS: continue models.add((t.get("runtime") or {}).get("model")) h = visible_timing(t)["hidden_think_ms"] adapter.append(rm) if rm > SILENT_MS: over_silent += 1 if h is not None: scorer.append(h) if h < 1: zero += 1 sp = p50(scorer) print(path.replace("\\", "/").split("/")[-1]) print(f" model(s): {sorted(m for m in models if m)}") print(f" traces: {n}; adapter recorded hidden reasoning > 1 s (timing.reasoningMs): {len(adapter)}") print(f" adapter reasoningMs p50: {p50(adapter)} ms; above the gate's 10 s silent threshold: {over_silent}") print(f" scorer hidden_think_ms p50 on the same traces: {sp} ms; reads 0 on {zero} of {len(adapter)}") print(f" report flag hidden_thinking on these traces (p50 > 1000 ms): {sp is not None and sp > 1000}") if __name__ == "__main__": if len(sys.argv) < 2: sys.exit(__doc__) main(sys.argv[1:])