#!/usr/bin/env python3 """Dataset-wide count: hidden reasoning recorded by the adapter vs the bench's hidden_think_ms. Usage: python3 -I count_all.py DATASET_DIR Input: a local copy of huggingface.co/datasets/r4topunk/boar-traces (read only). Every *.traces.jsonl file under DATASET_DIR is read; a trace is any JSON line that has an "events" key. The hidden_think_ms computation is transcribed from scorers/timing.py lines 24-42 of github.com/r4topunk/boar-bench at commit b41a25acfabf89f1d015103ca1329554f985a257. No repository code is run. """ import collections, json, os, sys SILENT_MS = 10_000 # harness/gate.py line 45 NONE_MS = 1_000 # harness/gate.py line 46 def hidden_think_ms(t): toks = [e for e in (t.get("events") or []) if e.get("type") == "token"] if not toks: return None, 0 first_tok = toks[0]["tMs"] blank = 0 for e in toks: if (e.get("text") or "").strip(): return max(0.0, e["tMs"] - first_tok), blank blank += 1 return None, blank def main(root): files = traces = 0 g = collections.defaultdict(collections.Counter) for d, _, names in sorted(os.walk(root)): for name in sorted(names): if not name.endswith(".traces.jsonl"): continue files += 1 with open(os.path.join(d, name), encoding="utf-8", errors="replace") as fh: for line in fh: try: t = json.loads(line) except ValueError: continue if not isinstance(t, dict) or "events" not in t: continue traces += 1 rt = t.get("runtime") or {} backend = "llama-server" if "llama-server" in str(rt.get("host")) else "node" c = g[(backend, str(rt.get("model")))] c["traces"] += 1 rm = (t.get("timing") or {}).get("reasoningMs") if not isinstance(rm, (int, float)) or rm <= NONE_MS: continue h, blank = hidden_think_ms(t) c["reasoning>1s"] += 1 c["reasoning>10s"] += rm > SILENT_MS c["blank_tokens_first"] += blank > 0 c["scorer==0"] += h == 0 c["scorer>1s"] += h is not None and h > NONE_MS c["scorer>10s"] += h is not None and h > SILENT_MS print(f"files: {files}; traces: {traces}") cols = ["traces", "reasoning>1s", "reasoning>10s", "blank_tokens_first", "scorer==0", "scorer>1s", "scorer>10s"] print("backend | model | " + " | ".join(cols)) tot = collections.defaultdict(collections.Counter) for (backend, model), c in sorted(g.items()): tot[backend].update(c) if c["reasoning>1s"]: print(f"{backend} | {model} | " + " | ".join(str(c[k]) for k in cols)) for backend, c in sorted(tot.items()): print(f"{backend} | ALL MODELS | " + " | ".join(str(c[k]) for k in cols)) if __name__ == "__main__": if len(sys.argv) != 2: sys.exit(__doc__) main(sys.argv[1])