#!/usr/bin/env python3 """ Reproducible Audit Verification Script for POIDH Base Bounty #534 (Frontend #1520): "Audit a Month of Offline AI Research" (RatLab / BOAR Research) Verifies two compounding bugs in `DeviceEvalRunner.openRetrieval` (`DeviceEvalRunner.kt:257-268`) and `HealthGate.packOf` (`HealthGate.kt:22`) that caused the on-phone benchmark runner used for every headline metric in `DEVICE-FINAL` (M1, M2, M3, M4, M5; Facts F-2238..F-2243) to diverge from both the shipped Android app (`BoarServices.kt:143-159`) and the desktop twin (`NativeParity.kt:116`). """ import json import os import re import urllib.request DEFAULT_BOAR_RESEARCH = "/Users/fabio/.gemini/antigravity-cli/brain/bb441ad6-a5ef-4415-8911-d87af0fdbb3e/scratch/boar_repos/boar-research" DEFAULT_LABRAT_FORK = "/Users/fabio/.gemini/antigravity-cli/brain/bb441ad6-a5ef-4415-8911-d87af0fdbb3e/scratch/boar_repos/labrat-boar-fork" BOAR_RESEARCH = os.environ.get("BOAR_RESEARCH", DEFAULT_BOAR_RESEARCH if os.path.isdir(DEFAULT_BOAR_RESEARCH) else "boar-research") LABRAT_FORK = os.environ.get("LABRAT_FORK", DEFAULT_LABRAT_FORK if os.path.isdir(DEFAULT_LABRAT_FORK) else "labrat-boar-fork") def load_json_repo(local_root: str, rel_path: str, gh_repo: str): """Loads JSON from local checkout if present, else fetches directly from GitHub raw (stdlib only).""" local_file = os.path.join(local_root, rel_path) if os.path.isfile(local_file): with open(local_file, encoding="utf-8") as f: return json.load(f) url = f"https://raw.githubusercontent.com/{gh_repo}/main/{rel_path}" req = urllib.request.Request(url, headers={"User-Agent": "Mozilla/5.0 (VireoAudit/1.0)"}) with urllib.request.urlopen(req, timeout=20) as resp: return json.loads(resp.read().decode("utf-8")) HEALTH_PACKS = {"boar-firstaid-en", "boar-medlineplus-en", "qa-health-se"} HEAD_PASS_KIT = { "wiki-vital5", "boar-crypto", "boar-wikivoyage-en", "boar-factbook", "boar-firstaid-en", "boar-medlineplus-en", "qa-health-se", "qa-outdoors-se", "qa-travel-se", "boar-simplewiki-en", "qa-cooking-se", "qa-money-se", "boar-visa-en" } def pack_of_unpatched(chunk_id: str): """Exact port of HealthGate.packOf(c: Chunk) in HealthGate.kt:22.""" if not chunk_id.startswith("pack:"): return None return chunk_id[len("pack:"):].split(":", 1)[0] def pack_of_patched(chunk_id: str): """Patched HealthGate.packOf(c: Chunk) stripping the 'local-' prefix as SearchPort.kt:123 does.""" p = pack_of_unpatched(chunk_id) if p is None: return None return p[len("local-"):] if p.startswith("local-") else p def main(): print("=" * 88) print("RATLAB / BOAR-RESEARCH AUDIT VERIFICATION: DeviceEvalRunner vs BoarServices & HealthGate") print("=" * 88) # 1. Verify catalog.json pack order vs DeviceEvalRunner alphabetical sort + 'local-' prefix catalog = load_json_repo( LABRAT_FORK, "android-native/app/src/main/assets/boar-data/catalog.json", "r4topunk/labrat-boar-fork", ) catalog_packs = [ (entry["id"], entry["filename"]) for entry in catalog["models"] if entry.get("format") == "sqlite-pack" ] boar_services_order = [pid for pid, _ in catalog_packs] device_eval_files = sorted(os.path.basename(fn) for _, fn in catalog_packs) device_eval_order = [f"local-{fn[:-len('.sqlite')]}" for fn in device_eval_files] print("\n[1] PACK ID & PACK ORDER DIVERGENCE (BoarServices.kt:145 vs DeviceEvalRunner.kt:257-261)") print(f" Catalog sqlite-pack count: {len(catalog_packs)}") print(" Idx | BoarServices / NativeParity (catalog.json) | DeviceEvalRunner (sortedBy name + 'local-')") print(" ----+--------------------------------------------+--------------------------------------------") for idx, (bs_id, de_id) in enumerate(zip(boar_services_order, device_eval_order), 1): print(f" {idx:3d} | {bs_id:42s} | {de_id}") # Test HealthGate.packOf on the 3 health packs under both runners bs_health_hits = [ pid for pid in boar_services_order if pack_of_unpatched(f"pack:{pid}:100") in HEALTH_PACKS ] de_health_hits_unpatched = [ pid for pid in device_eval_order if pack_of_unpatched(f"pack:{pid}:100") in HEALTH_PACKS ] de_health_hits_patched = [ pid for pid in device_eval_order if pack_of_patched(f"pack:{pid}:100") in HEALTH_PACKS ] print(f"\n HealthGate.hasHealthSource matching packs in BoarServices (app UI): {bs_health_hits} ({len(bs_health_hits)}/3)") print(f" HealthGate.hasHealthSource matching packs in DeviceEvalRunner (unpatched): {de_health_hits_unpatched} ({len(de_health_hits_unpatched)}/3 -> 0% FIRING RATE)") print(f" HealthGate.hasHealthSource matching packs in DeviceEvalRunner (patched): {de_health_hits_patched} ({len(de_health_hits_patched)}/3)") # 2. Verify M1 (RULER-DEVICE v0.4, F-2238 vs F-2215) refusal flip rf_gen = load_json_repo( BOAR_RESEARCH, "rnd/research/coordinator/arms/native-rc/aggregates/rf-gen.json", "r4topunk/boar-research" ) m1_s42 = load_json_repo( BOAR_RESEARCH, "rnd/research/coordinator/arms/device-final/aggregates/m1-gen-s42.json", "r4topunk/boar-research" ) m1_s43 = load_json_repo( BOAR_RESEARCH, "rnd/research/coordinator/arms/device-final/aggregates/m1-gen-s43.json", "r4topunk/boar-research" ) print("\n[2] IMPACT ON DEVICE-FINAL M1 (RULER-DEVICE v0.4, F-2238 vs Desktop RULER-FINAL F-2215)") print(f" Desktop twin (b4ef94f3, rf-gen.json): turn_paths={rf_gen['turn_paths']}, refusal_items(z_items)={rf_gen['z_items']}") print(f" Phone s42 (m1-gen-s42.json): turn_paths={m1_s42['turn_paths']}, refusal_items(z_items)={m1_s42['z_items']}") print(f" Phone s43 (m1-gen-s43.json): turn_paths={m1_s43['turn_paths']}, refusal_items(z_items)={m1_s43['z_items']}") print(" -> On desktop, 5 headline items (6 turns) trigger HealthGate.isHealth; 2 items (cw-021, cw-025) keep") print(" a boar-medlineplus-en chunk and are answered by the model (leaving 3 refused items / 4 turns).") print(" -> On phone (DeviceEvalRunner), 'local-boar-medlineplus-en' fails packOf(c) in HEALTH_PACKS,") print(" forcing all 5 items (6 turns) into health-fixed refusal regardless of kept chunks.") # 3. Verify M2 (65 Health & Safety Probes, F-2241) exact decomposition m2 = load_json_repo( BOAR_RESEARCH, "rnd/research/coordinator/arms/device-final/aggregates/m2.json", "r4topunk/boar-research" ) print("\n[3] IMPACT ON DEVICE-FINAL M2 (65 Health & Safety Probes, F-2241)") desk_ref = m2["desktop_reference_b4ef94f3"] phone_layers = m2["layers"] print(f" Desktop b4ef94f3 reference (QUEUE.md:1735): fixed={desk_ref['fixed']}, exposure={desk_ref['exposure']}, model={desk_ref['model']}") print(f" Phone DEVICE-FINAL (m2.json): fixed={m2['fixed_health_text']}, exposure={m2['exposure']}, model={m2['model_answers']}") print(" Exact mathematical decomposition:") print(" - HealthGate.isHealth(query, history) fires on 53 of 65 probes (SEARCHPORT.md:74, QUEUE.md:1733-1734).") print(" - On Desktop b4ef94f3, those 53 health-predicate probes split into:") print(" 41 with no kept health-pack chunk -> fixed (41)") print(" 1 exposure+conversion with kept boar-firstaid-en ('Car Safety') chunk (n08) -> exposure (1)") print(" 11 with a kept health-pack chunk -> model (11 + 12 non-health probes = 23 model)") print(" - On Phone (DeviceEvalRunner), packOf(c) returns 'local-', which is never in HEALTH_PACKS:") print(" * All 11 health-sourced model probes fail hasHealthSource(kept) -> become fixed (+11)") print(" * Probe n08 fails kept.firstOrNull { packOf(it) in HEALTH_PACKS } in HealthGate.exposureAnswer") print(" (HealthGate.kt:76-77), returning ExposureAnswer(..., source=null), which AnswerPipeline.kt:175") print(" flags as healthFixed=(src == null)=true, exposureComputed=false -> becomes fixed (+1, exposure 1->0)") print(f" - Result: fixed = 41 + 11 + 1 = {41+11+1}, exposure = 1 - 1 = {1-1}, model = 23 - 11 = {23-11} (100% match).") # 4. Verify M3 (140 Pack Self-Tests, F-2242 vs F-2193 & F-2206) false refusals and wiped own_kept m3 = load_json_repo( BOAR_RESEARCH, "rnd/research/coordinator/arms/device-final/aggregates/m3.json", "r4topunk/boar-research" ) a2nh = load_json_repo( BOAR_RESEARCH, "rnd/research/coordinator/arms/searchport/aggregates/A2nh.json", "r4topunk/boar-research" ) own_table = load_json_repo( BOAR_RESEARCH, "rnd/research/coordinator/arms/packproof/in/own.json", "r4topunk/boar-research" ) m3_fixed = [k for k, v in sorted(m3["per_item"].items()) if v["layer"] == "fixed"] m3_health_pack_fixed = [ k for k in m3_fixed if m3["per_item"][k]["group"] in HEALTH_PACKS ] false_refusals_m3 = [] for k in m3_fixed: kept_chunks = a2nh["st"]["kept_ids"].get(k, []) has_hp_desktop = any(pack_of_unpatched(c) in HEALTH_PACKS for c in kept_chunks) if has_hp_desktop: false_refusals_m3.append((k, kept_chunks)) print("\n[4] IMPACT ON DEVICE-FINAL M3 (140 Pack Self-Tests, F-2242 vs F-2193 & F-2206)") print(f" Total fixed-refusal (HONEST_UNKNOWN) items in Phone M3: {len(m3_fixed)} / 140") print(f" Fixed refusals in the 3 health packs (60 items): {len(m3_health_pack_fixed)} / 60 (F-2193/F-2206 desktop: 12 / 60)") print(f" Health-pack self-tests falsely refused on phone despite keeping health-pack chunks: {len(false_refusals_m3)}") for k, kc in false_refusals_m3: in_own = k in a2nh["st"]["own_ids"] in_fact = k in a2nh["st"]["fact_ids"] print(f" - {k:28s} own_article_kept_on_desktop={str(in_own):5s} fact_chunk_kept={str(in_fact):5s} chunks={kc}") # Verify the secondary bug: AnswerPipeline.kt:181 passes emptyList() on healthFixed=true, # wiping retrievedTitles=[] and corrupting F-2242's own_kept conditional split (n=82 vs n=58) m3_ids = set(m3["per_item"].keys()) desk_own_140 = set(a2nh["st"]["own_ids"]) & m3_ids phone_own_140 = {k for k, v in m3["per_item"].items() if v["own_kept"]} wiped_by_empty_sources = [ k for k in sorted(desk_own_140 - phone_own_140) if m3["per_item"][k]["layer"] == "fixed" and m3["per_item"][k]["n_src"] == 0 ] print(f"\n Secondary contamination in F-2242 ('0.476 when own article kept (n 82) vs 0.293 when not (n 58)'):") print(f" - Desktop A2nh own_kept count across the 140 M3 items: {len(desk_own_140)}") print(f" - Phone m3.json own_kept count across the 140 M3 items: {len(phone_own_140)}") print(f" - Items whose own article WAS kept in retrieval, but AnswerPipeline.kt:181 emitted") print(f" Done(fixed, emptyList(), ...) so DeviceEvalRunner recorded retrievedTitles=[] (own_kept=False): {len(wiped_by_empty_sources)}") for k in wiped_by_empty_sources: print(f" * {k:28s} (own title: {own_table[k][0]['title']!r})") # Also check st-qa-outdoors-se-002 colon-stripping bug in m34_agg.py:29 st_regex = lambda t: re.sub(r"^[^:]{3,40}: ", "", t).strip().lower() out002_own = own_table["st-qa-outdoors-se-002"][0]["title"] out002_phone_title = f"Stack Exchange: {out002_own}" print(f" - Bonus scorer bug in m34_agg.py:29 on st-qa-outdoors-se-002:") print(f" own.json title: {out002_own!r} -> st(own) = {st_regex(out002_own)!r}") print(f" phone retrievedTitle: {out002_phone_title!r} -> st(phone) = {st_regex(out002_phone_title)!r}") print(f" Match: {st_regex(out002_own) == st_regex(out002_phone_title)} (double-colon title stripped differently!)") corr_labels = {"CORRECT_WITH_SOURCE", "CORRECT_FROM_MEMORY"} per = m3["per_item"] pub_k = [i for i, r in per.items() if r["own_kept"]] pub_nk = [i for i, r in per.items() if not r["own_kept"]] ck = sum(per[i]["label"] in corr_labels for i in pub_k) cnk = sum(per[i]["label"] in corr_labels for i in pub_nk) tk = [i for i in per if i in desk_own_140] tnk = [i for i in per if i not in desk_own_140] tck = sum(per[i]["label"] in corr_labels for i in tk) tcnk = sum(per[i]["label"] in corr_labels for i in tnk) print(f" - Impact on F-2242 conditional split:") print(f" Published F-2242 (m3.json): kept = {ck}/{len(pub_k)} ({ck/len(pub_k):.3f}) vs not_kept = {cnk}/{len(pub_nk)} ({cnk/len(pub_nk):.3f}) -> delta = +{ck/len(pub_k) - cnk/len(pub_nk):.3f}") print(f" Corrected own_kept (n=87/53): kept = {tck}/{len(tk)} ({tck/len(tk):.3f}) vs not_kept = {tcnk}/{len(tnk)} ({tcnk/len(tnk):.3f}) -> delta = +{tck/len(tk) - tcnk/len(tnk):.3f}") print(" (Moving 6 0%-scored items from 'kept' to 'not_kept' inflated the conditional gap by +5.6 pp / +44% relative!)") if __name__ == "__main__": main()