"""Standalone CPU-only verifier for the retained OLMoE publication successor. This module is shipped as verify-bundle.py alongside the unchanged frozen scorer. It neither loads models nor executes generated responses. SHA-256 pins identify this exact retained experiment; publisher authentication remains a separate act. """ from __future__ import annotations import hashlib import importlib.util import json import math import re import sys from pathlib import Path EDITION = "olmoe-0924-fidelity-v2" CONTRACT_SHA = "867d6498f374200ac1cf7566aeaa5c492d2f8b253f8048bf48594171a8fa591b" CAMPAIGN_SHA = "74edc78e934e1da2840c1fd160635f24c47d1deacc480f3bfde56f1d900a59a6" SCORER_SHA = "be5e04c87ed14ee83102f483c87fb4731f633ad5e737707e0b3924916ef34ee3" PUBLIC_CONTRACT_SHA = "87f47fa8e303f2e45cd26392c0085c119a08454367f70fa8ea282dd4e0759a5e" ARM_HASHES = { "native-bf16": "363c6f2234a72c2656bedba23ca114fed84bec57a743799e4a0e05e1278adf4c", "horizon-c64": "6b00d4a9071a1f90ee621b6de2d06b72c57ad045f95af5e7baccebd07ed87d0d", } PINNED_FILES = { "native-results.json": "6f7dc8a559df5486d745ba88ce8807756b90719256103e55247aeae3804a3d0f", "horizon-results.json": "6e7d01f10cd7b2b0715630409a0f230f1b063efbd0359e777271be71fc6fd63c", "contract.json": PUBLIC_CONTRACT_SHA, "fidelity_scorer.py": SCORER_SHA, } FILES = frozenset( { "native-results.json", "horizon-results.json", "paired-results.json", "summary.json", "contract.json", "provenance.json", "report.md", "audit-method.md", "fidelity_scorer.py", "verify-bundle.py", "manifest.json", } ) PROVENANCE = { "schema_version": "horizon.olmoe-fidelity.recovery.v2", "edition_id": EDITION, "source_run_id": "olmoe-0924-behavioral-fidelity-20260914-06", "source_campaign_sha256": CAMPAIGN_SHA, "source_contract_sha256": CONTRACT_SHA, "source_arm_sha256": ARM_HASHES, "frozen_scorer_sha256": SCORER_SHA, "original_campaign_valid": False, "original_publication_failure": "behavioral_fidelity_public_content_unsafe", "generation_validation_status": "valid", "publication_validation_status": "valid", "publication_status": "public", "evidence_level": "E3", "independent_reproduction_status": "not_attempted", "new_generations": 0, "expected_answers_loaded": False, "original_responses_preserved": True, "recovery_scope": "CPU-only successor; decoded JSON string sanitization; unchanged scoring", } _WINDOWS = re.compile(r"(?i)(?:^|[\s\"'(])(?:[a-z]:[\\/])") _UNC = re.compile(r"\\\\[^\\\s]+[\\/]") _CREDENTIAL = re.compile(r"[a-z][a-z0-9+.-]*://[^/\s:@]+:[^/\s@]+@", re.I) def digest(data: bytes) -> str: return hashlib.sha256(data).hexdigest() def json_bytes(value: object) -> bytes: return ( json.dumps(value, indent=2, sort_keys=True, ensure_ascii=False, allow_nan=False) + "\n" ).encode("utf-8") def _unique_object(items): result = {} for key, value in items: if key in result: raise ValueError("duplicate JSON key") result[key] = value return result def _invalid_constant(value): raise ValueError("non-finite JSON number") def read_json(path: Path): return json.loads( path.read_bytes(), object_pairs_hook=_unique_object, parse_constant=_invalid_constant ) def assert_safe_json(value: object) -> None: """Inspect decoded strings, never the escaping added by JSON serialization.""" if isinstance(value, str): if ( any(pattern.search(value) for pattern in (_WINDOWS, _UNC, _CREDENTIAL)) or "traceback (most recent call last):" in value.casefold() ): raise ValueError("public decoded string contains a private path or credential pattern") elif isinstance(value, dict): for key, item in value.items(): assert_safe_json(key) assert_safe_json(item) elif isinstance(value, list): for item in value: assert_safe_json(item) elif isinstance(value, float) and not math.isfinite(value): raise ValueError("non-finite number") def load_scorer(path: Path): if path.is_symlink() or digest(path.read_bytes()) != SCORER_SHA: raise ValueError("frozen scorer differs") name = "horizon_frozen_fidelity_scorer" spec = importlib.util.spec_from_file_location(name, path) module = importlib.util.module_from_spec(spec) sys.modules[name] = module # Suppress bytecode so running the verifier does not change its own inventory. prior = sys.dont_write_bytecode sys.dont_write_bytecode = True try: spec.loader.exec_module(module) finally: sys.dont_write_bytecode = prior return module def _fence(text: str, language: str = "text") -> str: longest = max((len(m[0]) for m in re.finditer(r"`+", text)), default=0) fence = "`" * max(3, longest + 1) return f"{fence}{language}\n{text}\n{fence}" def render_report(contract, native, horizon, pairs, summary) -> str: n, h = native["rows"], horizon["rows"] lines = [ "# OLMoE 0924 — BF16 and Horizon behavioral comparison", "", "Edition: `olmoe-0924-fidelity-v2`. E3, original generations retained; " "independent reproduction not attempted.", "", "The two arms completed 120 requests each. Strict JSON + natural EOS conformity " "was 29/120 for BF16 and 28/120 for Horizon: a net difference of one response " "(-0.83 percentage points). Paired outcomes include 8 conformity losses and 7 " "gains. These counts measure structure and termination, not answer correctness.", "", "| Paired category | Cases / 120 |", "|---|---:|", *[f"| {category} | {count} |" for category, count in summary["category_counts"].items()], "", "Observable parity: 19/120 (16 exact-token + 3 semantic-JSON). " "Regression-eligible BF16 responses: 29; Horizon conformity losses: 8/29 (8/120 " "overall). Both conforming: 21; BF16-only: 8; Horizon-only: 7; neither: 84.", "", "Of the 86 directionless divergences, 82 pairs conform in neither arm and 4 " "conform in both with different JSON values. Shared structural failures are not " "counted as Horizon regressions. No answer key or correctness adjudication was " "used.", "", "| Termination | BF16 | Horizon |", "|---|---:|---:|", f"| Natural EOS | {sum(r['finish_reason'] == 'eos' for r in n)}" f" | {sum(r['finish_reason'] == 'eos' for r in h)} |", f"| 128-token limit | {sum(r['finish_reason'] == 'length' for r in n)}" f" | {sum(r['finish_reason'] == 'length' for r in h)} |", "| Empty responses | 0 | 0 |", "", "All 120 Horizon request records observe zero fallback, cache misses/fills, " "expert H2D operations and expert H2D bytes. Completion of a request does not " "imply a structurally conforming or correct response.", "", "## Configuration and scope", "", "Same OLMoE 0924 checkpoint, prompt order, tokenizer input hashes, greedy " "decoding and 128-new-token cap. Horizon uses c64 all-resident Grouped T3, " "compact INT4 expert sources and FP16 activations. Dequantized values are not " "original BF16 values. The BF16 reference uses original checkpoint values and " "its declared CPU/GPU placement. No speed, memory-saving, absolute correctness " "or universal quality equivalence claim is derived from this comparison.", "", _fence(json.dumps(contract["design"], indent=2, ensure_ascii=False), "json"), "", "## Publication lineage", "", "The original campaign-result remains invalid because its publication sanitizer " "rejected six JSON-serialized output strings. Both retained generation arms and " "their supervision passed. This successor checks decoded values, retains every " "original response and token, and recomputes the unchanged scoring. No new " "generation was executed. See provenance.json for immutable source hashes and " "audit-method.md for offline verification.", "", "## All 120 paired responses", "", ] for left, right, pair in zip(n, h, pairs, strict=True): lines += [ f"### {pair['prompt_id']}", "", f"Category: `{pair['category']}`.", "", "Prompt:", _fence(json.dumps(left["prompt"], ensure_ascii=False, indent=2), "json"), "", f"BF16 — finish: {left['finish_reason']}" f"; strict contract: {pair['native_contract_valid']}", _fence(left["output_text"]), "", f"Horizon — finish: {right['finish_reason']}" f"; strict contract: {pair['horizon_contract_valid']}", _fence(right["output_text"]), "", ] return "\n".join(lines) AUDIT_METHOD = """# Verify this behavioral comparison Edition: olmoe-0924-fidelity-v2. Download the ZIP linked by https://horizonrunmap.com/evidence/behavioral-index.json and extract into a new directory. Inspect the two Python files, then run with Python 3.10 or later: python -B verify-bundle.py . No package installation, network connection, GPU or model is required. The verifier checks a fixed inventory and SHA-256, the pinned original public arm projections, the public contract, decoded-string sanitization, original input equality and all 120 pairs. It recomputes summary categories and report text with the unchanged frozen scorer. Output text is data and is never executed. The original 20260914-06 campaign's publication failed on serialized JSON escapes; its status and bytes remain untouched. provenance.json identifies this successor and the original contract, arm and campaign hashes. contract.json is an explicitly sanitized public projection; it does not claim the bytes of the private contract. The two arm JSON files preserve their original public projections byte for byte, including all prompts, output text, token IDs, finish reasons and observations. E3 describes the experiment's evidence scope. Publication verification, response format conformity, answer correctness and independent reproduction are separate. Correctness was not graded; independent reproduction was not attempted. Hashes identify bytes against this edition and its verifier; they do not independently authenticate the publisher or prove hardware execution. Re-running a model is a separate procedure. The runtime source and model weights are not included. """ def verify_bundle(directory: Path | str) -> bool: root = Path(directory) if root.is_symlink() or not root.is_dir(): raise ValueError("bundle directory invalid") if {p.name for p in root.iterdir()} != FILES: raise ValueError("bundle inventory differs") for path in root.iterdir(): if path.is_symlink() or not path.is_file() or path.stat().st_size > 8_000_000: raise ValueError("unsafe bundle entry") manifest = read_json(root / "manifest.json") if ( set(manifest) != {"schema_version", "edition_id", "files"} or manifest["schema_version"] != "horizon.olmoe-fidelity.manifest.v2" or manifest["edition_id"] != EDITION ): raise ValueError("manifest invalid") entries = manifest["files"] if ( type(entries) is not list or len(entries) != len(FILES) - 1 or {e.get("path") for e in entries} != FILES - {"manifest.json"} ): raise ValueError("manifest inventory invalid") for entry in entries: if set(entry) != {"path", "bytes", "sha256"}: raise ValueError("manifest entry invalid") data = (root / entry["path"]).read_bytes() if ( type(entry["bytes"]) is not int or len(data) != entry["bytes"] or digest(data) != entry["sha256"] ): raise ValueError("bundle hash or size differs") for name, expected in PINNED_FILES.items(): if digest((root / name).read_bytes()) != expected: raise ValueError(f"original public projection or scorer changed: {name}") docs = {name: read_json(root / name) for name in FILES if name.endswith(".json")} for value in docs.values(): assert_safe_json(value) if docs["provenance.json"] != PROVENANCE: raise ValueError("publication lineage differs") contract = docs["contract.json"] native, horizon = docs["native-results.json"], docs["horizon-results.json"] prompt_ids = contract["prompt_inputs"]["prompt_ids"] if len(prompt_ids) != 120 or len(set(prompt_ids)) != 120: raise ValueError("prompt inventory differs") scorer = load_scorer(root / "fidelity_scorer.py") for arm, name in [(native, "native-bf16"), (horizon, "horizon-c64")]: if arm["valid"] is not True or arm["contract_sha256"] != CONTRACT_SHA: raise ValueError("arm invalid") scorer.validate_arm_records(prompt_ids, name, arm["rows"]) for n, h, audit in zip( native["rows"], horizon["rows"], contract["prompt_inputs"]["audit"], strict=True ): if ( n["prompt"] != h["prompt"] or n["input_token_sha256"] != h["input_token_sha256"] or n["input_token_sha256"] != audit["input_ids_sha256"] or n["input_token_count"] != audit["input_tokens"] or h["input_token_count"] != audit["input_tokens"] ): raise ValueError("paired inputs differ") observations = h["observations"] if observations["fallback_count"] != 0 or any( v != 0 for v in observations["request_counter_deltas"].values() ): raise ValueError("Horizon operational gate failed") pairs = scorer.compare_arms(prompt_ids, native["rows"], horizon["rows"]) summary = scorer.summarize_pairs(pairs) summary["contract_sha256"] = CONTRACT_SHA if ( docs["paired-results.json"] != {"schema_version": scorer.PAIR_SCHEMA, "contract_sha256": CONTRACT_SHA, "pairs": pairs} or docs["summary.json"] != summary ): raise ValueError("paired analysis does not recompute") if (root / "report.md").read_bytes() != render_report( contract, native, horizon, pairs, summary ).encode("utf-8"): raise ValueError("report does not match validated data") if (root / "audit-method.md").read_bytes() != AUDIT_METHOD.encode("utf-8"): raise ValueError("audit method differs") return True if __name__ == "__main__": try: if len(sys.argv) != 2: raise ValueError("usage: python -B verify-bundle.py DIRECTORY") verify_bundle(sys.argv[1]) print( "PASS: olmoe-0924-fidelity-v2; 240 original responses; 120 pairs recomputed; CPU only." ) except (ValueError, OSError, KeyError, TypeError) as error: print(f"FAIL: {error}", file=sys.stderr) sys.exit(1)