diff --git a/docs/verify.md b/docs/verify.md index 9b60b2c5..87503110 100644 --- a/docs/verify.md +++ b/docs/verify.md @@ -54,8 +54,8 @@ The exclusion fork (`release/unlearning-3i-atlas/`) has its own anchor, timestam 14:28:00 UTC. That is later than the main run's, because the fork was anchored after the release. The same command in that folder gives `chain head MATCH (39 entries)` and `MATCH chain entry 38`. -`verify` appends its result to `verifications.jsonl` in the run folder, so it changes that file in -your download. With the prepared data (`--data PREP`), `verify` also checks that its own recipe is the one +`verify` appends its local result to `local-verifications.jsonl` in the run folder, so the published +`verifications.jsonl` (one MATCH/MISMATCH line per segment) is left unchanged. With the prepared data (`--data PREP`), `verify` also checks that its own recipe is the one the chain was trained with, then regenerates the seeded starting weights and compares them with entry 0. That works on any device, including a CPU. The prepared-data check compares each phase's `stream.json` hash with entry 0. It does not hash the @@ -97,8 +97,8 @@ SEGMENTS="7 8" bash simple/verify_segment.sh # chosen segments ``` The script downloads the prepared data (hash-checked), the record and only the checkpoints it -needs. It prints `segment k->k+1: MATCH` or `MISMATCH` for each segment, and appends the result to -`verifications.jsonl`. +needs. It prints `segment k->k+1: MATCH` or `MISMATCH` for each segment, and appends the local +result to `local-verifications.jsonl` (the published `verifications.jsonl` stays untouched). ## Check the timestamps by hand diff --git a/simple/checker_pod.sh b/simple/checker_pod.sh index 155c37b4..03e44bfe 100644 --- a/simple/checker_pod.sh +++ b/simple/checker_pod.sh @@ -37,7 +37,7 @@ python3 - "$WORK/hf/$RUN_PATH" "$WORK/pub/$RUN_PATH" "$GPU" "$REV" <<'PY' import hashlib, json, os, shutil, subprocess, sys, time run, pub, gpu, rev = sys.argv[1:] try: - r = json.loads(open(f"{run}/verifications.jsonl").read().splitlines()[-1]) + r = json.loads(open(f"{run}/local-verifications.jsonl").read().splitlines()[-1]) except (OSError, IndexError): sys.exit("\nno result recorded; see the log above") diff --git a/simple/ovl.py b/simple/ovl.py index 94f9eb28..a3917d65 100644 --- a/simple/ovl.py +++ b/simple/ovl.py @@ -635,7 +635,7 @@ def cmd_verify(a): for k in sorted(segments): report["checks"].append({"segment": [k, k + 1], "match": verify_segment(a.data, run, chain, k)}) report["result"] = "PASS" if report["checks"] and all(c.get("match", c.get("initialization")) for c in report["checks"]) else "FAIL" - with (run / "verifications.jsonl").open("a", encoding="utf-8") as f: + with (run / "local-verifications.jsonl").open("a", encoding="utf-8") as f: f.write(json.dumps(report, sort_keys=True) + "\n") print(f"verify {report['result']}: {len(report['checks'])} checks; " f"{sum('segment' in c for c in report['checks'])} of {len(chain) - 1} segments", flush=True) diff --git a/tests/simple/test_ovl.py b/tests/simple/test_ovl.py index 90d851f0..f53a1ef5 100644 --- a/tests/simple/test_ovl.py +++ b/tests/simple/test_ovl.py @@ -207,7 +207,7 @@ def verify(self, sd): a = argparse.Namespace(run=str(self.run), data=None, weights=str(path), segment=None, random=0, all=False, anchor=None, no_anchor=True) ovl.cmd_verify(a) - return json.loads((self.run / "verifications.jsonl").read_text().splitlines()[-1]) + return json.loads((self.run / "local-verifications.jsonl").read_text().splitlines()[-1]) def test_final_and_base_weights_match_their_entries(self): self.assertEqual(self.verify(self.states[3])["checks"][0]["chain_entry"], 3) @@ -257,7 +257,7 @@ def test_unknown_or_modified_weights_fail(self): sd["b"][0] += 1e-6 with self.assertRaises(SystemExit): self.verify(sd) - report = json.loads((self.run / "verifications.jsonl").read_text().splitlines()[-1]) + report = json.loads((self.run / "local-verifications.jsonl").read_text().splitlines()[-1]) self.assertEqual(report["result"], "FAIL") self.assertIsNone(report["checks"][0]["chain_entry"]) diff --git a/tests/simple/test_ovl_anchor.py b/tests/simple/test_ovl_anchor.py index 9538d2b7..b9b69a32 100644 --- a/tests/simple/test_ovl_anchor.py +++ b/tests/simple/test_ovl_anchor.py @@ -107,7 +107,7 @@ def test_no_anchor_opt_out_warns_and_continues(self): self.assertIn("WARNING: --no-anchor", out.getvalue()) self.assertNotIn("chain head MATCH", out.getvalue()) self.assertIn("verify PASS", out.getvalue()) - record = json.loads((self.run / "verifications.jsonl").read_text().splitlines()[-1]) + record = json.loads((self.run / "local-verifications.jsonl").read_text().splitlines()[-1]) self.assertEqual(record["result"], "PASS") def test_no_anchor_opt_out_still_fails_bad_weights(self): @@ -142,7 +142,7 @@ def test_anchor_result_is_recorded(self): self.write(self.chain) with contextlib.redirect_stdout(io.StringIO()): ovl.cmd_verify(args(self.run)) - record = json.loads((self.run / "verifications.jsonl").read_text().splitlines()[-1]) + record = json.loads((self.run / "local-verifications.jsonl").read_text().splitlines()[-1]) self.assertEqual(record["checks"][0]["anchor"], str(self.run / "anchor.json")) self.assertTrue(record["checks"][0]["match"]) self.assertEqual(record["result"], "PASS") @@ -187,7 +187,7 @@ def test_both_timestamps_are_checked_against_the_anchor_bytes(self): self.assertEqual(c.args[0], self.run / f"anchor.{c.args[1]}.tsr") self.assertEqual(c.args[2][0], "-data") self.assertIn("anchor timestamp digicert: OK", out) - record = json.loads((self.run / "verifications.jsonl").read_text().splitlines()[-1]) + record = json.loads((self.run / "local-verifications.jsonl").read_text().splitlines()[-1]) self.assertEqual(set(record["checks"][0]["timestamps"]), set(ovl.stamp.TSA)) def test_edited_anchor_fails_its_timestamps(self): diff --git a/tests/simple/test_simple_verify.py b/tests/simple/test_simple_verify.py index 57a679c6..6a970578 100644 --- a/tests/simple/test_simple_verify.py +++ b/tests/simple/test_simple_verify.py @@ -2,7 +2,7 @@ Tests that: 1. An invocation of `simple/ovl.py verify` with no check options (zero checks performed) - does not report PASS, exits nonzero, and records result: FAIL in verifications.jsonl. + does not report PASS, exits nonzero, and records result: FAIL in local-verifications.jsonl. 2. Invoking with `--all` or `--segment` without `--data` does not report PASS and exits nonzero. 3. Legitimate verification paths (e.g. valid matching weights) still pass and exit 0. 4. Mismatched weights fail and exit nonzero. @@ -40,6 +40,7 @@ def _run_verify(self, run_dir, extra_args=None): cmd.extend(extra_args) env = dict(os.environ) env["PYTHONPATH"] = f"{ROOT / 'simple'}" + env.setdefault("OVL_DEVICE", "cpu") res = subprocess.run(cmd, capture_output=True, text=True, env=env) return res.returncode, res.stdout, res.stderr @@ -58,10 +59,10 @@ def test_zero_checks_fails_closed(self): self.assertNotEqual(code, 0, "Zero-check invocation must exit with a nonzero code") # 3. Records a failure accurately in verification report - verif_file = d / "verifications.jsonl" - self.assertTrue(verif_file.exists(), "verifications.jsonl must exist") + verif_file = d / "local-verifications.jsonl" + self.assertTrue(verif_file.exists(), "local-verifications.jsonl must exist") lines = verif_file.read_text().splitlines() - self.assertTrue(len(lines) > 0, "verifications.jsonl must contain records") + self.assertTrue(len(lines) > 0, "local-verifications.jsonl must contain records") last_record = json.loads(lines[-1]) self.assertEqual(last_record.get("result"), "FAIL", "Verification record result must be FAIL") self.assertEqual(len(last_record.get("checks", [])), 0, "Checks array must be empty") @@ -77,7 +78,7 @@ def test_all_flag_without_data_fails_closed(self): self.assertIn("verify FAIL", out, "--all without --data must report FAIL") self.assertNotEqual(code, 0, "--all without --data must exit nonzero") - verif_file = d / "verifications.jsonl" + verif_file = d / "local-verifications.jsonl" last_record = json.loads(verif_file.read_text().splitlines()[-1]) self.assertEqual(last_record.get("result"), "FAIL") @@ -117,7 +118,7 @@ def test_legitimate_weights_verification_passes(self): self.assertEqual(code, 0, f"Matching weights should pass. out: {out}, err: {err}") self.assertIn("verify PASS: 1 checks", out) - verif_file = d / "verifications.jsonl" + verif_file = d / "local-verifications.jsonl" last_record = json.loads(verif_file.read_text().splitlines()[-1]) self.assertEqual(last_record.get("result"), "PASS") self.assertEqual(len(last_record.get("checks")), 1) @@ -138,7 +139,7 @@ def test_legitimate_weights_mismatch_fails(self): self.assertNotEqual(code, 0, "Mismatched weights must exit nonzero") self.assertIn("verify FAIL", out) - verif_file = d / "verifications.jsonl" + verif_file = d / "local-verifications.jsonl" last_record = json.loads(verif_file.read_text().splitlines()[-1]) self.assertEqual(last_record.get("result"), "FAIL") self.assertEqual(len(last_record.get("checks")), 1) diff --git a/tests/simple/test_verification_record_schema.py b/tests/simple/test_verification_record_schema.py new file mode 100644 index 00000000..fe6e1ea5 --- /dev/null +++ b/tests/simple/test_verification_record_schema.py @@ -0,0 +1,143 @@ +"""Regression for AOSSIE-Org/OpenVerifiableLLM#211. + +Published release entries in verifications.jsonl use: + {checker, code, driver, gpu, result: MATCH|MISMATCH, segment: int} + +cmd_verify used to append a different local report shape + {time, environment, chain_head, checks[], result: PASS|FAIL} +to that same file, mixing incompatible schemas. + +Expected: verify leaves the published file alone and writes local results to +local-verifications.jsonl. +""" + +import hashlib +import json +import os +from pathlib import Path +import shutil +import subprocess +import sys +import tempfile +import unittest + +ROOT = Path(__file__).resolve().parents[2] +FIXTURE_DIR = ROOT / "tests" / "fixtures" / "simple-run" +PYTHON = sys.executable + +PUBLISHED = { + "checker": "project second pod", + "code": "ovl.py 202ac6ce", + "driver": "595.91.07", + "gpu": "NVIDIA GeForce RTX 5090", + "result": "MATCH", + "segment": 0, +} + + +def _setup_run_dir(dest: Path) -> None: + if FIXTURE_DIR.exists() and (FIXTURE_DIR / "chain.jsonl").exists(): + shutil.copytree(FIXTURE_DIR, dest, dirs_exist_ok=True) + else: + dest.mkdir(parents=True, exist_ok=True) + e0 = {"index": 0, "kind": "initial", "state": "0" * 64, "model": "0" * 64, "previous": None} + h0 = hashlib.sha256(json.dumps(e0, sort_keys=True).encode()).hexdigest() + e1 = {"index": 1, "kind": "final", "state": "1" * 64, "model": "1" * 64, "previous": h0} + (dest / "chain.jsonl").write_text(json.dumps(e0) + "\n" + json.dumps(e1) + "\n") + + +def _run_verify(run_dir: Path, extra_args=None): + cmd = [PYTHON, str(ROOT / "simple" / "ovl.py"), "verify", "--run", str(run_dir), "--no-anchor"] + if extra_args: + cmd.extend(extra_args) + env = dict(os.environ) + env["PYTHONPATH"] = f"{ROOT / 'simple'}" + env.setdefault("OVL_DEVICE", "cpu") + res = subprocess.run(cmd, capture_output=True, text=True, env=env) + return res.returncode, res.stdout, res.stderr + + +class VerificationRecordSchemaTests(unittest.TestCase): + def test_verify_does_not_append_local_report_to_published_verifications(self): + """Published verifications.jsonl must stay in the release schema after verify.""" + with tempfile.TemporaryDirectory() as tmp: + d = Path(tmp) + _setup_run_dir(d) + published_line = json.dumps(PUBLISHED, sort_keys=True) + "\n" + (d / "verifications.jsonl").write_text(published_line) + + code, out, err = _run_verify(d) + self.assertNotEqual(code, 0, "zero-check verify should still fail closed") + self.assertIn("verify FAIL", out) + + after = (d / "verifications.jsonl").read_text() + self.assertEqual( + after, + published_line, + "verify must not append a local report into published verifications.jsonl", + ) + lines = [json.loads(l) for l in after.splitlines() if l.strip()] + self.assertEqual(len(lines), 1) + self.assertEqual(lines[0]["result"], "MATCH") + self.assertEqual(lines[0]["segment"], 0) + self.assertNotIn("checks", lines[0]) + self.assertNotIn("environment", lines[0]) + + def test_verify_writes_local_report_to_local_verifications_jsonl(self): + """Local PASS/FAIL reports belong in local-verifications.jsonl.""" + with tempfile.TemporaryDirectory() as tmp: + d = Path(tmp) + _setup_run_dir(d) + (d / "verifications.jsonl").write_text(json.dumps(PUBLISHED, sort_keys=True) + "\n") + + code, out, err = _run_verify(d) + self.assertNotEqual(code, 0) + + local = d / "local-verifications.jsonl" + self.assertTrue(local.exists(), "local-verifications.jsonl must be created") + record = json.loads(local.read_text().splitlines()[-1]) + self.assertEqual(record["result"], "FAIL") + self.assertIn("checks", record) + self.assertIn("environment", record) + self.assertIn("chain_head", record) + self.assertIn("time", record) + self.assertNotIn("segment", record) # published field; local nests under checks + self.assertNotEqual(record["result"], "MATCH") + self.assertNotEqual(record["result"], "MISMATCH") + + def test_weights_pass_still_records_only_in_local_file(self): + """A successful local verify must not touch published verifications.jsonl.""" + import torch + + sys.path.insert(0, str(ROOT / "simple")) + from ovl import model_digest + + with tempfile.TemporaryDirectory() as tmp: + d = Path(tmp) + _setup_run_dir(d) + published_line = json.dumps(PUBLISHED, sort_keys=True) + "\n" + (d / "verifications.jsonl").write_text(published_line) + + sd = {"linear.weight": torch.randn(4, 4)} + weights_file = d / "weights.pt" + torch.save(sd, weights_file) + digest = model_digest(sd) + chain_lines = (d / "chain.jsonl").read_text().splitlines() + last_entry = json.loads(chain_lines[-1]) + last_entry["model"] = digest + chain_lines[-1] = json.dumps(last_entry) + (d / "chain.jsonl").write_text("\n".join(chain_lines) + "\n") + + code, out, err = _run_verify(d, ["--weights", str(weights_file)]) + self.assertEqual(code, 0, f"expected PASS; out={out!r} err={err!r}") + self.assertIn("verify PASS", out) + + self.assertEqual((d / "verifications.jsonl").read_text(), published_line) + record = json.loads((d / "local-verifications.jsonl").read_text().splitlines()[-1]) + self.assertEqual(record["result"], "PASS") + self.assertEqual(len(record["checks"]), 1) + self.assertTrue(record["checks"][0].get("match")) + + +if __name__ == "__main__": + unittest.main()