#!/usr/bin/env python3 """Freeze EXP-3D-RGBD-SEGMENT-DIVERSITY-V21: two GPU arms of the frozen V17 code (G1 and G5). Both arms run the frozen V17 L training and EVAL code unchanged, each in its own V17-style root outputs/EXP-3D-RGBD-SEGMENT-DIVERSITY-V21//EXP-3D-RGBD-LONG-TRAIN-V17. The frozen V17 prepare makes each root; then the V21 contract is applied and registered in the root's own lock: - both arms: device cuda on GPU 4 (launched through cv_gpu_run.sh), TF32 off. GPU training is not bit-reproducible, so there is no bit-identical control; the control file the frozen EVAL code requires says so (status PASS, meaning NOT_APPLICABLE_GPU); - G5: the SEG-V1 segment records are merged into the manifest, the frame-role lock and the data hashes, and the variant's train set, which the frozen code reads under the key TRAIN72, becomes TRAIN72 plus every valid SEG-V1 segment. Then V21's freeze record (freeze.json) holds every hash, before any V21 training. Run from the repository root under cv_host_run.py (records store zonghan_o paths). """ from __future__ import annotations import datetime import hashlib import importlib.util import json import shutil from pathlib import Path from mcss.mechanism_pilot.geometry_carrier_experiment import verify_lock from mcss.mechanism_pilot.rgbd_stream_write import stream_frame_ids V21 = Path("outputs/EXP-3D-RGBD-SEGMENT-DIVERSITY-V21") PLAN = V21 / "PLAN_BEFORE_V21_DATA.md" PLAN_SHA256 = "f04e581e398ff1a47e14f3a0d46df2ebb0275a5d017b03ba3ce2fb26f74f2d24" SEG = Path("outputs/EXP-3D-RGBD-SEGMENT-DATA-V1") V17 = Path("outputs/EXP-3D-RGBD-LONG-TRAIN-V17") V17_PREREGISTRATION_SHA256 = "36b8aa6e4e527025d3746aa7af5ffc060fbf7683fc1a26ff50a94e3c744c1d1f" V17_NAME = "EXP-3D-RGBD-LONG-TRAIN-V17" ARMS = ("G1", "G5") GPU = "4" SCRIPTS = ( Path("scripts/prepare_rgbd_segment_diversity_v21.py"), Path("scripts/run_rgbd_segment_diversity_v21.sh"), Path("scripts/eval_rgbd_segment_diversity_v21.sh"), Path("scripts/analyze_rgbd_segment_diversity_v21.py"), Path("scripts/prepare_rgbd_segment_data.py"), Path.home() / "cv_tools/fp32_run.py", Path.home() / "cv_tools/cv_gpu_run.sh", Path.home() / "cv_tools/cv_host_run.py", ) def sha(path): return hashlib.sha256(Path(path).read_bytes()).hexdigest() def read(path): return json.loads(Path(path).read_text()) def write(path, value): Path(path).parent.mkdir(parents=True, exist_ok=True) Path(path).write_text(json.dumps(value, indent=2, sort_keys=True) + "\n") def module(name, path): spec = importlib.util.spec_from_file_location(name, path) loaded = importlib.util.module_from_spec(spec) spec.loader.exec_module(loaded) return loaded def relock(root, names, extra_data=None, note=None): """Re-register the rewritten inputs (and added data) in the root's own preregistration.""" lock = read(root / "preregistration.json") for name in names: key = str((root / name).resolve()) if key not in lock["input_sha256"]: raise KeyError(key) lock["input_sha256"][key] = sha(root / name) if extra_data: lock["data_sha256"].update(extra_data) lock["v21"] = note write(root / "preregistration.json", lock) def seg_records(): integrity = read(SEG / "preparation_integrity.json") manifest = read(SEG / "manifest_seg.json") if integrity["status"] != "PASS" or integrity["candidate_lock_sha256"] != sha( SEG / "candidate_lock.json" ): raise PermissionError("SEG-V1 must PASS against its frozen lock") if manifest["candidate_lock_sha256"] != sha(SEG / "candidate_lock.json"): raise PermissionError("SEG-V1 manifest is not from its lock") if read(SEG / "candidate_lock.json")["plan_sha256"] != PLAN_SHA256: raise PermissionError("SEG-V1 was not locked under the V21 plan") return manifest["scenes"] def make_arm(arm, prepare_v17, stream_frame_ids, segments): root = (V21 / arm / V17_NAME).resolve() if root.exists(): raise FileExistsError(root) root.mkdir(parents=True) for name in ("PLAN_BEFORE_V17_TRAINING.md", "PROTOCOL.md"): shutil.copyfile(V17 / name, root / name) prepare_v17(root) contract = read(root / "training_contract.json") rewritten = ["training_contract.json"] added = {} if arm == "G5": manifest = read(root / "scene_split.json") ids = {r["scene_id"] for r in manifest["scenes"]} if ids & {r["scene_id"] for r in segments}: raise PermissionError("Segment ids collide with the manifest") manifest["scenes"] = manifest["scenes"] + segments write(root / "scene_split.json", manifest) roles = read(root / "frame_role_lock.json") for record in segments: roles["roles"][record["scene_id"]] = record["roles"] extra = stream_frame_ids(record, "A") if extra is None or len(extra) != 4: raise PermissionError(f"Segment without 4 free frames: {record['scene_id']}") roles["train_extra_view_frames"][record["scene_id"]] = list(extra) roles["manifest_sha256"] = sha(root / "scene_split.json") roles["v21_segments"] = "SEG-V1 records merged by the V21 freeze" write(root / "frame_role_lock.json", roles) contract["train_sets"]["TRAIN72"] = sorted( contract["train_sets"]["TRAIN72"] + [r["scene_id"] for r in segments] ) contract["train_set_note"] = ( "V21 G5: the frozen V17 code reads the variant's train set under the key TRAIN72; " "here it holds TRAIN72 plus every valid SEG-V1 segment" ) hashes = read(root / "audit/data_hashes.json") for record in segments: for frame in record["frames"]: for kind in ("rgb", "depth"): path = str(Path(frame[kind])) added[path] = hashes[path] = sha(path) write(root / "audit/data_hashes.json", hashes) rewritten += ["scene_split.json", "frame_role_lock.json", "audit/data_hashes.json"] contract.update( device="cuda", gpu=GPU, tf32="off (NVIDIA_TF32_OVERRIDE=0 and the fp32 launcher)", v21_arm=arm, execution="six trainings (two arms x three seeds) concurrently on GPU 4 through " "cv_gpu_run.sh; EVAL on CPU, AVX2", ) write(root / "training_contract.json", contract) write( root / "audit/v11_prefix_control.json", { "status": "PASS", "meaning": "NOT_APPLICABLE_GPU", "note": "V21 GPU arm: GPU training is not bit-reproducible (HIT-REPRO-V1), so the V17 " "built-in control cannot apply; this file only lets the frozen V17 EVAL code run", }, ) write(root / "tests.json", read(V21 / "tests.json")) note = { "arm": arm, "plan_sha256": PLAN_SHA256, "rewritten": rewritten, "added_data": len(added), } relock(root, rewritten, added, note) return root def main(): if sha(PLAN) != PLAN_SHA256: raise PermissionError("The V21 plan changed") if sha(V17 / "preregistration.json") != V17_PREREGISTRATION_SHA256: raise PermissionError("V17 is not the frozen V17") if read(V21 / "tests.json")["status"] != "PASS": raise PermissionError("The full suite must PASS before the V21 freeze") if (V21 / "freeze.json").exists(): raise FileExistsError("V21 is frozen once") segments = seg_records() v17_prepare = module("v17_prepare", "scripts/prepare_rgbd_long_train_experiment.py") roots = {arm: make_arm(arm, v17_prepare.prepare, stream_frame_ids, segments) for arm in ARMS} for arm, root in roots.items(): manifest, config = verify_lock(root) names = set(config["train_sets"]["TRAIN72"]) print(arm, "train records", len(names), "manifest records", len(manifest["scenes"])) write( V21 / "freeze.json", { "status": "FROZEN_BEFORE_V21_TRAINING", "frozen_utc": datetime.datetime.now(datetime.UTC).isoformat(), "plan_sha256": PLAN_SHA256, "seg_v1": { "candidate_lock_sha256": sha(SEG / "candidate_lock.json"), "manifest_sha256": sha(SEG / "manifest_seg.json"), "integrity_sha256": sha(SEG / "preparation_integrity.json"), "segments": len(segments), "scenes_with_segments": len({r["source_scene_id"] for r in segments}), }, "arms": { arm: { "root": str(root), "preregistration_sha256": sha(root / "preregistration.json"), "training_contract_sha256": sha(root / "training_contract.json"), } for arm, root in roots.items() }, "v17_preregistration_sha256": V17_PREREGISTRATION_SHA256, "scripts_sha256": {str(p): sha(p) for p in SCRIPTS}, "tests_sha256": sha(V21 / "tests.json"), }, ) print("V21_FROZEN", {a: str(r) for a, r in roots.items()}, "segments", len(segments)) if __name__ == "__main__": main()