scripts/repair_dl3dv_frame_download.py

"""Supplement missing protocol frames, retaining the original download evidence."""

from __future__ import annotations

import argparse
import hashlib
import json
from pathlib import Path

from mcss.data.dl3dv_download import (
    REVISION,
    atomic_write_json,
    download_selected,
    load_inventory_artifacts,
    protocol_frame_selection,
    select_files,
    write_inventory_artifacts,
)

PROJECT = Path(__file__).resolve().parents[1]


def main() -> int:
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("--previous", type=Path, required=True)
    parser.add_argument("--output", type=Path, required=True)
    parser.add_argument("--data-root", type=Path, required=True)
    parser.add_argument("--protocol-json", type=Path, required=True)
    args = parser.parse_args()
    output = args.output.resolve()
    output.relative_to(PROJECT / "outputs")
    output.mkdir(parents=True, exist_ok=False)
    old_status_path = args.previous / "status.json"
    old_status = json.loads(old_status_path.read_text(encoding="utf-8"))
    old_verification_path = args.previous / "completion_verification.json"
    old_verification = json.loads(old_verification_path.read_text(encoding="utf-8"))
    if old_status.get("status") != "complete" or old_status.get("revision") != REVISION:
        raise ValueError("previous download must be complete and use the pinned revision")
    if old_verification.get("status") != "verified":
        raise ValueError("previous file-integrity verification is absent")
    inventory = load_inventory_artifacts(args.previous)
    frames = protocol_frame_selection(
        args.protocol_json, inventory, data_root=args.data_root, target_every=8,
    )
    selected = select_files(
        inventory, args.data_root, include_images4=True, frame_ids_by_scene=frames,
    )
    if len(selected) != 12777 or sum(x.remote.size for x in selected) != 11313259491:
        raise ValueError("corrected protocol selection differs from the audited selection")
    old_by_path = {entry["repo_path"]: entry for entry in old_status["entries"]}
    old_verified = {entry["repo_path"]: entry for entry in old_verification["entries"]}
    required = {x.remote.path for x in selected}
    missing = [x for x in selected if not x.destination.is_file()]
    if len(missing) != 281 or sum(x.remote.size for x in missing) != 245981462:
        raise ValueError("missing files differ from the audited 281-file supplement")
    plan = {
        "reason": "protocol frame positions were incorrectly mapped to filename index + 1",
        "revision": REVISION,
        "mapping": "transforms_frame_order",
        "required_file_count": len(selected),
        "required_bytes": sum(x.remote.size for x in selected),
        "missing_file_count": len(missing),
        "missing_bytes": sum(x.remote.size for x in missing),
        "previous_status_sha256": hashlib.sha256(old_status_path.read_bytes()).hexdigest(),
        "previous_verification_sha256": hashlib.sha256(
            old_verification_path.read_bytes()
        ).hexdigest(),
        "retained_unused_repo_paths": sorted(set(old_by_path) - required),
        "missing_repo_paths": [x.remote.path for x in missing],
        "independent_verification": "pending",
    }
    atomic_write_json(output / "repair_plan.json", plan)
    write_inventory_artifacts(inventory, output)
    print(json.dumps({k: v for k, v in plan.items() if not isinstance(v, list)}), flush=True)
    supplement = download_selected(
        missing, status_path=output / "supplement_status.json",
        lock_path=args.previous / "download.lock", workers=4, max_attempts=3,
    )
    downloaded = {entry["repo_path"]: entry for entry in supplement["entries"]}
    entries = []
    for item in selected:
        path = item.remote.path
        if path in downloaded:
            entry = downloaded[path]
        else:
            entry = dict(old_by_path[path])
            if (
                entry.get("status") not in {"reused", "downloaded"}
                or entry.get("sha256") != old_verified[path]["local_sha256"]
            ):
                raise ValueError(f"prior status and prior verifier disagree: {path}")
            entry["status"] = "reused"
        entries.append(entry)
    # This records transfer completion only. The separate verifier re-hashes
    # every required and retained file before any benchmark restart.
    atomic_write_json(output / "status.json", {
        "schema": "mcss.dl3dv.download.v1", "status": "complete",
        "revision": REVISION, "entries": entries, "repair_plan": str(output / "repair_plan.json"),
        "independent_verification": "pending",
    })
    print(json.dumps({"status": "complete", "supplement_files": len(missing)}), flush=True)
    return 0


if __name__ == "__main__":
    raise SystemExit(main())