"""Regenerate the static qualification report and summaries from frozen raw results.""" from __future__ import annotations import argparse import csv import gzip import json import shutil from pathlib import Path import numpy as np from mcss.mechanism_pilot.small_training import sha, write_json from mcss.mechanism_pilot.static_evaluation import summarize from mcss.mechanism_pilot.statistics import paired_scene_bootstrap def report(root: Path, docs: Path): def load(name): return json.loads((root / name).read_text()) old_raw = load("old_static/static_results.json") new_raw = load("unseen_static/static_results.json") old, new = summarize(old_raw), summarize(new_raw) assert old == load("old_static/bootstrap_results.json") assert new == load("unseen_static/bootstrap_results.json") result = {"old": old, "unseen": new} primary = new["primary"] methods, pairs = primary["methods"], primary["paired_depth_absrel"] def metric(name, metric_name="depth_absrel"): return methods[name]["full_image"]["metrics"][metric_name]["mean"] def by_scene(name): return methods[name]["full_image"]["metrics"]["depth_absrel"]["per_scene"] scenes = sorted(by_scene("A")) fixed = (metric("A") + metric("B")) / 2 residual = (metric("Residual_A") + metric("Residual_B")) / 2 # Same locked average of A/B as the primary full-context comparison. residual_delta = paired_scene_bootstrap( { s: ( by_scene("Residual_A")[s] + by_scene("Residual_B")[s] - by_scene("A")[s] - by_scene("B")[s] ) / 2 for s in scenes } ) result["residual_average_AB_minus_fixed"] = residual_delta write_json(root / "static_results.json", {"old": old_raw, "unseen": new_raw}) write_json(root / "bootstrap_results.json", result) write_json( root / "residual_control.json", { "training": load("residual_30000/summary.json"), "pilot": load("residual/summary.json"), "budget_amendment": load("residual_training_amendment.json"), "average_AB_minus_fixed": residual_delta, "per_context_and_state_controls": { k: v for k, v in pairs.items() if k.startswith("residual") }, "interpretation": "B improves; A worsens; average AB CI crosses zero. " "Shuffled/zero controls do not reliably hurt A. No general fixed-decoder " "bottleneck or successful state-dependent residual qualification is established.", "limitation": "Shared residual controls use A; B-specific bypass attribution " "not tested. Averaged loss is diagnostic across the two predeclared contexts; " "not selecting B post hoc.", }, ) write_json( root / "dataset_manifest.json", { "old": json.loads( Path("outputs/EXP-3D-20260927-small-training-v1/data/manifest.json").read_text() ), "unseen": load("unseen_data/manifest.json"), }, ) write_json( root / "scene_split.json", { "TRAIN_SCENE_SET": ["ai_001_001", "ai_002_001", "ai_003_001"], "UNSEEN_DEV_SCENE_SET": scenes, "candidate_lock": load("unseen_data/candidate_lock.json"), "roles": load("unseen_data/role_lock.json"), "qualification_split": "Research dev drawn only from official and repository train; " "never checkpoint training. Existing protected/test/official validation not opened.", }, ) write_json( root / "data_quality_audit.json", { "old_rgb": load("rgb_audit/integrity.json"), "ai003": load("rgb_audit/ai003_black_root_cause.json"), "official_status": load("rgb_audit/official_source_validity.json"), "unseen": load("unseen_data/frame_quality.json"), "unseen_native_geometry": load("unseen_data/geometry_checks.json"), "no_old_scene_removed": True, }, ) for target, source in { "support_audit.json": "support_audit.json", "geometry_audit.json": "geometry_audit.json", "camera_geometry_audit.json": "camera_geometry_audit.json", "volume_coverage.json": "volume_coverage.json", }.items(): write_json( root / target, { "old": load("geometry_audit/" + source), "unseen": load("unseen_geometry_audit/" + source), }, ) for source, target in [ ("rgb_audit/ai003_frame_audit.csv", "ai003_frame_audit.csv"), ("rgb_audit/ai003_black_root_cause.json", "ai003_black_root_cause.json"), ("geometry_audit/support_audit_ai_003_001.json", "support_audit_ai003.json"), ("geometry_audit/depth_definition.md", "depth_definition.md"), ]: shutil.copy2(root / source, root / target) checkpoint = load("old_static/lock.json")["checkpoint"] write_json( root / "checkpoint_manifest.json", { "carrier": checkpoint, "residual": { "path": str(root / "residual_30000/best.pt"), "sha256": sha(root / "residual_30000/best.pt"), "selected_step": 7500, }, "previous_checkpoint_hash_match": True, "carrier_retrained": False, }, ) gain, damage = pairs["full_context_gain"], pairs["wrong_scene_damage"] assert ( gain["ci95"][1] < 0 and damage["ci95"][0] < 0 ) # Recorded run, no success threshold tuning. old_methods = old["primary"]["methods"] old_values = { k: old_methods[k]["full_image"]["metrics"]["depth_absrel"]["mean"] for k in ("A", "B", "anchor") } status = { "CHECKPOINT": checkpoint.get( "path", str( root.parent / "EXP-3D-20260927-centered-training-1000a-600b-v1/training/phase_b_final.pt" ), ), "CHECKPOINT_HASH": checkpoint["sha256"], "DATA_PIPELINE_STATUS": "PASS_SOURCE_BLACK_RETAINED", "CAMERA_GEOMETRY_STATUS": "PASS_ELIGIBLE_COHORT", "DEPTH_SEMANTICS_STATUS": "PASS_RAY_DISTANCE_UNCONDITIONAL_RENDER_DEPTH", "PREVIOUS_DEPTH_METRIC_INVALID": False, "VOLUME_COVERAGE_STATUS": "LIMITED_SOME_QUERY_SURFACES_UNCOVERED", "AI003_BLACK_ROOT_CAUSE": "SOURCE_FRAME_IS_BLACK", "AI003_SUPPORT_ROOT_CAUSE": "A_HAS_ZERO_TWO_VIEW_SUPPORTED_CANDIDATES", "N_CHECKPOINT_TRAIN_SCENES": 3, "N_UNSEEN_DEV_SCENES": len(scenes), "UNSEEN_DEV_STATUS": "EVALUATED", "OLD_CONTEXT_A_ABSREL": old_values["A"], "OLD_CONTEXT_B_ABSREL": old_values["B"], "OLD_ANCHOR_ABSREL": old_values["anchor"], "UNSEEN_FIXED_A_ABSREL": metric("A"), "UNSEEN_FIXED_B_ABSREL": metric("B"), "UNSEEN_ANCHOR_ABSREL": metric("anchor"), "UNSEEN_WRONG_SCENE_ABSREL": metric("wrong_scene"), "FULL_CONTEXT_MINUS_ANCHOR_GAIN": gain["mean"], "FULL_CONTEXT_MINUS_ANCHOR_CI": gain["ci95"], "WRONG_SCENE_DAMAGE": damage["mean"], "WRONG_SCENE_DAMAGE_CI": damage["ci95"], "R_FIXED_ABSREL": fixed, "R_RESIDUAL_ABSREL": residual, "R_RESIDUAL_MINUS_FIXED": residual - fixed, "TRAIN_QUERY_ABSREL": old["secondary"]["methods"]["A"]["full_image"]["metrics"][ "depth_absrel" ]["mean"], "HELDOUT_QUERY_ABSREL": old_values["A"], "TRAIN_HELDOUT_GAP_DIAGNOSIS": "CONSISTENT_WITH_SUPPORT_VIEW_AND_VOLUME_LIMITATIONS; " "CAUSE_NOT_ISOLATED; SOURCE_BLACK_SHARED; MEMORIZATION_NOT_IDENTIFIED", "STATIC_STATE_STATUS": "NOT_ESTABLISHED", "CARRIER_STATUS": "NOT_QUALIFIED_FOR_UNSEEN_GEOMETRY", "DYNAMIC_TTT_RUN": False, "TESTS": load("tests.json")["summary"], "FINAL_INTEGRITY": "PASS", "REPORT_DIR": str(docs.resolve()), } write_json(root / "status.json", status) terminal = ( "=" * 60 + "\n3D STATIC CARRIER ATTRIBUTION + UNSEEN QUALIFICATION\n" + "=" * 60 + "\n" ) terminal += ( "\n".join( f"{k}={json.dumps(v, ensure_ascii=False) if not isinstance(v, str) else v}" for k, v in status.items() ) + "\n" ) (root / "STATUS.md").write_text("```text\n" + terminal + "```\n") (root / "terminal_summary.txt").write_text(terminal) table = ( "| Method | Depth AbsRel ↓ | RGB MSE ↓ | SSIM ↑ | RMSE ↓ | δ1 ↑ | Opacity | Coverage |\n" ) table += "|---|---:|---:|---:|---:|---:|---:|---:|\n" for name in [ "A", "B", "anchor", "prior", "wrong_scene", "Residual_A", "Residual_B", "Residual_shuffled", "Residual_zero", "feature_shuffle", "channel_permutation", "state_mask", ]: table += ( "| " + name + " | " + " | ".join( f"{metric(name, m):.6f}" for m in [ "depth_absrel", "rgb_mse", "rgb_ssim", "depth_rmse", "depth_delta1", "opacity", "coverage", ] ) + " |\n" ) scene_table = ( "| Scene | A | B | Anchor | Wrong scene | Full-context gain | Residual A | Residual B |\n" ) scene_table += "|---|---:|---:|---:|---:|---:|---:|---:|\n" for scene in scenes: vals = [by_scene(n)[scene] for n in ("A", "B", "anchor", "wrong_scene")] vals += [ gain["per_scene"][scene], by_scene("Residual_A")[scene], by_scene("Residual_B")[scene], ] scene_table += f"| {scene} | " + " | ".join(f"{v:.6f}" for v in vals) + " |\n" diagnostic = {} for name in [ "feature_shuffle", "channel_permutation", "state_mask", "wrong_scene", "Residual_channel_permutation", ]: selected = [r for r in new_raw if r["method"] == name] diagnostic[name] = { k: float(np.mean([r["prediction_change_from_A"][k]["rms"] for r in selected])) for k in ["rgb", "depth"] } write_json( root / "state_use_diagnostics.json", { "diagnostic_only": True, "mean_prediction_rms_difference_vs_fixed_A": diagnostic, "note": "Feature-channel permutation leaves fixed renderer unchanged by design: " "typed density/color fields are unchanged. Residual can read feature channels. " "Different output is sensitivity, not proof of useful scene understanding.", }, ) with (root / "static_results.csv").open("w") as f: fields = [ "dataset", "scene_id", "query_id", "cohort", "method", "rgb_mse", "rgb_ssim", "depth_absrel", "depth_rmse", "depth_delta1", "opacity", "coverage", "seconds", ] writer = csv.DictWriter(f, fieldnames=fields, extrasaction="ignore") writer.writeheader() for dataset, rows in [("old", old_raw), ("unseen", new_raw)]: writer.writerows({"dataset": dataset, **r} for r in rows) readme = f"""# EXP-3D-STATIC-CARRIER-ATTRIBUTION-V1 **STATIC_STATE_STATUS=NOT_ESTABLISHED.** The frozen 1000A+600B B-final carrier does not qualify for unseen-scene geometry. Seven checkpoint-unseen scenes were evaluated; full context is worse than anchor in six scenes and ties in one. No Dynamic TTT, controller training, write-rule tuning or new history experiment was run. ## Frozen experiment and data Carrier SHA256 `{checkpoint["sha256"]}` exactly matches the previous experiment. 8³ volume, 128 candidates, centered bounds, RGB preprocessing, calibrated metric cameras, normalized-ray depth and 64 renderer samples remain unchanged. Old A/B roles and primary frames 8/9 are retained. All 90 old static rows reproduce exactly for RGB MSE/SSIM, depth AbsRel/RMSE/δ1, opacity and coverage, with identical query-camera hashes. The prelocked eight candidates came only from the existing research-train and Hypersim official-train pools. Seven qualified. ai_005_003 failed the pre-existing 10 mm native-position p95 check (11.24169 mm); this is a threshold failure, not proof of a camera bug. It was retained and never replaced after selection. Frame quality and A/B feasibility used only metadata, RGB/depth validity and cameras; no model score. Download: 81,591,203 bytes using bounded HTTP ranges, no full archive. Checkpoint-unseen IDs: {", ".join(scenes)}. Training IDs are ai_001_001, ai_002_001, ai_003_001, verified against manifest, logs and checkpoint provenance. Official volume/archive/asset identities are distinct; supplier reuse across distinct assets cannot be independently excluded. ai_007_002 has an older `previously_observed` flag: it is unseen to this checkpoint, not claimed historically unobserved by all researchers. No protected/test or official validation split was opened. See `scene_split.json`, `unseen_independence_audit.json`, and the preserved candidate/data/role locks. ## Unseen qualification All rows use scene-equal means; primary full-context error is the predeclared average of A and B. Positive gain means anchor error minus full-context error. Bootstrap resamples seven paired scenes, 10,000 draws, seed 20260927, percentile 95% CI. {table} Full-context gain: **{gain["mean"]:.6f}**, CI **[{gain["ci95"][0]:.6f}, {gain["ci95"][1]:.6f}]**. Wrong-scene damage relative to A: **{damage["mean"]:.6f}**, CI **[{damage["ci95"][0]:.6f}, {damage["ci95"][1]:.6f}]**. Both qualification requirements fail. Every leave-one-scene-out full-context gain remains negative; this failure is not caused by one scene. All states were sealed before query/GT access and served two queries unchanged. That structural property alone does not establish useful state. {scene_table} Metrics include every GT-valid depth pixel, including uncovered rays. Coverage means opacity >1e-6, not true visibility. A/B have higher mean opacity than anchor, yet worse depth. Since rendered depth is the unnormalized sum(weights * ray distance), opacity can affect apparent accuracy; no claim of opacity-independent geometry success is made. `opacity_diagnostic.json` reports separately labeled diagnostic normalization and common-high-opacity checks. They do not replace the frozen primary metric. In that diagnostic, A/B/anchor AbsRel is .805089/.748956/.544680: anchor still wins. The joint high-opacity subset is empty for some scenes; its full-cohort macro is null. Common-visible and volume/support bucket metrics are in `bootstrap_results.json` and raw records, including empty-mask counts. Missing scenes are not silently removed from regional macros. Per-object metrics are unavailable because object segmentation was not acquired; per-support-bucket metrics are supplied as requested. ## Why did ai_003_001 fail? 1. **Why is query RGB black?** All published source JPEGs 0–15 are already exactly zero RGB. PIL/OpenCV agree; resizing and model/evaluator tensors preserve them. ROOT_CAUSE=SOURCE_FRAME_IS_BLACK. Official metadata includes these images as train, with no exclusion and a non-BAD camera. Why the published source is black—render failure, transition, or another source issue—remains **UNKNOWN**. It is not evidence of a decoder/range/preprocessing bug. ai003 was never removed. 2. **Why no A two-view support?** Projection of all 128 candidates through cameras 0/1/2 gives 13 single-view candidates and **zero** candidates with >=2 supporting views. B has 3 such candidates. The full coordinate/projection/validity records are in `support_audit_ai003.json`. A+stream has 4, but that does not repair static A. 3. **Why A equals anchor?** The current fusion/completion path needs two-view support. With none, both materialize the same prior branch and give identical predictions. This is expected behavior of the frozen carrier, not a newly discovered code bug. Primary GT surfaces are 100% inside its volume; A-supported surface neighborhoods are 0%. Inside the box does not imply observed support. Unseen geometry also reveals coverage failure: ai009 primary GT surfaces are 0% inside the volume on both queries; ai007 query 15 is only 0.06% inside, and ai008 queries are 30.2% and 6.67% inside. Geometry feasibility of the context does not ensure query coverage. These scenes stay in the locked primary cohort; no post-score filtering or volume expansion is applied. Thus support design is an evidenced bottleneck; representation and training contributions are not isolated. Camera/depth audit found no convention/unit/resize bug: old projection roundtrip max 1.17e-11 pixels, ray-depth roundtrip max 3.55e-15 m; native GT positions satisfy the locked check. PREVIOUS_DEPTH_METRIC_INVALID=false. Depth is metric ray distance; opacity normalization is a separate readout question. All old scenes received the same audit. See `depth_definition.md`, `geometry_audit.json`, `volume_coverage.json`. ## Residual readout and state-use attribution The existing 308-parameter 14→16→4 residual head uses exactly the same frozen states as R-Fixed. Adam lr=.001, clip=1; only original three training scenes, query frames 12–14 for fitting and 15 for train-side validation, A/B contexts. The initial 3,000-step pilot was still improving, so a documented budget-only amendment extended fresh same-seed training to 30,000 **before any residual primary/unseen evaluation**. First 3,000 logs match exactly. The fixed validation rule selected step **7,500**. No unseen/8–9 label was used for residual training or selection. Selected train-side validation objective: .303350 versus .375932 initially; terminal 30,000-step validation worsened to .423491. The final 3,000-step training window still improved 1.45%; global convergence is not established. This control is trained and validated, but its limited capacity/training does not rule out every possible readout. Head linear operations: 11,796,480 FLOPs per 128×160 query (multiply+add=2; excludes renderer/activation). Measured fixed 5.21 ms, residual 10.81 ms including state safety checks; head alone .0478 ms. See training curves, audit and timing definitions. Average A/B: Fixed {fixed:.6f}, Residual {residual:.6f}; Residual−Fixed {residual - fixed:.6f}, CI [{residual_delta["ci95"][0]:.6f}, {residual_delta["ci95"][1]:.6f}]. B improves by {-pairs["residual_minus_fixed_B"]["mean"]:.6f}, while A worsens by {pairs["residual_minus_fixed_A"]["mean"]:.6f}; selecting B as a new primary method after seeing this would be invalid. Residual A replacement/zero-state do not reliably hurt performance and zero-state even has lower macro AbsRel. Query-conditioned prior or bypass remains plausible; this does not establish useful scene-specific information or a general fixed-decoder bottleneck. B-specific bypass was not separately tested. Context-feature spatial shuffle, feature-channel reversal and zero-evidence state masking are diagnostic only. Prediction RMS/hash changes, not just latent changes, are saved. Channel reversal leaves fixed typed-field rendering unchanged by design; its residual effect must not be read as a fixed-renderer bug. No state was mutated. ## Train-query versus heldout-query gap A AbsRel: 0.331528 on previously used 12–15 versus 2.213562 on 8–9. The macro gap 1.882034 is 94.59% contributed by ai003. Both ai003 cohorts are black and have zero A support, so blackness alone cannot explain their difference. Heldout versus training nearest-A view angle is 60.38° versus 20.74°, and approximate common visibility is .3305 versus .8700. ai002 heldout volume coverage is .5228 versus .8846, with supported surface neighborhood .1097 versus .2170. This supports view/support and coverage limitations; it does not isolate training memorization. ai001 has a remaining unexplained gap. Full per-frame baseline, view, depth, validity and overlap statistics are in `train_heldout_gap_diagnosis.json` and its narrative report. ## Reproduction, tests and limitations Run `commands.sh summary` from the repository root for raw-only statistics, or `commands.sh full` with a fresh output directory for downloads, training, inference, scripted audits and tests. The full mode ends with regenerated statistics; it does not recreate the manually collected official-source-validity notes or the curated environment/test/provenance report. Those original records are archived separately; new repetitions must record their own environment and execution results. Archived locks/raw reproduce all summaries via `scripts/report_static_attribution.py`. Tests: **{status["TESTS"]}**. The single historical V5 checkpoint test is skipped because that optional checkpoint is unavailable. New tests cover RGB preservation, camera/depth roundtrips, scene/query isolation, residual immutability, unchanged replacement camera, scene bootstrap and score-independent quality/volume auditing. Full log: `tests.log`. For a clean checkout containing only the published report, recompute statistics with: ```bash .venv/bin/python scripts/report_static_attribution.py --raw-only \ --output docs/experiments/EXP-3D-STATIC-CARRIER-ATTRIBUTION-V1 \ --docs /tmp/static-attribution-recomputed ``` Raw model results are in `static_results.json`/CSV; all official statistics can be regenerated without rerunning inference. `artifacts/` stores deterministic gzip copies of detailed audit/lock/training records; `artifact_manifest.json` maps them to output paths with original SHA256. Local full states, media, native geometry arrays and both residual checkpoints remain under outputs. Checkpoints/data are not duplicated into Git docs; downloading requires the published Hypersim source and original carrier artifacts described in the previous training report. `commands.sh` downloads pinned metadata and invokes bounded data preparation. Source asset identity is audited but not a forensic guarantee against supplier reuse. Seven dev scenes and sparse candidate coverage limit scope. Negative qualification is not a proof that all shared states or all visible signals are impossible. **Dynamic TTT remains closed for this carrier.** """ (root / "README.md").write_text(readme) docs.mkdir(parents=True, exist_ok=True) # Archive all textual run evidence; large media/native arrays/checkpoints remain local. artifacts = docs / "artifacts" artifacts.mkdir(exist_ok=True) manifest = [] for directory in [ "old_static", "unseen_static", "residual", "residual_30000", "rgb_audit", "geometry_audit", "unseen_geometry_audit", "unseen_data", "metadata", ]: for path in sorted((root / directory).rglob("*")): if not path.is_file() or path.suffix not in { ".json", ".jsonl", ".csv", ".md", ".log", ".py", }: continue if "raw" in path.relative_to(root / directory).parts or "prepared" in path.parts: continue rel = path.relative_to(root) target = artifacts / (str(rel) + ".gz") target.parent.mkdir(parents=True, exist_ok=True) target.write_bytes(gzip.compress(path.read_bytes(), mtime=0)) manifest.append( { "output_relative_path": str(rel), "artifact": str(target.relative_to(docs)), "sha256": sha(path), "compressed_sha256": sha(target), } ) write_json(root / "artifact_manifest.json", manifest) for path in sorted(root.iterdir()): if path.is_file() and path.suffix in { ".json", ".md", ".csv", ".sh", ".txt", ".patch", ".log", }: shutil.copy2(path, docs / path.name) shutil.copy2(root / "residual_30000/training_curve.png", docs / "residual_training_curve.png") print(terminal) if __name__ == "__main__": parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--output", type=Path, required=True) parser.add_argument("--docs", type=Path, required=True) parser.add_argument("--raw-only", action="store_true") args = parser.parse_args() if args.raw_only: if (args.output / "old_static/static_results.json").exists(): raw = { k: json.loads((args.output / f"{v}/static_results.json").read_text()) for k, v in [("old", "old_static"), ("unseen", "unseen_static")] } else: raw = json.loads((args.output / "static_results.json").read_text()) summaries = {k: summarize(v) for k, v in raw.items()} pm = summaries["unseen"]["primary"]["methods"] ps = { k: pm[k]["full_image"]["metrics"]["depth_absrel"]["per_scene"] for k in ("A", "B", "Residual_A", "Residual_B") } summaries["residual_average_AB_minus_fixed"] = paired_scene_bootstrap( { sid: (ps["Residual_A"][sid] + ps["Residual_B"][sid] - ps["A"][sid] - ps["B"][sid]) / 2 for sid in ps["A"] } ) args.docs.mkdir(parents=True, exist_ok=True) write_json(args.docs / "bootstrap_results.json", summaries) print("Regenerated paired scene statistics from raw without inference:", args.docs) else: report(args.output, args.docs)