tests/test_observability_supervision_statistics_audit.py

"""Synthetic-only tests of seven-artifact byte-exact reconstruction auditing."""

import importlib.util
import json
from pathlib import Path

import pytest


def audit_module():
    source = (
        Path(__file__).resolve().parents[1]
        / "scripts/audit_observability_supervision_statistics.py"
    )
    spec = importlib.util.spec_from_file_location("observability_statistics_audit", source)
    module = importlib.util.module_from_spec(spec)
    spec.loader.exec_module(module)
    return module


def fixture(root, names):
    root.mkdir(parents=True, exist_ok=True)
    for name in names:
        (root / name).write_text(json.dumps({"x": [0.25, 0.5], "flag": True}, sort_keys=True))


@pytest.mark.parametrize("reference_enabled", [False, True])
def test_reproduction_compares_original_and_preserves_reference_routing(
    tmp_path, reference_enabled
):
    module = audit_module()
    root = tmp_path / "original"
    fixture(root, module.SCIENTIFIC_FILES)
    reference = tmp_path / "published" if reference_enabled else None
    if reference is not None:
        reference.mkdir()
    calls = []

    def runner(source, output):
        calls.append((source, output))
        output.mkdir()
        for name in module.SCIENTIFIC_FILES:
            (output / name).write_bytes((root / name).read_bytes())
        (output / "statistics_reproduction.json").write_text('{"raw_only":true}')

    for _ in range(2):
        report = module.run_audit(root, reference, runner=runner)
        assert report["status"] == "PASS"
        assert report["compared_files"] == 7 and report["max_abs_numeric_error"] == 0
        assert report["input_provenance"] == {"raw_only": True}
    assert calls[0][1] != calls[1][1]
    assert all(call[0] == (reference or root).resolve() for call in calls)
    filename = (
        "published_statistics_reproduction_audit.json"
        if reference_enabled
        else "statistics_reproduction_audit.json"
    )
    assert json.loads((root / "audit" / filename).read_text())["status"] == "PASS"


@pytest.mark.parametrize("mutation,expected_error", [("numeric", 0.5), ("whitespace", 0)])
def test_numerical_and_byte_only_mismatch_both_fail(tmp_path, mutation, expected_error):
    module = audit_module()
    fixture(tmp_path, module.SCIENTIFIC_FILES)

    def runner(source, output):
        output.mkdir()
        for name in module.SCIENTIFIC_FILES:
            payload = (source / name).read_text()
            if name == "cost_analysis.json":
                payload = payload.replace("0.5", "1.0") if mutation == "numeric" else payload + "\n"
            (output / name).write_text(payload)
        (output / "statistics_reproduction.json").write_text('{"raw_only":true}')

    report = module.run_audit(tmp_path, runner=runner)
    assert report["status"] == "FAIL"
    assert report["byte_mismatch_count"] == 1
    assert report["max_abs_numeric_error"] == expected_error


def test_incomplete_analysis_does_not_invoke_analyzer(tmp_path):
    module = audit_module()

    def forbidden(*args, **kwargs):
        raise AssertionError("Analyzer cannot run before all original scientific artifacts exist")

    report = module.run_audit(tmp_path, runner=forbidden)
    assert report["status"] == "FAIL" and report["error"]["type"] == "FileNotFoundError"