tests/test_rgbd_nonlearned_fusion.py

"""Non-learned RGB-D fusion: a measured plane is rendered back at its measured depth."""

import torch

from mcss.dynamic.types import OnlineObservation
from mcss.geometry import transform_cameras
from mcss.measurements import FixedMeasurementRenderer
from mcss.mechanism_pilot.rgbd_evidence_carrier import RGBDContext
from mcss.mechanism_pilot.rgbd_nonlearned_fusion import fused_state, voxel_centers
from mcss.types import Cameras


def camera(x, height=32, width=40, focal=30.0):
    cx, cy = (width - 1) / 2, (height - 1) / 2
    pose = torch.eye(4)
    pose[0, 3] = x
    intrinsics = torch.tensor([[focal, 0, cx], [0, focal, cy], [0, 0, 1.0]])
    return Cameras(intrinsics, pose, (height, width))


def ray_distance(cam, z):
    height, width = cam.image_size
    v, u = torch.meshgrid(
        torch.arange(height, dtype=torch.float32),
        torch.arange(width, dtype=torch.float32),
        indexing="ij",
    )
    directions = torch.stack((u, v, torch.ones_like(u)), -1) @ torch.linalg.inv(cam.intrinsics).T
    return z * directions.norm(dim=-1)


def test_voxel_centres_follow_the_renderer_layout():
    bounds = torch.tensor([[0.0, 0.0, 0.0], [16.0, 32.0, 48.0]])
    points = voxel_centers(bounds).reshape(16, 16, 16, 3)
    assert torch.allclose(points[0, 0, 0], torch.tensor([0.5, 1.0, 1.5]))
    assert torch.allclose(points[0, 0, 1], torch.tensor([1.5, 1.0, 1.5]))  # w -> x
    assert torch.allclose(points[0, 1, 0], torch.tensor([0.5, 3.0, 1.5]))  # h -> y
    assert torch.allclose(points[1, 0, 0], torch.tensor([0.5, 1.0, 4.5]))  # d -> z


def render_plane(z):
    cams = [camera(x) for x in (0.0, -0.3, 0.3)]
    obs = [OnlineObservation("s", i, torch.full((3, 32, 40), 0.4), c) for i, c in enumerate(cams)]
    depths = torch.stack([ray_distance(c, torch.full((32, 40), z)) for c in cams])
    bounds = torch.tensor([[-2.0, -2.0, 0.2], [2.0, 2.0, 4.2]])
    params = {"k": 0.5, "a": 64.0, "b": 4.0, "c": -4.0}
    state, anchor = fused_state(RGBDContext(obs, depths), bounds, params)
    query = transform_cameras(
        Cameras(cams[0].intrinsics[None, None], cams[0].c2w[None, None], (32, 40)),
        torch.linalg.inv(anchor),
    )
    pred = FixedMeasurementRenderer(n_samples=64)(state, query, ("depth", "visibility", "rgb"))
    truth = ray_distance(cams[0], torch.full((32, 40), z))[12:20, 16:24]
    return pred, truth


def test_fused_plane_renders_near_its_measured_depth_and_moves_with_it():
    near, truth_near = render_plane(2.0)
    far, truth_far = render_plane(3.0)
    depth_near = near["depth"][0, 0, 0][12:20, 16:24]
    depth_far = far["depth"][0, 0, 0][12:20, 16:24]
    # A sharp, opaque fused surface: within about one 0.25 m voxel of the measured depth.
    assert (depth_near - truth_near).abs().median() < 0.25
    assert (depth_far - truth_far).abs().median() < 0.25
    shift = (depth_far - depth_near).median()
    assert 0.8 < shift < 1.2
    assert near["visibility"][0, 0, 0][12:20, 16:24].min() > 0.97
    assert (near["rgb"][0, 0][:, 12:20, 16:24] - 0.4).abs().max() < 0.05