GitHub - ori-drs/drs-vpt: DRS-VPT (ECCV 2026): localize a camera image directly in a LiDAR scan or point cloud map, and calibrate camera to LiDAR without targets.

python scripts/demo.py                                 # KITTI sample with its K: prints the pose and its error
python scripts/demo.py --predict-intrinsics            # the same without K: the intrinsics are predicted
python scripts/demo.py --sample 7scenes --viz3d        # indoor sample, with a 3-D view
python scripts/demo.py --sample iphone_stream          # many frames against one map

from drs_vpt.hub import load_model, load_sample
from drs_vpt.inference import calibrate, read_image

model = load_model(device="cuda")

sample = load_sample("kitti")
frame = sample["frames"][0]
image = read_image(frame["img_path"])   # (H, W, 3) uint8 RGB
K = frame["K"]                          # 3x3 intrinsics of the image
cloud = sample["cloud"]                 # (N, 3) float32 point cloud in metres
grid_size = sample["grid_size"]         # voxel size in metres: 0.02 outdoors, 0.005-0.01 indoors

out = calibrate(model, [(image, K, cloud)], grid_size)
print(out["T_cam_lidar"])               # 4x4 cloud-to-camera: x_cam = R @ x_cloud + t

out = calibrate(model, [(image, None, cloud)], grid_size)   # without K: the intrinsics are predicted
print(out["T_cam_lidar"])

python scripts/eval.py --dataset kitti    --data /data/kitti_vp2p_eval_data
python scripts/eval.py --dataset nuscenes --data /data/nuscenes_test_processed/test
python scripts/eval.py --dataset kitti    --data /data/kitti_vp2p_eval_data --limit 20   # quick check

from drs_vpt.eval import evaluate, kitti_samples, summarize
from drs_vpt.hub import load_model

model = load_model(device="cuda")
results = evaluate(model, kitti_samples("/data/kitti_vp2p_eval_data"), out_dir="eval_output/kitti")
summarize(results)   # {"n", "recall", "rre_mean_deg", "rre_median_deg", "rte_mean_m", "rte_median_m"}

python scripts/download_kitti_eval.py --out /data/kitti_vp2p_eval_data

mkdir -p /raw/nuscenes/test
for f in ~/Downloads/nuscenes/v1.0-test_{meta,blobs}.tgz; do tar -xzf "$f" -C /raw/nuscenes/test; done
conda create -n nuscenes python=3.8 -y
conda run -n nuscenes pip install nuscenes-devkit opencv-python
conda run -n nuscenes python data_processing/nuscenes_preprocess.py --dataroot /raw/nuscenes/test \
    --version v1.0-test --frame_skip 2 --out /data/nuscenes_test_processed/test

from drs_vpt.hub import load_model, load_sample
from drs_vpt.inference import encode_map, localize, read_image

model = load_model(device="cuda")
stream = load_sample("iphone_stream")   # 37 frames in one map
map_encoding = encode_map(model, stream["cloud"], stream["grid_size"])
for frame in stream["frames"]:
    out = localize(model, read_image(frame["img_path"]), map_encoding, frame["K"])   # K=None predicts it
    print(out["T_w2c"])                 # 4x4 world-to-camera; out["K"]: the K used, at the model's resolution

import numpy as np
from drs_vpt.hub import load_model
from drs_vpt.inference import calibrate, read_image

model = load_model(device="cuda")
kitti = "/data/kitti_vp2p_eval_data"
sequence = f"{kitti}/sequences/09"

# Intrinsics of the left colour camera, from KITTI's calibration file
calib = {}
for line in open(f"{kitti}/calib/09/calib.txt"):
    key, values = line.split(":", 1)
    calib[key] = np.array(values.split(), dtype=np.float64)
K = calib["P2"].reshape(3, 4)[:, :3]

# Six synced image / LiDAR scan pairs
frames = []
for i in range(6):
    image = read_image(f"{sequence}/image_2/{i:06d}.png")
    scan = np.fromfile(f"{sequence}/velodyne/{i:06d}.bin", dtype=np.float32).reshape(-1, 4)   # x, y, z, intensity
    frames.append((image, K, scan[:, :3]))

out = calibrate(model, frames)
print(out["T_cam_lidar"])   # 4x4 LiDAR-to-camera: x_cam = R @ x_lidar + t

pip install -e ".[train]"
bash bash_scripts/train/examples/drsvpt_kitti_4v_15ipg.sh 1    # number of GPUs

@inproceedings{fu2026drsvpt,
  title     = {{DRS}-{VPT}: Directly Relocalizing in a Scan with Vision Point Transformers},
  author    = {Fu, Lanke Frank Tarimo and Fallon, Maurice},
  booktitle = {Proceedings of the European Conference on Computer Vision (ECCV)},
  year      = {2026}
}