Camera Support

pip install lerobot

pip install pyrealsense2

lerobot-find-cameras opencv

OpenCV Camera @ 0
  Backend: V4L2
  Default profile: 640x480 @ 30 FPS (MJPG)

OpenCV Camera @ /dev/video4
  Backend: V4L2
  Default profile: 1280x720 @ 30 FPS (YUYV)

lerobot-find-cameras realsense

Intel RealSense D405
  Serial: 123456789
  Firmware: 5.15.0.2
  USB: USB 3.2
  Default color profile: 640x480 @ 30 FPS (RGB8)
  Default depth profile: 640x480 @ 30 FPS (Z16)

from lerobot.cameras.opencv import OpenCVCamera, OpenCVCameraConfig
from lerobot.cameras.configs import ColorMode, Cv2Rotation

# Basic configuration (640x480 @ 30 FPS)
config = OpenCVCameraConfig(
    index_or_path=0,
    fps=30,
    width=640,
    height=480
)

# Advanced configuration with rotation and format
config = OpenCVCameraConfig(
    index_or_path="/dev/video4",
    fps=60,
    width=1280,
    height=720,
    color_mode=ColorMode.RGB,
    rotation=Cv2Rotation.ROTATE_90,
    fourcc="MJPG"  # Motion JPEG format for higher frame rates
)

camera = OpenCVCamera(config)

from lerobot.cameras.realsense import RealSenseCamera, RealSenseCameraConfig
from lerobot.cameras.configs import ColorMode, Cv2Rotation

# Basic configuration (color only)
config = RealSenseCameraConfig(
    serial_number_or_name="123456789",
    fps=30,
    width=640,
    height=480
)

# Configuration with depth sensing
config = RealSenseCameraConfig(
    serial_number_or_name="123456789",
    fps=30,
    width=1280,
    height=720,
    use_depth=True,
    color_mode=ColorMode.RGB,
    rotation=Cv2Rotation.NO_ROTATION
)

camera = RealSenseCamera(config)

from lerobot.cameras.opencv import OpenCVCamera, OpenCVCameraConfig

# Configure and connect
config = OpenCVCameraConfig(index_or_path=0, fps=30, width=640, height=480)
camera = OpenCVCamera(config)
camera.connect()

print(f"Connected: {camera.is_connected}")
print(f"Resolution: {camera.width}x{camera.height}")
print(f"FPS: {camera.fps}")

frame = camera.read()  # Blocks until frame is ready
print(f"Frame shape: {frame.shape}")  # (height, width, 3)

try:
    frame = camera.async_read(timeout_ms=200)
    print(f"Frame captured")
except TimeoutError:
    print("No frame received within timeout")

try:
    frame = camera.read_latest(max_age_ms=500)
    print(f"Frame age: <500ms")
except TimeoutError:
    print("Latest frame is too old")

from lerobot.cameras.realsense import RealSenseCamera, RealSenseCameraConfig

config = RealSenseCameraConfig(
    serial_number_or_name="123456789",
    fps=30,
    width=640,
    height=480,
    use_depth=True
)
camera = RealSenseCamera(config)
camera.connect()

# Read color and depth
color_frame = camera.read()
depth_frame = camera.read_depth()  # uint16 array, depth in millimeters

print(f"Color: {color_frame.shape}")  # (480, 640, 3)
print(f"Depth: {depth_frame.shape}")  # (480, 640)
print(f"Depth at center: {depth_frame[240, 320]} mm")

camera.disconnect()
print(f"Connected: {camera.is_connected}")  # False

from lerobot.cameras.opencv import OpenCVCamera, OpenCVCameraConfig
import numpy as np

# Configure multiple cameras
cameras = {
    "front": OpenCVCamera(OpenCVCameraConfig(index_or_path=0, fps=30, width=640, height=480)),
    "wrist": OpenCVCamera(OpenCVCameraConfig(index_or_path=2, fps=30, width=640, height=480)),
}

# Connect all
for name, cam in cameras.items():
    cam.connect()
    print(f"{name} camera connected")

# Capture synchronized frames
frames = {}
for name, cam in cameras.items():
    frames[name] = cam.read()

print(f"Captured {len(frames)} frames")

# Disconnect all
for cam in cameras.values():
    cam.disconnect()

config = OpenCVCameraConfig(
    index_or_path=0,
    fps=60,
    width=640,
    height=480,
    fourcc="MJPG"  # Enables hardware JPEG compression
)
camera = OpenCVCamera(config)
camera.connect()

# Verify actual FPS
print(f"Configured FPS: {camera.fps}")

import time

camera.connect()

try:
    frame_count = 0
    start_time = time.time()
    
    while True:
        frame = camera.read()
        frame_count += 1
        
        # Process frame here
        # cv2.imshow("Camera", frame)
        
        if time.time() - start_time > 10:  # Run for 10 seconds
            break
    
    elapsed = time.time() - start_time
    print(f"Captured {frame_count} frames in {elapsed:.1f}s")
    print(f"Actual FPS: {frame_count / elapsed:.1f}")
    
finally:
    camera.disconnect()

import cv2

# Camera configured for RGB output
config = OpenCVCameraConfig(
    index_or_path=0,
    fps=30,
    width=640,
    height=480,
    color_mode=ColorMode.RGB
)
camera = OpenCVCamera(config)
camera.connect()

frame_rgb = camera.read()  # RGB format

# Convert to other formats as needed
frame_bgr = cv2.cvtColor(frame_rgb, cv2.COLOR_RGB2BGR)
frame_gray = cv2.cvtColor(frame_rgb, cv2.COLOR_RGB2GRAY)
frame_hsv = cv2.cvtColor(frame_rgb, cv2.COLOR_RGB2HSV)

import cv2
import numpy as np

# Capture calibration images
camera.connect()
images = []
for i in range(20):
    frame = camera.read()
    images.append(frame)
    print(f"Captured calibration image {i+1}/20")
    time.sleep(1)

# Use OpenCV calibration
# This is a simplified example - see OpenCV docs for complete calibration
chessboard_size = (9, 6)
objpoints = []  # 3D points in real world space
imgpoints = []  # 2D points in image plane

for img in images:
    gray = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)
    ret, corners = cv2.findChessboardCorners(gray, chessboard_size)
    if ret:
        imgpoints.append(corners)
        # Add corresponding 3D points...

# Camera intrinsics will be computed here
# ret, mtx, dist, rvecs, tvecs = cv2.calibrateCamera(...)

sudo usermod -a -G video $USER
# Log out and log back in

# List video devices
ls /dev/video*

# Check supported formats with v4l2
v4l2-ctl --device=/dev/video0 --list-formats-ext

# Find camera serial
udevadm info --name=/dev/video0 | grep ID_SERIAL

# Create rule in /etc/udev/rules.d/99-cameras.rules
SUBSYSTEM=="video4linux", ATTRS{serial}=="12345", SYMLINK+="camera-front"

# Reload rules
sudo udevadm control --reload-rules
sudo udevadm trigger