🧠 Image Tracking Fundamentals
🎯 What is Image Tracking?
Image tracking (also called 2D image tracking or image target tracking) detects and tracks known 2D images in the camera feed — posters, cards, product packaging, book covers, magazine pages. When an image is recognized, AR content can be anchored to it, appearing as if printed on the surface. It's the most reliable form of markerless AR.
How Image Tracking Works
1. Reference images are provided to the AR system. 2. Features are extracted from each reference image (corners, edges, textures). 3. Live camera frames are scanned for matching features. 4. Pose estimation determines the image's 3D position and orientation. 5. AR content is anchored to the tracked image. 6. Continuous tracking updates the pose as the camera or image moves.
📌 2D Images📌 Feature Matching📌 Pose Estimation📌 Anchoring
# Image tracking pipeline (conceptual)
class ImageTrackingAR:
def __init__(self, reference_images):
self.reference_db = ReferenceImageDatabase()
for img in reference_images:
features = extract_features(img)
self.reference_db.add(img.id, features)
self.tracker = ImageTracker()
def process_frame(self, camera_frame, camera_pose):
# 1. Extract features from live frame
live_features = extract_features(camera_frame)
# 2. Match against reference database
matches = self.reference_db.match(live_features)
# 3. For each matched reference image, estimate pose
tracked_images = []
for match in matches:
pose = estimate_pose(
match.reference_features,
match.live_features,
camera_matrix
)
if pose.is_valid():
tracked_images.append(TrackedImage(
reference_id=match.reference_id,
pose=pose,
tracking_quality=match.quality
))
# 4. Update tracking (smooth over time)
self.tracker.update(tracked_images)
return self.tracker.get_tracked_images()📌 Image vs Marker vs Markerless
🎯 Three approaches to AR tracking
Marker-based: Uses fiducial markers (QR, ArUco) — simple, robust, but requires printed markers. Image tracking: Uses natural images as targets — no special markers, just a photo/poster. Markerless: Uses environment features (planes, objects) — no target at all.
Comparison
Marker: Fast, robust, but requires printed markers. Image: Uses existing images (posters, packaging), more natural. Markerless: No target, works anywhere, but less reliable. Best for: Marker = industrial, Image = marketing/packaging, Markerless = navigation/placement.
📌 Marker🖼️ Image🌍 Markerless⚖️ Trade-offs
# Comparison of tracking approaches
class TrackingComparison:
@staticmethod
def marker_based():
"""Fiducial marker tracking (ArUco, QR)."""
# Pros: Fast, robust, low CPU
# Cons: Requires printed markers, not aesthetic
import cv2
aruco_dict = cv2.aruco.Dictionary_get(cv2.aruco.DICT_6X6_250)
corners, ids, _ = cv2.aruco.detectMarkers(frame, aruco_dict)
return corners, ids
@staticmethod
def image_tracking(reference_images):
"""Natural image tracking."""
# Pros: Uses existing images, natural, no markers
# Cons: Requires good features, slower, sensitive to lighting
from ar_image_tracker import ImageTracker
tracker = ImageTracker()
for img in reference_images:
tracker.add_reference(img)
return tracker.track(frame)
@staticmethod
def markerless_tracking():
"""Environment feature tracking (SLAM)."""
# Pros: No target needed, works anywhere
# Cons: Less reliable, needs good lighting/features
from slam import SLAMSystem
slam = SLAMSystem()
return slam.track(frame)
# When to use what:
# - Marker: Industrial AR, robot navigation, calibration
# - Image: Marketing, packaging, books, posters, business cards
# - Markerless: Furniture placement, navigation, gaming🔍 Feature Detection for Images
🎯 Finding distinctive points
Image tracking relies on detecting distinctive features in reference images: corners, edges, blobs, and textured regions. The more features an image has, the better it tracks. Detectors: ORB, FAST, SIFT, AKAZE, BRISK.
Feature Detectors
ORB: Fast, rotation-invariant, patent-free — best for AR. FAST: Very fast corner detector. SIFT: Scale-invariant, accurate, but slow and patented. AKAZE: Good balance, nonlinear scale space. BRISK: Binary, fast. Good features: High contrast, corners, textured areas.
⚡ ORB🔍 FAST📐 SIFT🌊 AKAZE
# Feature detection with OpenCV
import cv2
import numpy as np
def extract_features(image, method='ORB'):
"""Extract features from an image."""
if method == 'ORB':
detector = cv2.ORB_create(nfeatures=1000)
elif method == 'SIFT':
detector = cv2.SIFT_create(nfeatures=1000)
elif method == 'AKAZE':
detector = cv2.AKAZE_create()
elif method == 'BRISK':
detector = cv2.BRISK_create()
elif method == 'FAST':
detector = cv2.FastFeatureDetector_create(threshold=25)
else:
raise ValueError(f"Unknown method: {method}")
# Detect and compute descriptors
if method == 'FAST':
keypoints = detector.detect(image, None)
# FAST doesn't compute descriptors; use ORB for descriptors
orb = cv2.ORB_create()
keypoints, descriptors = orb.compute(image, keypoints)
else:
keypoints, descriptors = detector.detectAndCompute(image, None)
return keypoints, descriptors
# Evaluate image trackability
def evaluate_image_quality(image):
"""Score an image for tracking suitability (0-100)."""
gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
# 1. Feature count
keypoints, descriptors = extract_features(gray, 'ORB')
feature_score = min(len(keypoints) / 500.0, 1.0) * 40
# 2. Feature distribution (should be spread out)
if len(keypoints) > 0:
points = np.array([kp.pt for kp in keypoints])
h, w = gray.shape
hist, _, _ = np.histogram2d(
points[:, 0], points[:, 1],
bins=[8, 8], range=[[0, w], [0, h]]
)
# Entropy of distribution
hist = hist / hist.sum()
entropy = -np.sum(hist * np.log2(hist + 1e-10))
distribution_score = min(entropy / 6.0, 1.0) * 30
else:
distribution_score = 0
# 3. Contrast
contrast = gray.std()
contrast_score = min(contrast / 64.0, 1.0) * 30
total = feature_score + distribution_score + contrast_score
return {
'total': total,
'features': len(keypoints),
'feature_score': feature_score,
'distribution_score': distribution_score,
'contrast_score': contrast_score
}
# Usage
image = cv2.imread('poster.jpg')
score = evaluate_image_quality(image)
print(f"Image score: {score['total']:.1f}/100")
print(f"Features: {score['features']}")
if score['total'] < 50:
print("Warning: Poor tracking quality. Use a more textured image.")🧬 Descriptors & Matching
🎯 Encoding and matching features
Descriptors encode the local appearance around each feature point. Matching compares descriptors between the live frame and reference images. Binary descriptors (ORB, BRISK) use Hamming distance; float descriptors (SIFT, SURF) use L2 distance.
Matching Strategies
Brute-force: Compare all descriptors. FLANN: Approximate nearest neighbor. Ratio test: Lowe's ratio (0.7-0.8). Cross-check: Mutual best matches. RANSAC: Geometric verification (homography). Homography: Maps reference image to camera view.
⚡ Brute-force🌲 FLANN📊 Ratio Test🎯 RANSAC
# Feature matching for image tracking
import cv2
import numpy as np
def match_features(reference_desc, live_desc, method='ORB'):
"""Match descriptors between reference and live frame."""
if method == 'ORB' or method == 'BRISK' or method == 'AKAZE':
# Binary descriptors: Hamming distance
matcher = cv2.BFMatcher(cv2.NORM_HAMMING)
else:
# Float descriptors: L2 distance
matcher = cv2.BFMatcher(cv2.NORM_L2)
# KNN matching (k=2 for ratio test)
matches = matcher.knnMatch(reference_desc, live_desc, k=2)
# Lowe's ratio test
good_matches = []
for match_pair in matches:
if len(match_pair) == 2:
m, n = match_pair
if m.distance < 0.75 * n.distance:
good_matches.append(m)
return good_matches
def verify_with_homography(ref_keypoints, live_keypoints, matches, min_inliers=15):
"""Verify matches with homography (RANSAC)."""
if len(matches) < 4:
return None, 0
# Extract matched points
ref_pts = np.float32([ref_keypoints[m.queryIdx].pt for m in matches])
live_pts = np.float32([live_keypoints[m.trainIdx].pt for m in matches])
# Compute homography with RANSAC
H, mask = cv2.findHomography(ref_pts, live_pts, cv2.RANSAC, 5.0)
if H is None:
return None, 0
inliers = np.sum(mask)
if inliers < min_inliers:
return None, inliers
return H, inliers
# Complete image tracking pipeline
def track_image(reference_image, live_frame):
# 1. Extract features
ref_kp, ref_desc = extract_features(reference_image, 'ORB')
live_kp, live_desc = extract_features(live_frame, 'ORB')
if ref_desc is None or live_desc is None:
return None
# 2. Match features
matches = match_features(ref_desc, live_desc, 'ORB')
if len(matches) < 10:
return None
# 3. Verify with homography
H, inliers = verify_with_homography(ref_kp, live_kp, matches)
if H is None:
return None
# 4. Compute pose from homography
pose = homography_to_pose(H, camera_matrix)
return {
'homography': H,
'pose': pose,
'inliers': inliers,
'matches': len(matches)
}
def homography_to_pose(H, K):
"""Decompose homography into rotation and translation."""
# Normalize by K
K_inv = np.linalg.inv(K)
H_normalized = K_inv @ H
# Extract columns
h1 = H_normalized[:, 0]
h2 = H_normalized[:, 1]
h3 = H_normalized[:, 2]
# Compute scale
scale = 1.0 / np.linalg.norm(h1)
# Rotation columns
r1 = h1 * scale
r2 = h2 * scale
r3 = np.cross(r1, r2)
# Translation
t = h3 * scale
# Orthonormalize rotation (SVD)
R = np.column_stack((r1, r2, r3))
U, _, Vt = np.linalg.svd(R)
R = U @ Vt
# Build 4x4 pose matrix
pose = np.eye(4)
pose[:3, :3] = R
pose[:3, 3] = t
return pose📐 Pose Estimation
🎯 Where is the image in 3D?
Pose estimation determines the image's 3D position and orientation relative to the camera. For planar images, this is a homography (2D-to-2D mapping), which can be decomposed into a 3D pose (rotation + translation).
Pose from Homography
Homography: 3x3 matrix mapping image plane to camera plane. Decomposition: Extract R and t from H using camera intrinsics. PnP: Alternative — use 3D points of the image corners. Ambiguity: Two solutions for pose from homography; choose the one with positive depth. Refinement: Levenberg-Marquardt optimization.
📐 Homography🔄 Decomposition📏 PnP✨ Refinement
# Pose estimation from homography with OpenCV
import cv2
import numpy as np
def estimate_pose_from_image(reference_image, live_frame, camera_matrix):
"""Estimate 3D pose of a tracked image."""
# 1. Feature matching
ref_kp, ref_desc = extract_features(reference_image, 'ORB')
live_kp, live_desc = extract_features(live_frame, 'ORB')
matches = match_features(ref_desc, live_desc, 'ORB')
if len(matches) < 10:
return None
# 2. Homography
ref_pts = np.float32([ref_kp[m.queryIdx].pt for m in matches])
live_pts = np.float32([live_kp[m.trainIdx].pt for m in matches])
H, mask = cv2.findHomography(ref_pts, live_pts, cv2.RANSAC, 5.0)
if H is None:
return None
# 3. Decompose homography
num_solutions, rotations, translations, normals = cv2.decomposeHomographyMat(
H, camera_matrix
)
# 4. Choose best solution (positive depth, in front of camera)
best_pose = None
best_score = -1
for i in range(num_solutions):
R = rotations[i]
t = translations[i]
# Check if image is in front of camera
if t[2] > 0:
# Compute reprojection error
score = compute_reprojection_error(
ref_pts, live_pts, R, t, camera_matrix
)
if score > best_score:
best_score = score
best_pose = build_pose_matrix(R, t)
return best_pose
def build_pose_matrix(R, t):
"""Build 4x4 pose matrix from rotation and translation."""
pose = np.eye(4)
pose[:3, :3] = R
pose[:3, 3] = t.flatten()
return pose
def compute_reprojection_error(ref_pts, live_pts, R, t, K):
"""Compute reprojection error for pose refinement."""
# Project reference points to image
rvec, _ = cv2.Rodrigues(R)
projected, _ = cv2.projectPoints(
np.hstack((ref_pts, np.zeros((len(ref_pts), 1)))),
rvec, t, K, None
)
# Compute error
error = np.mean(np.linalg.norm(projected - live_pts.reshape(-1, 1, 2), axis=2))
return -error # negative for maximization
# Alternative: PnP with known image dimensions
def estimate_pose_pnp(reference_image, live_frame, camera_matrix, image_width_m, image_height_m):
"""Estimate pose using PnP with known image dimensions."""
# 3D corners of the image (in image coordinate frame)
half_w = image_width_m / 2
half_h = image_height_m / 2
object_points = np.array([
[-half_w, -half_h, 0],
[ half_w, -half_h, 0],
[ half_w, half_h, 0],
[-half_w, half_h, 0]
], dtype=np.float32)
# 2D corners in live frame (from homography)
# ... detect corners ...
# Solve PnP
success, rvec, tvec = cv2.solvePnP(
object_points, image_points, camera_matrix, None
)
if success:
R, _ = cv2.Rodrigues(rvec)
return build_pose_matrix(R, tvec)
return None🎯 Continuous Tracking
🎯 Keeping track over time
Once an image is detected, continuous tracking maintains its pose across frames. This avoids re-detection every frame (expensive) and provides smooth, stable tracking. Methods: optical flow, template matching, feature re-matching.
Tracking Strategies
Optical flow: Track features from frame to frame (KLT). Template matching: Match previous image patch. Feature re-matching: Re-match features to reference. Hybrid: Optical flow + periodic re-detection. Quality metric: Inlier count, reprojection error. Lost detection: Re-detect when quality drops.
🌊 Optical Flow🖼️ Template🔄 Re-match📊 Quality
# Continuous image tracking with optical flow
import cv2
import numpy as np
class ImageTracker:
def __init__(self, reference_image, camera_matrix):
self.reference_image = reference_image
self.camera_matrix = camera_matrix
# Extract reference features
self.ref_kp, self.ref_desc = extract_features(reference_image, 'ORB')
# Tracking state
self.is_tracking = False
self.current_pose = None
self.prev_gray = None
self.prev_pts = None
def detect(self, frame):
"""Detect the image in the frame."""
gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
live_kp, live_desc = extract_features(gray, 'ORB')
if live_desc is None:
return False
# Match features
matches = match_features(self.ref_desc, live_desc, 'ORB')
if len(matches) < 15:
return False
# Verify with homography
ref_pts = np.float32([self.ref_kp[m.queryIdx].pt for m in matches])
live_pts = np.float32([live_kp[m.trainIdx].pt for m in matches])
H, mask = cv2.findHomography(ref_pts, live_pts, cv2.RANSAC, 5.0)
if H is None or np.sum(mask) < 15:
return False
# Compute pose
self.current_pose = homography_to_pose(H, self.camera_matrix)
self.is_tracking = True
# Initialize optical flow
self.prev_gray = gray
self.prev_pts = live_pts[mask.ravel() == 1].reshape(-1, 1, 2)
return True
def track(self, frame):
"""Track the image using optical flow."""
if not self.is_tracking:
return self.detect(frame)
gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
# Optical flow
next_pts, status, _ = cv2.calcOpticalFlowPyrLK(
self.prev_gray, gray,
self.prev_pts, None,
winSize=(21, 21),
maxLevel=3
)
# Filter good points
good_new = next_pts[status == 1]
good_old = self.prev_pts[status == 1]
if len(good_new) < 10:
# Tracking lost
self.is_tracking = False
return self.detect(frame)
# Estimate new pose from tracked points
# (using homography from previous frame to current)
H, mask = cv2.findHomography(good_old, good_new, cv2.RANSAC, 5.0)
if H is not None:
self.current_pose = homography_to_pose(H, self.camera_matrix)
self.prev_pts = good_new.reshape(-1, 1, 2)
self.prev_gray = gray
return True
self.is_tracking = False
return False
# Tracking quality assessment
def assess_tracking_quality(matches, inliers, reprojection_error):
"""Assess tracking quality (0-1)."""
if matches < 10:
return 0.0
inlier_ratio = inliers / matches
error_score = max(0, 1.0 - reprojection_error / 5.0)
quality = 0.4 * min(matches / 50.0, 1.0) + \
0.3 * inlier_ratio + \
0.3 * error_score
return min(quality, 1.0)
# State machine for tracking
class TrackingState:
DETECTING = "detecting" # Looking for the image
TRACKING = "tracking" # Actively tracking
LOST = "lost" # Tracking lost, try to re-detect
PAUSED = "paused" # Temporarily paused
class RobustImageTracker:
def __init__(self, reference_image):
self.tracker = ImageTracker(reference_image)
self.state = TrackingState.DETECTING
self.frames_since_detection = 0
self.quality_history = []
def update(self, frame):
if self.state == TrackingState.DETECTING:
if self.tracker.detect(frame):
self.state = TrackingState.TRACKING
self.frames_since_detection = 0
elif self.state == TrackingState.TRACKING:
if self.tracker.track(frame):
self.frames_since_detection = 0
else:
self.frames_since_detection += 1
if self.frames_since_detection > 5:
self.state = TrackingState.LOST
elif self.state == TrackingState.LOST:
if self.tracker.detect(frame):
self.state = TrackingState.TRACKING
self.frames_since_detection = 0
return self.state == TrackingState.TRACKING, self.tracker.current_pose📚 Reference Image Libraries
🎯 Managing multiple targets
AR apps often track multiple images — a catalog of products, a set of posters, a deck of cards. The reference library stores features for each image and efficiently matches live features against all targets.
Library Management
Database: Store features per image. Indexing: Use a vocabulary tree or FLANN for fast matching. Prioritization: Track only nearby/likely images. Memory: Large libraries need compression. Dynamic loading: Load images on demand. ARKit/ARCore: Limit to ~100 images per session.
📚 Database🌲 Vocabulary Tree⚡ FLANN💾 Compression
# Reference image library with efficient matching
import cv2
import numpy as np
from collections import defaultdict
class ReferenceImageLibrary:
def __init__(self):
self.images = {}
self.features = {}
self.matcher = cv2.FlannBasedMatcher(
dict(algorithm=6, table_number=12, key_size=20, multi_probe_level=2),
dict(checks=50)
)
def add_image(self, image_id, image, physical_width=0.0):
"""Add a reference image to the library."""
gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
kp, desc = extract_features(gray, 'ORB')
if desc is None or len(kp) < 20:
raise ValueError(f"Image {image_id} has insufficient features")
self.images[image_id] = {
'image': image,
'width': physical_width,
'height': physical_width * image.shape[0] / image.shape[1]
}
self.features[image_id] = {
'keypoints': kp,
'descriptors': desc
}
def match(self, live_desc, top_k=5):
"""Match live descriptors against all reference images."""
results = []
for image_id, feat in self.features.items():
# Match against this image
matches = match_features(
feat['descriptors'], live_desc, 'ORB'
)
if len(matches) < 10:
continue
# Verify with homography
H, inliers = verify_with_homography(
feat['keypoints'],
None, # live keypoints passed separately
matches
)
if H is not None:
results.append({
'image_id': image_id,
'matches': len(matches),
'inliers': inliers,
'homography': H,
'score': inliers / len(matches)
})
# Sort by score
results.sort(key=lambda x: x['score'], reverse=True)
return results[:top_k]
def get_image(self, image_id):
return self.images.get(image_id)
def save(self, filepath):
"""Save library to disk."""
import pickle
with open(filepath, 'wb') as f:
pickle.dump({
'images': self.images,
'features': self.features
}, f)
def load(self, filepath):
"""Load library from disk."""
import pickle
with open(filepath, 'rb') as f:
data = pickle.load(f)
self.images = data['images']
self.features = data['features']
# Vocabulary tree for fast image retrieval
class VocabularyTree:
def __init__(self, branching_factor=10, depth=6):
self.branching_factor = branching_factor
self.depth = depth
self.vocabulary = None
def build(self, all_descriptors):
"""Build vocabulary tree from descriptors."""
from scipy.cluster.vq import kmeans2
# Hierarchical k-means
current = all_descriptors
for level in range(self.depth):
centroids, labels = kmeans2(
current,
self.branching_factor ** (level + 1)
)
current = centroids
self.vocabulary = current
def encode(self, descriptors):
"""Encode descriptors as a path through the tree."""
# Find nearest word for each descriptor
from scipy.spatial import KDTree
tree = KDTree(self.vocabulary)
_, indices = tree.query(descriptors)
return indices⭐ Image Quality & Rating
🎯 What makes a good tracking image?
Not all images track well. Good tracking images have: high contrast, rich texture, many corners, uniform feature distribution, and no repetitive patterns. ARKit/ARCore provide a quality rating.
Image Quality Criteria
Good: Textured, high contrast, many features, non-repetitive. Bad: Plain colors, repetitive patterns, blurry, low contrast. Rating: ARKit/ARCore rate images on a scale. Improvement: Add texture, increase contrast. Physical size: Must be specified for correct scale.
✅ High Contrast🧶 Rich Texture📊 Feature Distribution🚫 No Repetition
# Image quality rating (similar to ARKit/ARCore)
import cv2
import numpy as np
def rate_image_quality(image):
"""
Rate an image for AR tracking quality.
Returns a score from 0 (poor) to 100 (excellent).
"""
gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
h, w = gray.shape
# 1. Feature count and distribution
kp, desc = extract_features(gray, 'ORB')
feature_count = len(kp)
if feature_count < 30:
return 0, "Too few features"
# 2. Feature distribution (grid entropy)
points = np.array([kp_.pt for kp_ in kp])
grid_size = 8
hist, _, _ = np.histogram2d(
points[:, 0], points[:, 1],
bins=[grid_size, grid_size],
range=[[0, w], [0, h]]
)
hist = hist / hist.sum()
entropy = -np.sum(hist * np.log2(hist + 1e-10))
max_entropy = np.log2(grid_size * grid_size)
distribution_score = (entropy / max_entropy) * 100
# 3. Contrast
contrast = gray.std()
contrast_score = min(contrast / 80.0, 1.0) * 100
# 4. Edge density
edges = cv2.Canny(gray, 50, 150)
edge_density = np.sum(edges > 0) / (h * w)
edge_score = min(edge_density / 0.1, 1.0) * 100
# 5. Repetitiveness (autocorrelation)
f = np.fft.fft2(gray)
fshift = np.fft.fftshift(f)
magnitude = np.abs(fshift)
# Check for peaks (repetitive patterns)
peak_ratio = np.max(magnitude) / np.mean(magnitude)
repetitiveness_score = max(0, 100 - (peak_ratio - 10) * 5)
# Combined score
total = (
0.25 * min(feature_count / 500.0, 1.0) * 100 +
0.20 * distribution_score +
0.20 * contrast_score +
0.20 * edge_score +
0.15 * repetitiveness_score
)
# Rating
if total >= 80:
rating = "Excellent"
elif total >= 60:
rating = "Good"
elif total >= 40:
rating = "Fair"
else:
rating = "Poor"
return total, rating
# Test on sample images
def test_image_quality(image_path):
image = cv2.imread(image_path)
score, rating = rate_image_quality(image)
print(f"Image: {image_path}")
print(f"Score: {score:.1f}/100 ({rating})")
return score, rating
# ARKit image quality (Swift)
"""
let referenceImage = ARReferenceImage(
cgImage,
orientation: .up,
physicalWidth: 0.2 // 20cm
)
// ARKit automatically rates the image
// Use ARImageTrackingConfiguration to check
"""
# ARCore image quality (Java)
"""
AugmentedImageDatabase database = new AugmentedImageDatabase(session);
database.addImage("poster", bitmap, 0.2f); // 20cm width
// ARCore rates images internally
// Check with getTrackingState()
"""🖼️ Multiple Image Tracking
🎯 Tracking several images at once
AR apps often need to track multiple images simultaneously — a board game with multiple cards, a museum with several paintings, a catalog with many products. The system must detect, track, and manage multiple targets efficiently.
Multi-Image Strategies
Parallel tracking: Track all images simultaneously. Priority: Focus on images in view. Slot management: Limit max tracked images (ARKit: 4, ARCore: 20+). Shared anchors: Multiple images can share content. Dynamic loading: Load/unload images based on location.
🔄 Parallel🎯 Priority📊 Slots⚓ Shared
// ARKit multiple image tracking (Swift)
import ARKit
class MultiImageTracking: NSObject, ARSessionDelegate {
var session: ARSession!
var referenceImages: Set!
func setup() {
// Load reference images from asset catalog
guard let images = ARReferenceImage.referenceImages(
inGroupNamed: "AR Resources",
bundle: nil
) else { return }
referenceImages = images
// Configure session
let config = ARWorldTrackingConfiguration()
config.detectionImages = referenceImages
config.maximumNumberOfTrackedImages = 4 // ARKit limit
session.run(config)
}
func session(_ session: ARSession, didAdd anchors: [ARAnchor]) {
for anchor in anchors {
if let imageAnchor = anchor as? ARImageAnchor {
let name = imageAnchor.referenceImage.name ?? "unknown"
print("Detected image: \(name)")
// Add AR content specific to this image
addContent(for: name, at: imageAnchor)
}
}
}
func session(_ session: ARSession, didUpdate anchors: [ARAnchor]) {
for anchor in anchors {
if let imageAnchor = anchor as? ARImageAnchor {
// Update tracking
let isTracked = imageAnchor.isTracked
let transform = imageAnchor.transform
updateContent(for: imageAnchor, isTracked: isTracked, transform: transform)
}
}
}
func session(_ session: ARSession, didRemove anchors: [ARAnchor]) {
for anchor in anchors {
if let imageAnchor = anchor as? ARImageAnchor {
removeContent(for: imageAnchor)
}
}
}
}
// ARCore multiple image tracking (Kotlin)
class MultiImageTracking {
private lateinit var session: Session
private lateinit var imageDatabase: AugmentedImageDatabase
fun setup() {
// Create image database
imageDatabase = AugmentedImageDatabase(session)
// Add images
val bitmap1 = loadBitmap("poster1.jpg")
imageDatabase.addImage("poster1", bitmap1, 0.3f) // 30cm
val bitmap2 = loadBitmap("poster2.jpg")
imageDatabase.addImage("poster2", bitmap2, 0.2f) // 20cm
// Configure session
val config = Config(session)
config.augmentedImageDatabase = imageDatabase
config.focusMode = Config.FocusMode.AUTO
session.configure(config)
}
fun onUpdate(frame: Frame) {
// Get all tracked images
val images = frame.getUpdatedTrackables(AugmentedImage::class.java)
for (image in images) {
when (image.trackingState) {
TrackingState.TRACKING -> {
val name = image.name
val pose = image.centerPose
val extentX = image.extentX
val extentZ = image.extentZ
// Update AR content
updateContent(name, pose, extentX, extentZ)
}
TrackingState.PAUSED -> {
// Tracking paused
}
TrackingState.STOPPED -> {
// Remove content
}
}
}
}
}
// Three.js multiple image tracking (WebXR)
/*
const imageTracker = new ImageTracker();
imageTracker.addImage('poster1', 'poster1.jpg', 0.3);
imageTracker.addImage('poster2', 'poster2.jpg', 0.2);
function onXRFrame(time, frame) {
const results = frame.getImageTrackingResults();
for (const result of results) {
const imageIndex = result.index;
const pose = frame.getPose(result.imageSpace, refSpace);
const state = result.trackingState;
if (state === 'tracked') {
updateContent(imageIndex, pose);
}
}
}
*/ 📱 ARKit & ARCore Image Tracking
🎯 Platform-native image tracking
ARKit (iOS) and ARCore (Android) provide built-in image tracking APIs. Both support reference image libraries, physical size specification, and tracking quality assessment.
Platform Differences
ARKit: Asset catalog for reference images, max 4 tracked images, excellent tracking. ARCore: Runtime image database, 20+ images, good tracking. Physical size: Required for correct scale. Tracking quality: Both provide states (tracked, paused, stopped). Unity: AR Foundation unifies both.
🍎 ARKit🤖 ARCore📏 Physical Size📊 Quality
// ARKit image tracking setup (Swift)
import ARKit
class ARKitImageTracking: UIViewController, ARSCNViewDelegate {
@IBOutlet var sceneView: ARSCNView!
override func viewDidLoad() {
super.viewDidLoad()
sceneView.delegate = self
// Load reference images
guard let referenceImages = ARReferenceImage.referenceImages(
inGroupNamed: "AR Resources",
bundle: nil
) else {
fatalError("Missing expected asset catalog resources.")
}
// Configure session
let configuration = ARWorldTrackingConfiguration()
configuration.detectionImages = referenceImages
configuration.maximumNumberOfTrackedImages = 4
sceneView.session.run(configuration)
}
func renderer(_ renderer: SCNSceneRenderer, nodeFor anchor: ARAnchor) -> SCNNode? {
guard let imageAnchor = anchor as? ARImageAnchor else { return nil }
let plane = SCNPlane(
width: imageAnchor.referenceImage.physicalSize.width,
height: imageAnchor.referenceImage.physicalSize.height
)
// Create AR content based on image name
let node = SCNNode()
switch imageAnchor.referenceImage.name {
case "poster":
node.addChildNode(createPosterContent())
case "card":
node.addChildNode(createCardContent())
default:
node.addChildNode(createDefaultContent())
}
return node
}
}
// ARCore image tracking setup (Kotlin)
import com.google.ar.core.*
class ARCoreImageTracking : AppCompatActivity() {
private lateinit var session: Session
private lateinit var arFragment: ArFragment
override fun onCreate(savedInstanceState: Bundle?) {
super.onCreate(savedInstanceState)
setContentView(R.layout.activity_main)
arFragment = supportFragmentManager.findFragmentById(R.id.ar_fragment) as ArFragment
session = arFragment.arSceneView.session
// Create image database
val imageDatabase = AugmentedImageDatabase(session)
// Add images from assets
val bitmap = loadBitmapFromAssets("poster.jpg")
imageDatabase.addImage("poster", bitmap, 0.3f) // 30cm
// Configure session
val config = Config(session)
config.augmentedImageDatabase = imageDatabase
config.focusMode = Config.FocusMode.AUTO
config.lightEstimationMode = Config.LightEstimationMode.ENVIRONMENTAL_HDR
session.configure(config)
// Set up AR scene
arFragment.arSceneView.scene.addOnUpdateListener { frameTime ->
onUpdate(frameTime)
}
}
private fun onUpdate(frameTime: FrameTime) {
val frame = arFragment.arSceneView.arFrame ?: return
val images = frame.getUpdatedTrackables(AugmentedImage::class.java)
for (image in images) {
when (image.trackingState) {
TrackingState.TRACKING -> {
// Image is being tracked
if (!trackedImages.containsKey(image.name)) {
// First time seeing this image
addARContent(image)
trackedImages[image.name] = image
} else {
// Update existing content
updateARContent(image)
}
}
TrackingState.PAUSED -> {
// Tracking paused (image partially visible)
pauseARContent(image)
}
TrackingState.STOPPED -> {
// Tracking stopped (image no longer visible)
removeARContent(image)
trackedImages.remove(image.name)
}
}
}
}
}
// AR Foundation (Unity) — cross-platform
using UnityEngine;
using UnityEngine.XR.ARFoundation;
using UnityEngine.XR.ARSubsystems;
public class ARFoundationImageTracking : MonoBehaviour {
public ARTrackedImageManager imageManager;
public GameObject[] imagePrefabs;
void OnEnable() {
imageManager.trackedImagesChanged += OnTrackedImagesChanged;
}
void OnDisable() {
imageManager.trackedImagesChanged -= OnTrackedImagesChanged;
}
void OnTrackedImagesChanged(ARTrackedImagesChangedEventArgs args) {
// Added
foreach (var image in args.added) {
var prefab = GetPrefabForImage(image.referenceImage.name);
var go = Instantiate(prefab, image.transform);
trackedObjects[image.trackableId] = go;
}
// Updated
foreach (var image in args.updated) {
if (trackedObjects.TryGetValue(image.trackableId, out var go)) {
go.transform.position = image.transform.position;
go.transform.rotation = image.transform.rotation;
go.SetActive(image.trackingState == TrackingState.Tracking);
}
}
// Removed
foreach (var image in args.removed) {
if (trackedObjects.TryGetValue(image.trackableId, out var go)) {
Destroy(go);
trackedObjects.Remove(image.trackableId);
}
}
}
GameObject GetPrefabForImage(string imageName) {
// Map image name to prefab
switch (imageName) {
case "poster": return imagePrefabs[0];
case "card": return imagePrefabs[1];
default: return imagePrefabs[0];
}
}
}🌐 WebXR Image Tracking
🎯 Image tracking in the browser
WebXR supports image tracking via the Image Tracking API (part of the WebXR Augmented Reality Module). It allows web apps to detect and track images without native installation.
WebXR Image Tracking
Feature: image-tracking required feature. Image library: Provide images to the XR session. Results: XRImageTrackingResult with pose and tracking state. Browser support: Chrome Android, Samsung Internet. Libraries: Three.js, A-Frame, Babylon.js support WebXR image tracking.
🌐 WebXR📚 Image Library📊 Tracking State📦 Three.js
// WebXR Image Tracking (JavaScript)
async function startImageTracking() {
// Check support
if (!navigator.xr) {
console.log("WebXR not supported");
return;
}
const supported = await navigator.xr.isSessionSupported('immersive-ar');
if (!supported) {
console.log("Immersive AR not supported");
return;
}
// Create image library
const imageLibrary = await createImageLibrary([
{ name: 'poster', url: 'poster.jpg', width: 0.3 },
{ name: 'card', url: 'card.jpg', width: 0.1 }
]);
// Request session
const session = await navigator.xr.requestSession('immersive-ar', {
requiredFeatures: ['image-tracking'],
optionalFeatures: ['local-floor', 'hit-test']
});
// Set up image tracking
const imageTracking = await session.requestImageTracking(imageLibrary);
// Get reference space
const refSpace = await session.requestReferenceSpace('local');
// Render loop
function onXRFrame(time, frame) {
session.requestAnimationFrame(onXRFrame);
// Get tracked images
const results = frame.getImageTrackingResults();
for (const result of results) {
const imageIndex = result.index;
const trackingState = result.trackingState;
const pose = frame.getPose(result.imageSpace, refSpace);
if (trackingState === 'tracked' && pose) {
// Update AR content for this image
const imageName = imageLibrary[imageIndex].name;
updateARContent(imageName, pose);
}
}
// Render scene
renderer.render(scene, camera);
}
session.requestAnimationFrame(onXRFrame);
}
// Create image library
async function createImageLibrary(images) {
const library = [];
for (const img of images) {
const image = new Image();
image.src = img.url;
await image.decode();
library.push({
name: img.name,
image: image,
width: img.width
});
}
return library;
}
// Three.js WebXR image tracking
import * as THREE from 'three';
import { ARButton } from 'three/addons/webxr/ARButton.js';
const scene = new THREE.Scene();
const camera = new THREE.PerspectiveCamera();
const renderer = new THREE.WebGLRenderer({ antialias: true });
renderer.xr.enabled = true;
// Add AR button
document.body.appendChild(ARButton.createButton(renderer, {
requiredFeatures: ['image-tracking'],
optionalFeatures: ['local-floor']
}));
// Load reference images
const imageLibrary = [
{ name: 'poster', image: 'poster.jpg', width: 0.3 }
];
// AR content
const content = {};
// Render loop
renderer.setAnimationLoop((time, frame) => {
if (frame) {
const results = frame.getImageTrackingResults();
for (const result of results) {
const index = result.index;
const state = result.trackingState;
const pose = frame.getPose(result.imageSpace, renderer.xr.getReferenceSpace());
if (state === 'tracked' && pose) {
const name = imageLibrary[index].name;
if (!content[name]) {
// Create AR content for this image
content[name] = createARContent(name);
scene.add(content[name]);
}
// Update position from image pose
content[name].position.copy(pose.transform.position);
content[name].quaternion.copy(pose.transform.orientation);
}
}
}
renderer.render(scene, camera);
});🎨 AR Content on Images
🎯 What to place on tracked images
Once an image is tracked, AR content is anchored to it: 3D models, videos, animations, text, buttons, portals. The content should feel like it belongs on the image — as if printed on the surface.
Content Types
3D models: Characters, products, objects. Video: Play video on the image. Animations: 2D/3D animations. Portals: Look into another world. Interactive: Buttons, sliders, games. Text: Info, labels, translations. Effects: Particles, shaders.
🧊 3D Models🎥 Video🌀 Portals🎮 Interactive
// AR content on tracked images (Unity)
using UnityEngine;
using UnityEngine.XR.ARFoundation;
using UnityEngine.Video;
public class ImageARContent : MonoBehaviour {
public ARTrackedImageManager imageManager;
public GameObject[] contentPrefabs;
public VideoPlayer videoPlayer;
void OnTrackedImagesChanged(ARTrackedImagesChangedEventArgs args) {
foreach (var image in args.added) {
// Create content based on image name
GameObject content = CreateContent(image.referenceImage.name);
// Parent to image
content.transform.SetParent(image.transform, false);
content.transform.localPosition = Vector3.zero;
content.transform.localRotation = Quaternion.identity;
// Scale content to match image size
float imageWidth = image.referenceImage.size.x;
float imageHeight = image.referenceImage.size.y;
ScaleContentToImage(content, imageWidth, imageHeight);
}
}
GameObject CreateContent(string imageName) {
switch (imageName) {
case "movie_poster":
return CreateMoviePosterContent();
case "product_box":
return CreateProductContent();
case "book_cover":
return CreateBookContent();
case "business_card":
return CreateBusinessCardContent();
default:
return Instantiate(contentPrefabs[0]);
}
}
GameObject CreateMoviePosterContent() {
// 3D model + video + animation
var root = new GameObject("MoviePosterContent");
// 3D character
var character = Instantiate(characterPrefab);
character.transform.SetParent(root.transform);
// Video screen
var screen = GameObject.CreatePrimitive(PrimitiveType.Quad);
screen.transform.SetParent(root.transform);
screen.transform.localPosition = new Vector3(0, 0, 0.01f);
screen.transform.localScale = new Vector3(0.8f, 0.6f, 1);
videoPlayer = screen.AddComponent();
videoPlayer.url = "https://example.com/trailer.mp4";
videoPlayer.playOnAwake = false;
videoPlayer.isLooping = true;
// Play button
var button = CreateButton("Play Trailer");
button.transform.SetParent(root.transform);
button.transform.localPosition = new Vector3(0, -0.5f, 0.05f);
button.onClick.AddListener(() => {
videoPlayer.Play();
});
return root;
}
void ScaleContentToImage(GameObject content, float imageWidth, float imageHeight) {
// Ensure content fits within image bounds
var renderers = content.GetComponentsInChildren();
if (renderers.Length == 0) return;
Bounds bounds = renderers[0].bounds;
for (int i = 1; i < renderers.Length; i++) {
bounds.Encapsulate(renderers[i].bounds);
}
float contentWidth = bounds.size.x;
float contentHeight = bounds.size.z;
float scaleX = imageWidth / contentWidth;
float scaleZ = imageHeight / contentHeight;
float scale = Mathf.Min(scaleX, scaleZ) * 0.9f; // 90% of image
content.transform.localScale = Vector3.one * scale;
}
}
// Portal effect on image
public class ImagePortal : MonoBehaviour {
public Camera portalCamera;
public RenderTexture portalTexture;
public Material portalMaterial;
void Start() {
// Create portal camera
portalCamera = new GameObject("PortalCamera").AddComponent();
portalCamera.enabled = false;
// Render portal view
portalCamera.targetTexture = portalTexture;
// Apply to image plane
GetComponent().material = portalMaterial;
portalMaterial.mainTexture = portalTexture;
}
void Update() {
// Position portal camera relative to user
var userCamera = Camera.main;
portalCamera.transform.position = transform.position +
(userCamera.transform.position - transform.position).normalized * -1;
portalCamera.transform.LookAt(transform.position);
// Render
portalCamera.Render();
}
}
// Interactive content on image
public class ImageInteractiveContent : MonoBehaviour {
public GameObject menuPrefab;
void Start() {
// Add tap interaction
var collider = gameObject.AddComponent();
// Raycast for taps
if (Input.touchCount > 0) {
var touch = Input.GetTouch(0);
if (touch.phase == TouchPhase.Began) {
Ray ray = Camera.main.ScreenPointToRay(touch.position);
if (Physics.Raycast(ray, out RaycastHit hit)) {
if (hit.collider.gameObject == gameObject) {
ShowMenu();
}
}
}
}
}
void ShowMenu() {
var menu = Instantiate(menuPrefab, transform.position, transform.rotation);
// Add menu options...
}
} ⚠️ Challenges & Failure Modes
🎯 When image tracking fails
Image tracking faces challenges: lighting changes, occlusion, motion blur, scale changes, and poor image quality. Robust systems must handle these gracefully.
Common Issues
Lighting: Glare, shadows, low light. Occlusion: Image partially covered. Motion: Fast movement causes blur. Angle: Extreme angles reduce features. Distance: Too far = too few pixels. Scale: Wrong physical size = wrong content scale. Repetition: Repetitive patterns confuse matching.
💡 Lighting🚫 Occlusion🏃 Motion📐 Angle
# Robust image tracking with failure handling
class RobustImageTracking:
def __init__(self, reference_images):
self.library = ReferenceImageLibrary()
for img in reference_images:
self.library.add_image(img.id, img.image, img.width)
self.trackers = {} # image_id -> ImageTracker
self.state = {} # image_id -> TrackingState
self.lost_frames = {} # image_id -> count
def update(self, frame):
# First, try to detect new images
detected = self.detect_images(frame)
# Update existing trackers
for image_id, tracker in self.trackers.items():
if self.state[image_id] == TrackingState.TRACKING:
if tracker.track(frame):
self.lost_frames[image_id] = 0
else:
self.lost_frames[image_id] += 1
if self.lost_frames[image_id] > 5:
self.state[image_id] = TrackingState.LOST
# Keep content visible for a few more frames
if self.lost_frames[image_id] > 15:
self.state[image_id] = TrackingState.STOPPED
del self.trackers[image_id]
# Handle detected images
for detection in detected:
image_id = detection['image_id']
if image_id not in self.trackers:
# New detection
self.trackers[image_id] = ImageTracker(
self.library.get_image(image_id)['image']
)
self.state[image_id] = TrackingState.TRACKING
self.lost_frames[image_id] = 0
elif self.state[image_id] == TrackingState.LOST:
# Re-acquired
self.state[image_id] = TrackingState.TRACKING
self.lost_frames[image_id] = 0
return self.get_tracked_images()
def detect_images(self, frame):
"""Detect images in the frame."""
gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
live_kp, live_desc = extract_features(gray, 'ORB')
if live_desc is None:
return []
detections = []
for image_id, feat in self.library.features.items():
# Skip if already tracking
if image_id in self.trackers and self.state[image_id] == TrackingState.TRACKING:
continue
# Match
matches = match_features(feat['descriptors'], live_desc, 'ORB')
if len(matches) < 15:
continue
# Verify with homography
ref_pts = np.float32([feat['keypoints'][m.queryIdx].pt for m in matches])
live_pts = np.float32([live_kp[m.trainIdx].pt for m in matches])
H, mask = cv2.findHomography(ref_pts, live_pts, cv2.RANSAC, 5.0)
if H is None or np.sum(mask) < 15:
continue
detections.append({
'image_id': image_id,
'homography': H,
'inliers': np.sum(mask),
'matches': len(matches)
})
return detections
def get_tracked_images(self):
"""Get list of currently tracked images with their poses."""
tracked = []
for image_id, tracker in self.trackers.items():
if self.state[image_id] == TrackingState.TRACKING:
tracked.append({
'image_id': image_id,
'pose': tracker.current_pose,
'quality': assess_tracking_quality(
tracker.matches, tracker.inliers, tracker.error
)
})
return tracked
# Handling lighting changes
def enhance_for_tracking(frame):
"""Enhance frame for better tracking in poor lighting."""
# 1. Histogram equalization
gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
equalized = cv2.equalizeHist(gray)
# 2. CLAHE (adaptive)
clahe = cv2.createCLAHE(clipLimit=2.0, tileGridSize=(8, 8))
enhanced = clahe.apply(gray)
# 3. Denoise
denoised = cv2.fastNlMeansDenoising(enhanced, None, 10, 7, 21)
return denoised
# Handling occlusion
def handle_occlusion(matches, inliers, total_features):
"""Detect and handle occlusion."""
visibility = inliers / total_features
if visibility < 0.3:
return "heavily_occluded"
elif visibility < 0.6:
return "partially_occluded"
else:
return "clear"
# Handling motion blur
def deblur_image(image, kernel_size=15):
"""Deblur image using Wiener filter."""
# Estimate blur kernel
kernel = np.zeros((kernel_size, kernel_size))
kernel[int((kernel_size-1)/2), :] = np.ones(kernel_size)
kernel = kernel / kernel_size
# Wiener deconvolution
from skimage import restoration
deconvolved, _ = restoration.wiener(image, kernel, 0.1)
return deconvolved🚀 Future: Neural Image Tracking
🎯 AI-powered image tracking
Deep learning is transforming image tracking: learned features (SuperPoint, D2-Net), learned matching (SuperGlue, LoFTR), pose estimation networks, and generative AR content.
Neural Image Tracking Advances
SuperPoint: Self-supervised feature detection. SuperGlue: Graph neural network matching. LoFTR: Detector-free matching. D2-Net: Joint detection and description. PoseNet: Direct pose regression. Foundation models: SAM, DINOv2 for robust tracking.
🧠 SuperPoint🔗 SuperGlue🦖 DINOv2📐 PoseNet
# SuperPoint + SuperGlue for robust image tracking
import torch
import cv2
import numpy as np
class SuperPointSuperGlue:
def __init__(self):
# Load models
self.superpoint = torch.hub.load(
'magicleap/SuperGluePretrainedNetwork',
'superpoint'
).eval().cuda()
self.superglue = torch.hub.load(
'magicleap/SuperGluePretrainedNetwork',
'superglue',
'indoor' # or 'outdoor'
).eval().cuda()
def extract_features(self, image):
"""Extract SuperPoint features."""
gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
inp = torch.from_numpy(gray).float()[None, None].cuda() / 255.0
with torch.no_grad():
pred = self.superpoint({'image': inp})
keypoints = pred['keypoints'][0].cpu().numpy()
descriptors = pred['descriptors'][0].cpu().numpy()
scores = pred['scores'][0].cpu().numpy()
return keypoints, descriptors, scores
def match(self, kp1, desc1, kp2, desc2):
"""Match features with SuperGlue."""
# Prepare inputs
inp = {
'image0': torch.zeros(1, 1, 480, 640).cuda(), # placeholder
'image1': torch.zeros(1, 1, 480, 640).cuda(),
'keypoints0': [torch.from_numpy(kp1).float().cuda()],
'keypoints1': [torch.from_numpy(kp2).float().cuda()],
'descriptors0': [torch.from_numpy(desc1).float().cuda()],
'descriptors1': [torch.from_numpy(desc2).float().cuda()],
'scores0': [torch.ones(len(kp1)).cuda()],
'scores1': [torch.ones(len(kp2)).cuda()],
}
with torch.no_grad():
pred = self.superglue(inp)
matches = pred['matches0'][0].cpu().numpy()
confidence = pred['matching_scores0'][0].cpu().numpy()
# Filter valid matches
valid = matches > -1
matched_kp1 = kp1[valid]
matched_kp2 = kp2[matches[valid]]
matched_conf = confidence[valid]
return matched_kp1, matched_kp2, matched_conf
def track(self, reference_image, live_frame):
"""Track reference image in live frame."""
# Extract features
kp1, desc1, _ = self.extract_features(reference_image)
kp2, desc2, _ = self.extract_features(live_frame)
# Match
mkp1, mkp2, conf = self.match(kp1, desc1, kp2, desc2)
if len(mkp1) < 10:
return None
# Estimate homography
H, mask = cv2.findHomography(mkp1, mkp2, cv2.RANSAC, 5.0)
if H is None or np.sum(mask) < 10:
return None
return H
# LoFTR: Detector-free matching
class LoFTRTracker:
def __init__(self):
self.matcher = torch.hub.load(
'zju3dv/LoFTR',
'indoor_ds'
).eval().cuda()
def track(self, reference_image, live_frame):
"""Track with LoFTR (no feature detection needed)."""
# Prepare inputs
img1 = torch.from_numpy(reference_image).float()[None].cuda() / 255.0
img2 = torch.from_numpy(live_frame).float()[None].cuda() / 255.0
# Match
with torch.no_grad():
correspondences = self.matcher({
'image0': img1,
'image1': img2
})
mkp1 = correspondences['keypoints0'].cpu().numpy()
mkp2 = correspondences['keypoints1'].cpu().numpy()
conf = correspondences['confidence'].cpu().numpy()
# Filter by confidence
valid = conf > 0.5
mkp1 = mkp1[valid]
mkp2 = mkp2[valid]
if len(mkp1) < 10:
return None
# Estimate homography
H, mask = cv2.findHomography(mkp1, mkp2, cv2.RANSAC, 5.0)
return H
# Neural pose estimation
class NeuralPoseEstimator(torch.nn.Module):
def __init__(self):
super().__init__()
# Backbone (e.g., ResNet)
import torchvision.models as models
self.backbone = models.resnet50(pretrained=True)
self.backbone.fc = torch.nn.Identity()
# Pose regression head
self.pose_head = torch.nn.Sequential(
torch.nn.Linear(2048, 512),
torch.nn.ReLU(),
torch.nn.Linear(512, 256),
torch.nn.ReLU(),
torch.nn.Linear(256, 7) # quaternion (4) + translation (3)
)
def forward(self, image):
features = self.backbone(image)
pose = self.pose_head(features)
# Split into quaternion and translation
quaternion = pose[:, :4]
translation = pose[:, 4:]
# Normalize quaternion
quaternion = quaternion / torch.norm(quaternion, dim=1, keepdim=True)
return quaternion, translation
# Training
model = NeuralPoseEstimator().cuda()
optimizer = torch.optim.Adam(model.parameters(), lr=1e-4)
for images, poses_gt in dataloader:
quat_pred, trans_pred = model(images)
quat_gt = poses_gt[:, :4]
trans_gt = poses_gt[:, 4:]
loss = torch.nn.MSELoss()(quat_pred, quat_gt) + \
torch.nn.MSELoss()(trans_pred, trans_gt)
optimizer.zero_grad()
loss.backward()
optimizer.step()