🧠 XR UX Fundamentals

🎯 Designing for Humans in 3D

XR UX (Extended Reality User Experience) is the practice of designing intuitive, comfortable, and delightful experiences for AR, VR, and MR. Unlike 2D screens, XR involves the whole body — head, hands, eyes, and even balance. Great XR UX respects human physiology, perception, and cognition.

Core Principles of XR UX

1. Comfort first: Never make the user sick. 2. Embodiment: Let users feel present in their virtual body. 3. Intuitive interaction: Leverage natural gestures and real-world metaphors. 4. Spatial consistency: Objects behave as expected in 3D. 5. Feedback: Every action has a visible, audible, or haptic response. 6. Accessibility: Design for diverse bodies and abilities.

📌 Comfort📌 Presence📌 Intuitive📌 Accessible

// XR UX checklist (pseudocode)
class XRExperience {
    validateUX() {
        // 1. Comfort
        assert(frameRate >= 72);  // never drop below refresh rate
        assert(latency < 20);     // motion-to-photon latency
        assert(noForcedCameraMovement);  // user controls camera
        
        // 2. Embodiment
        assert(handsPresent);     // show hands/controllers
        assert(virtualBodyMatchesRealBody);  // avoid mismatch
        
        // 3. Intuitive interaction
        assert(interactionsUseNaturalGestures);
        assert(feedbackOnEveryAction);
        
        // 4. Spatial consistency
        assert(gravityDirectionConsistent);
        assert(objectsHaveReasonableScale);
        
        // 5. Accessibility
        assert(supportsSeatedMode);
        assert(supportsOneHandedMode);
        assert(hasSubtitleOptions);
    }
}

🛋️ Comfort & Motion Sickness

🎯 The #1 UX Priority

Motion sickness (cybersickness) is the biggest barrier to XR adoption. It's caused by a mismatch between what your eyes see and what your inner ear feels. Comfort must be designed into every interaction.

Comfort Guidelines

Maintain 72+ fps: Dropped frames cause judder. Low latency: <20ms motion-to-photon. User-controlled camera: Never move the camera without user input. Vignetting: Darken periphery during movement. Snap turning: Instead of smooth rotation. Teleportation: Instead of smooth locomotion. Stable horizon: Keep the horizon level.

⚡ 72+ fps⏱️ <20ms🎥 User Camera🌑 Vignette🔄 Snap Turn

// Comfort settings in Unity XR
using UnityEngine;
using UnityEngine.XR.Interaction.Toolkit;

public class ComfortSettings : MonoBehaviour {
    public ContinuousMoveProvider continuousMove;
    public ContinuousTurnProvider continuousTurn;
    public SnapTurnProvider snapTurn;
    public TeleportationProvider teleport;
    
    // Vignette during movement
    public VignetteController vignette;
    
    void Start() {
        // Default: teleport + snap turn (most comfortable)
        teleport.enabled = true;
        snapTurn.enabled = true;
        continuousMove.enabled = false;
        continuousTurn.enabled = false;
    }
    
    public void SetComfortLevel(ComfortLevel level) {
        switch (level) {
            case ComfortLevel.Maximum:
                teleport.enabled = true;
                snapTurn.enabled = true;
                continuousMove.enabled = false;
                continuousTurn.enabled = false;
                break;
                
            case ComfortLevel.Medium:
                teleport.enabled = true;
                snapTurn.enabled = true;
                continuousMove.enabled = true;  // with vignette
                continuousTurn.enabled = false;
                vignette.enabled = true;
                break;
                
            case ComfortLevel.Minimum:
                teleport.enabled = false;
                snapTurn.enabled = false;
                continuousMove.enabled = true;
                continuousTurn.enabled = true;
                vignette.enabled = false;
                break;
        }
    }
}

public enum ComfortLevel {
    Maximum,   // teleport + snap turn
    Medium,    // smooth move + snap turn + vignette
    Minimum    // full smooth locomotion
}

// Frame rate guardian
public class FrameRateGuardian : MonoBehaviour {
    public int targetFrameRate = 72;
    public GameObject comfortWarning;
    
    void Update() {
        float fps = 1.0f / Time.unscaledDeltaTime;
        if (fps < targetFrameRate * 0.9f) {
            // Frame rate drop — reduce quality
            QualitySettings.SetQualityLevel(
                QualitySettings.GetQualityLevel() - 1
            );
            
            // Warn user
            comfortWarning.SetActive(true);
        }
    }
}

📏 Scale & Proportions

🎯 Getting the size right

In XR, scale is critical. Objects that are too large feel intimidating; too small feel toy-like. Incorrect scale breaks presence. Use real-world units (meters) and respect human proportions.

Scale Guidelines

1 unit = 1 meter: Standard in Unity/Unreal. Eye height: 1.6-1.8m average. Interactions: Buttons 2-5cm, grab objects 5-30cm. Text: Minimum 1° of visual angle (see Text section). Avatar: Match user's real proportions. World scale: Room-scale (2x2m) vs standing (1x1m) vs seated.

📐 1 unit = 1m👁️ 1.6-1.8m🔘 2-5cm buttons🧍 Room-scale

// Scale validation in Unity
using UnityEngine;

public class ScaleValidator : MonoBehaviour {
    [Header("Reference Heights (meters)")]
    public float averageEyeHeight = 1.7f;
    public float averageArmLength = 0.7f;
    public float averageShoulderWidth = 0.45f;
    
    [Header("Object Scale Ranges")]
    public Vector2 buttonSizeRange = new Vector2(0.02f, 0.05f);  // 2-5cm
    public Vector2 grabObjectRange = new Vector2(0.05f, 0.30f);  // 5-30cm
    public Vector2 furnitureRange = new Vector2(0.4f, 2.0f);     // 40cm-2m
    
    void Start() {
        ValidateScene();
    }
    
    void ValidateScene() {
        // Check camera height
        Camera xrCamera = Camera.main;
        float cameraHeight = xrCamera.transform.position.y;
        
        if (Mathf.Abs(cameraHeight - averageEyeHeight) > 0.3f) {
            Debug.LogWarning($"Camera height {cameraHeight}m is unusual. " +
                           $"Average is {averageEyeHeight}m.");
        }
        
        // Check interactable sizes
        foreach (var interactable in FindObjectsOfType()) {
            float size = interactable.GetComponent().bounds.size.magnitude;
            
            if (size < grabObjectRange.x) {
                Debug.LogWarning($"{interactable.name} is too small to grab " +
                               $"({size}m). Min: {grabObjectRange.x}m");
            }
            if (size > grabObjectRange.y) {
                Debug.LogWarning($"{interactable.name} is too large to grab " +
                               $"({size}m). Max: {grabObjectRange.y}m");
            }
        }
    }
}

// Dynamic scale adjustment for seated vs standing
public class DynamicScale : MonoBehaviour {
    public Transform xrOrigin;
    public float standingHeight = 1.7f;
    public float seatedHeight = 1.2f;
    
    void Update() {
        // Detect if user is seated (head height < threshold)
        float headHeight = Camera.main.transform.position.y;
        bool isSeated = headHeight < 1.4f;
        
        // Adjust world scale if needed
        if (isSeated) {
            // Raise virtual floor to match seated height
            xrOrigin.position = new Vector3(
                xrOrigin.position.x,
                seatedHeight - headHeight,
                xrOrigin.position.z
            );
        }
    }
}

🚪 Affordances & Signifiers

🎯 Showing what can be done

Affordances are the perceived action possibilities of an object. In XR, users can't rely on 2D conventions — objects must clearly communicate how to interact with them: grab, push, pull, rotate, or point.

XR Affordance Patterns

Visual: Highlight on hover, outline, glow, scale pulse. Audio: Hover sound, grab sound, release sound. Haptic: Vibration on hover, grab, collision. Spatial: Objects within reach, at comfortable angles. Metaphor: Use real-world metaphors (door handle, button, lever).

✨ Hover🔊 Audio📳 Haptic🚪 Metaphor

// Affordance system in Unity XR
using UnityEngine;
using UnityEngine.XR.Interaction.Toolkit;

public class AffordanceSystem : MonoBehaviour {
    public Material normalMaterial;
    public Material hoverMaterial;
    public Material grabMaterial;
    public AudioClip hoverSound;
    public AudioClip grabSound;
    public XRBaseController controller;
    
    private XRGrabInteractable grabInteractable;
    private Renderer objectRenderer;
    private AudioSource audioSource;
    
    void Start() {
        grabInteractable = GetComponent();
        objectRenderer = GetComponent();
        audioSource = GetComponent();
        
        // Visual affordances
        grabInteractable.hoverEntered.AddListener(OnHoverEnter);
        grabInteractable.hoverExited.AddListener(OnHoverExit);
        grabInteractable.selectEntered.AddListener(OnGrab);
        grabInteractable.selectExited.AddListener(OnRelease);
    }
    
    void OnHoverEnter(HoverEnterEventArgs args) {
        // Visual: change material
        objectRenderer.material = hoverMaterial;
        
        // Audio: hover sound
        audioSource.PlayOneShot(hoverSound);
        
        // Haptic: light vibration
        controller.SendHapticImpulse(0.2f, 0.05f);
        
        // Spatial: slight scale pulse
        StartCoroutine(PulseScale(1.05f, 0.1f));
    }
    
    void OnHoverExit(HoverExitEventArgs args) {
        objectRenderer.material = normalMaterial;
        StartCoroutine(PulseScale(1.0f, 0.1f));
    }
    
    void OnGrab(SelectEnterEventArgs args) {
        objectRenderer.material = grabMaterial;
        audioSource.PlayOneShot(grabSound);
        controller.SendHapticImpulse(0.5f, 0.1f);
    }
    
    void OnRelease(SelectExitEventArgs args) {
        objectRenderer.material = hoverMaterial;
        controller.SendHapticImpulse(0.3f, 0.08f);
    }
    
    System.Collections.IEnumerator PulseScale(float targetScale, float duration) {
        Vector3 startScale = transform.localScale;
        Vector3 endScale = Vector3.one * targetScale;
        float elapsed = 0;
        
        while (elapsed < duration) {
            transform.localScale = Vector3.Lerp(startScale, endScale, elapsed / duration);
            elapsed += Time.deltaTime;
            yield return null;
        }
        
        transform.localScale = endScale;
    }
}

🎮 Interaction Paradigms

🎯 How users interact in XR

XR interaction paradigms: direct manipulation (grab, push, throw), ray casting (point and click), gaze (look and select), voice, and gesture. Choose based on distance, precision, and context.

Interaction Types

Direct: Natural, precise, for near objects. Ray: For distant objects, UI, menus. Gaze: Hands-free, for accessibility. Voice: For commands, search. Gesture: For system control, navigation. Hybrid: Combine for best UX (gaze + pinch).

🖐️ Direct🎯 Ray👁️ Gaze🗣️ Voice

// Multi-modal interaction in Unity XR
using UnityEngine;
using UnityEngine.XR.Interaction.Toolkit;
using UnityEngine.XR.Interaction.Toolkit.Samples.StarterAssets;

public class MultiModalInteraction : MonoBehaviour {
    // Direct interaction (near)
    public XRDirectInteractor leftDirectInteractor;
    public XRDirectInteractor rightDirectInteractor;
    
    // Ray interaction (far)
    public XRRayInteractor leftRayInteractor;
    public XRRayInteractor rightRayInteractor;
    
    // Gaze interaction
    public XRGazeInteractor gazeInteractor;
    
    // Voice
    public VoiceCommandSystem voiceSystem;
    
    void Update() {
        // Automatically switch between direct and ray based on distance
        UpdateInteractorMode();
        
        // Voice commands
        if (voiceSystem.HasCommand("select")) {
            // Select the currently gazed/hovered object
            var hovered = gazeInteractor.GetHoveredObject();
            if (hovered != null) {
                hovered.Select();
            }
        }
        
        if (voiceSystem.HasCommand("grab")) {
            var hovered = rightRayInteractor.GetHoveredObject();
            if (hovered != null) {
                hovered.Select();
            }
        }
    }
    
    void UpdateInteractorMode() {
        // If hand is close to object, use direct; otherwise ray
        float handDistance = GetClosestObjectDistance(rightDirectInteractor);
        
        if (handDistance < 0.3f) {
            rightDirectInteractor.enabled = true;
            rightRayInteractor.enabled = false;
        } else {
            rightDirectInteractor.enabled = false;
            rightRayInteractor.enabled = true;
        }
    }
    
    float GetClosestObjectDistance(XRBaseInteractor interactor) {
        // Find closest grabbable object
        float minDist = float.MaxValue;
        foreach (var interactable in FindObjectsOfType()) {
            float dist = Vector3.Distance(
                interactor.transform.position,
                interactable.transform.position
            );
            minDist = Mathf.Min(minDist, dist);
        }
        return minDist;
    }
}

🚶 Locomotion UX

🎯 Moving through virtual space

Locomotion is how users move in XR. It's the most challenging UX problem — balancing comfort, immersion, and ease. Options: teleport, smooth locomotion, room-scale, arm swinging, vehicle-based.

Locomotion Comparison

Teleport: Most comfortable, less immersive. Smooth: Immersive, can cause sickness. Room-scale: Natural, limited space. Arm swing: Natural walking motion, moderate comfort. Vehicle: Cockpit, comfortable. Blink: Teleport with visual blink.

🌀 Teleport🕹️ Smooth🚶 Room-scale💪 Arm Swing

// Locomotion system with comfort options
using UnityEngine;
using UnityEngine.XR.Interaction.Toolkit;

public class LocomotionManager : MonoBehaviour {
    public TeleportationProvider teleport;
    public ContinuousMoveProvider continuousMove;
    public ActionBasedContinuousMoveProvider armSwingMove;
    public SnapTurnProvider snapTurn;
    public ContinuousTurnProvider continuousTurn;
    
    [Header("Comfort Settings")]
    public bool enableVignette = true;
    public VignetteController vignette;
    public float vignetteIntensity = 0.5f;
    
    public enum LocomotionMode {
        Teleport,
        Smooth,
        ArmSwing
    }
    
    public void SetLocomotionMode(LocomotionMode mode) {
        // Disable all
        teleport.enabled = false;
        continuousMove.enabled = false;
        armSwingMove.enabled = false;
        
        switch (mode) {
            case LocomotionMode.Teleport:
                teleport.enabled = true;
                vignette.enabled = false;
                break;
                
            case LocomotionMode.Smooth:
                continuousMove.enabled = true;
                vignette.enabled = enableVignette;
                break;
                
            case LocomotionMode.ArmSwing:
                armSwingMove.enabled = true;
                vignette.enabled = false;
                break;
        }
    }
    
    public void SetTurnMode(bool snapTurnEnabled) {
        snapTurn.enabled = snapTurnEnabled;
        continuousTurn.enabled = !snapTurnEnabled;
    }
    
    void Update() {
        // Auto-vignette during smooth movement
        if (continuousMove.enabled && enableVignette) {
            float speed = continuousMove.GetVelocity().magnitude;
            float targetVignette = Mathf.Clamp01(speed / 3.0f) * vignetteIntensity;
            vignette.SetVignette(targetVignette);
        }
    }
}

// Teleportation with preview
public class TeleportPreview : MonoBehaviour {
    public XRRayInteractor rayInteractor;
    public GameObject teleportReticle;
    public LineRenderer teleportLine;
    
    void Update() {
        if (rayInteractor.TryGetCurrent3DRaycastHit(out RaycastHit hit)) {
            // Show reticle at hit point
            teleportReticle.SetActive(true);
            teleportReticle.transform.position = hit.point;
            teleportReticle.transform.rotation = Quaternion.LookRotation(hit.normal);
            
            // Color reticle based on validity
            bool isValid = hit.point.y > -1.0f;  // not below floor
            teleportReticle.GetComponent().material.color = 
                isValid ? Color.green : Color.red;
            
            // Draw line
            teleportLine.enabled = true;
            teleportLine.SetPosition(0, rayInteractor.transform.position);
            teleportLine.SetPosition(1, hit.point);
        } else {
            teleportReticle.SetActive(false);
            teleportLine.enabled = false;
        }
    }
}

🖥️ Spatial UI Design

🎯 UI that lives in 3D space

Spatial UI breaks free from 2D screens. It can be world-locked (fixed in space), body-locked (follows the user), or head-locked (always in view). Each has different use cases.

Spatial UI Types

World-locked: Menus, signs, dashboards — stay in place. Body-locked: Inventory, health — follow but don't obscure. Head-locked: HUD, notifications — always visible. Hand-locked: Tools, wrist menus. Best practices: Curved panels, 1-2m distance, 30-40° field of view.

🌍 World-locked🧍 Body-locked👤 Head-locked✋ Hand-locked

// Spatial UI system in Unity
using UnityEngine;
using UnityEngine.UI;

public class SpatialUI : MonoBehaviour {
    public enum UIAttachment {
        World,   // fixed in space
        Body,    // follows user, ignores head rotation
        Head,    // always in view
        Hand     // attached to controller
    }
    
    public UIAttachment attachment = UIAttachment.Body;
    public float distance = 1.5f;        // meters from user
    public float heightOffset = -0.2f;   // below eye level
    public float followSpeed = 5.0f;     // smooth follow
    
    public Transform xrCamera;
    public Transform xrOrigin;
    
    void Update() {
        switch (attachment) {
            case UIAttachment.World:
                // Fixed — do nothing
                break;
                
            case UIAttachment.Body:
                // Follow user position but not rotation
                Vector3 targetPos = xrOrigin.position + 
                    Vector3.forward * distance + 
                    Vector3.up * (xrCamera.position.y + heightOffset);
                transform.position = Vector3.Lerp(
                    transform.position, targetPos, 
                    Time.deltaTime * followSpeed
                );
                // Face the user
                transform.LookAt(new Vector3(
                    xrCamera.position.x, 
                    transform.position.y, 
                    xrCamera.position.z
                ));
                break;
                
            case UIAttachment.Head:
                // Always in front of the camera
                transform.position = xrCamera.position + 
                    xrCamera.forward * distance;
                transform.rotation = xrCamera.rotation;
                break;
                
            case UIAttachment.Hand:
                // Attached to hand (set parent)
                break;
        }
    }
    
    // Curved UI panel
    public void CreateCurvedPanel(int segments = 10, float radius = 2.0f) {
        Mesh mesh = new Mesh();
        Vector3[] vertices = new Vector3[(segments + 1) * 2];
        Vector2[] uv = new Vector2[vertices.Length];
        int[] triangles = new int[segments * 6];
        
        for (int i = 0; i <= segments; i++) {
            float angle = Mathf.Lerp(-30, 30, (float)i / segments) * Mathf.Deg2Rad;
            float x = Mathf.Sin(angle) * radius;
            float z = Mathf.Cos(angle) * radius - radius;
            
            vertices[i * 2] = new Vector3(x, 0.5f, z);
            vertices[i * 2 + 1] = new Vector3(x, -0.5f, z);
            uv[i * 2] = new Vector2((float)i / segments, 1);
            uv[i * 2 + 1] = new Vector2((float)i / segments, 0);
        }
        
        // Build triangles...
        mesh.vertices = vertices;
        mesh.uv = uv;
        mesh.triangles = triangles;
        mesh.RecalculateNormals();
        
        GetComponent().mesh = mesh;
    }
}

📝 Text & Legibility

🎯 Reading in 3D

Text in XR is challenging: low resolution, subpixel rendering, and head movement. Text must be large enough, high contrast, and positioned at comfortable reading distances.

Text Guidelines

Minimum size: 1° of visual angle (about 1cm at 1m distance). Recommended: 2-3° for body text. Distance: 0.5-2m for reading. Font: Sans-serif, high x-height. Contrast: Dark text on light or vice versa. Anti-aliasing: SDF (Signed Distance Field) text. Dynamic: Scale with distance.

📏 1° min📖 0.5-2m🔤 Sans-serif✨ SDF

// Dynamic text sizing in XR
using UnityEngine;
using TMPro;

public class XRTextSizer : MonoBehaviour {
    public TextMeshProUGUI textMesh;
    public float minVisualAngle = 1.0f;  // degrees
    public float preferredVisualAngle = 2.5f;
    public float maxDistance = 5.0f;
    
    private Transform xrCamera;
    
    void Start() {
        xrCamera = Camera.main.transform;
    }
    
    void Update() {
        float distance = Vector3.Distance(
            transform.position, 
            xrCamera.position
        );
        
        // Compute required font size for minimum visual angle
        // visual_angle = 2 * atan(size / (2 * distance))
        // size = 2 * distance * tan(visual_angle / 2)
        float requiredSize = 2 * distance * Mathf.Tan(
            minVisualAngle * Mathf.Deg2Rad / 2
        );
        
        float preferredSize = 2 * distance * Mathf.Tan(
            preferredVisualAngle * Mathf.Deg2Rad / 2
        );
        
        // Scale text
        float scale = preferredSize / textMesh.fontSize;
        transform.localScale = Vector3.one * scale;
        
        // Hide if too far
        textMesh.enabled = distance < maxDistance;
    }
}

// SDF text shader (Signed Distance Field)
// SDF text stays sharp at any scale
Shader "XR/SDFText" {
    Properties {
        _MainTex ("Font Atlas", 2D) = "white" {}
        _Color ("Text Color", Color) = (1,1,1,1)
        _Sharpness ("Sharpness", Range(0, 1)) = 0.5
    }
    SubShader {
        Tags { "Queue"="Transparent" "RenderType"="Transparent" }
        
        Pass {
            Blend SrcAlpha OneMinusSrcAlpha
            ZWrite Off
            Cull Off
            
            CGPROGRAM
            #pragma vertex vert
            #pragma fragment frag
            #include "UnityCG.cginc"
            
            struct appdata {
                float4 vertex : POSITION;
                float2 uv : TEXCOORD0;
            };
            
            struct v2f {
                float4 pos : SV_POSITION;
                float2 uv : TEXCOORD0;
            };
            
            sampler2D _MainTex;
            float4 _Color;
            float _Sharpness;
            
            v2f vert(appdata v) {
                v2f o;
                o.pos = UnityObjectToClipPos(v.vertex);
                o.uv = v.uv;
                return o;
            }
            
            fixed4 frag(v2f i) : SV_Target {
                // Sample SDF
                float distance = tex2D(_MainTex, i.uv).a;
                
                // Smoothstep for anti-aliasing
                float alpha = smoothstep(0.5 - _Sharpness, 0.5 + _Sharpness, distance);
                
                return fixed4(_Color.rgb, _Color.a * alpha);
            }
            ENDCG
        }
    }
}

🎧 Spatial Audio UX

🎯 Sound that surrounds

Spatial audio is critical for XR presence. It positions sound in 3D space, matching visual locations. It also provides feedback, guides attention, and increases immersion.

Spatial Audio Guidelines

HRTF: Head-related transfer function for 3D audio. Occlusion: Sounds muffled behind walls. Reverb: Matches environment (cave, room, outdoor). Distance: Volume falls off with distance. Direction: Sounds come from their visual source. Ambisonics: Full-sphere audio for environments. Feedback: UI sounds for interactions.

🎧 HRTF🧱 Occlusion🌊 Reverb📏 Distance

// Spatial audio in Unity XR
using UnityEngine;

public class XRSpatialAudio : MonoBehaviour {
    public AudioSource audioSource;
    public AudioClip clip;
    public float maxDistance = 20f;
    public float minDistance = 1f;
    
    void Start() {
        // Configure spatial audio
        audioSource.clip = clip;
        audioSource.spatialBlend = 1.0f;  // 3D audio
        audioSource.rolloffMode = AudioRolloffMode.Logarithmic;
        audioSource.minDistance = minDistance;
        audioSource.maxDistance = maxDistance;
        audioSource.dopplerLevel = 0.5f;
        
        // Enable spatialization (HRTF)
        audioSource.spatialize = true;
        audioSource.spatializePostEffects = true;
    }
    
    // Audio occlusion
    void Update() {
        // Check if there's an obstacle between listener and source
        Vector3 listenerPos = Camera.main.transform.position;
        Vector3 sourcePos = transform.position;
        Vector3 direction = sourcePos - listenerPos;
        float distance = direction.magnitude;
        
        if (Physics.Raycast(listenerPos, direction.normalized, out RaycastHit hit, distance)) {
            // Occluded — reduce volume and apply low-pass filter
            audioSource.volume = 0.3f;
            audioSource.GetComponent().cutoffFrequency = 1000f;
        } else {
            audioSource.volume = 1.0f;
            audioSource.GetComponent().cutoffFrequency = 22000f;
        }
    }
}

// UI audio feedback
public class XRUIAudio : MonoBehaviour {
    public AudioSource audioSource;
    public AudioClip hoverSound;
    public AudioClip clickSound;
    public AudioClip grabSound;
    
    public void PlayHover() {
        audioSource.PlayOneShot(hoverSound, 0.5f);
    }
    
    public void PlayClick() {
        audioSource.PlayOneShot(clickSound, 0.8f);
    }
    
    public void PlayGrab() {
        audioSource.PlayOneShot(grabSound, 0.7f);
    }
}

🚀 Onboarding & Tutorials

🎯 First impressions matter

XR onboarding teaches users how to interact. It must be quick, contextual, and non-intrusive. Best practices: learn by doing, just-in-time hints, progressive disclosure.

Onboarding Best Practices

Keep it short: Under 2 minutes. Interactive: Learn by doing, not reading. Contextual: Show hints when needed. Progressive: Introduce one concept at a time. Skippable: Let experienced users skip. Reinforce: Remind with subtle cues.

⏱️ <2 min🎮 Interactive📍 Contextual⏭️ Skippable

// XR onboarding system in Unity
using UnityEngine;
using UnityEngine.XR.Interaction.Toolkit;
using System.Collections;
using System.Collections.Generic;

public class XROnboarding : MonoBehaviour {
    [System.Serializable]
    public class TutorialStep {
        public string instruction;
        public GameObject highlightObject;
        public AudioClip voiceOver;
        public float duration;
        public System.Action onComplete;
    }
    
    public List steps = new List();
    public AudioSource voiceOverSource;
    public GameObject instructionPanel;
    public TextMeshPro instructionText;
    
    private int currentStep = 0;
    private bool isPlaying = false;
    
    void Start() {
        SetupSteps();
        StartCoroutine(PlayTutorial());
    }
    
    void SetupSteps() {
        steps.Add(new TutorialStep {
            instruction = "Look at your hands. These are your controllers.",
            duration = 3f,
            onComplete = () => { /* move to next */ }
        });
        
        steps.Add(new TutorialStep {
            instruction = "Point at the cube and press the trigger to select it.",
            highlightObject = GameObject.Find("Cube"),
            duration = 5f,
            onComplete = () => { /* check if cube selected */ }
        });
        
        steps.Add(new TutorialStep {
            instruction = "Grab the cube by squeezing the grip button.",
            highlightObject = GameObject.Find("Cube"),
            duration = 5f,
            onComplete = () => { /* check if cube grabbed */ }
        });
        
        steps.Add(new TutorialStep {
            instruction = "Use the thumbstick to move. Try it now.",
            duration = 5f,
            onComplete = () => { /* check if moved */ }
        });
        
        steps.Add(new TutorialStep {
            instruction = "You're ready! Enjoy the experience.",
            duration = 3f,
            onComplete = () => { /* end tutorial */ }
        });
    }
    
    IEnumerator PlayTutorial() {
        isPlaying = true;
        
        foreach (var step in steps) {
            // Show instruction
            instructionPanel.SetActive(true);
            instructionText.text = step.instruction;
            
            // Highlight object
            if (step.highlightObject != null) {
                HighlightObject(step.highlightObject, true);
            }
            
            // Play voice over
            if (step.voiceOver != null) {
                voiceOverSource.PlayOneShot(step.voiceOver);
            }
            
            // Wait for completion
            float elapsed = 0;
            bool completed = false;
            
            while (elapsed < step.duration && !completed) {
                elapsed += Time.deltaTime;
                
                // Check if step completed (custom logic)
                if (step.onComplete != null && CheckStepComplete(step)) {
                    completed = true;
                }
                
                yield return null;
            }
            
            // Clean up
            if (step.highlightObject != null) {
                HighlightObject(step.highlightObject, false);
            }
            
            yield return new WaitForSeconds(0.5f);
        }
        
        // End tutorial
        instructionPanel.SetActive(false);
        isPlaying = false;
        
        // Save completion
        PlayerPrefs.SetInt("TutorialComplete", 1);
    }
    
    bool CheckStepComplete(TutorialStep step) {
        // Custom completion checks
        return false;
    }
    
    void HighlightObject(GameObject obj, bool highlight) {
        var renderer = obj.GetComponent();
        if (highlight) {
            renderer.material.EnableKeyword("_EMISSION");
            renderer.material.SetColor("_EmissionColor", Color.yellow * 0.5f);
        } else {
            renderer.material.DisableKeyword("_EMISSION");
        }
    }
    
    public void SkipTutorial() {
        StopAllCoroutines();
        instructionPanel.SetActive(false);
        isPlaying = false;
        PlayerPrefs.SetInt("TutorialComplete", 1);
    }
}

♿ Accessibility in XR

🎯 XR for everyone

XR accessibility ensures people with diverse abilities can use immersive experiences. Considerations: seated mode, one-handed mode, subtitles, colorblind modes, audio cues, reduced motion.

Accessibility Features

Seated mode: For wheelchair users, fatigue. One-handed: For users with one hand. Subtitles: For deaf/hard-of-hearing. Visual cues: For deaf users (flash, icons). Audio cues: For blind/low-vision users. Colorblind: Alternative color palettes. Reduced motion: Less camera movement. Adjustable height: For different statures.

🪑 Seated✋ One-handed📝 Subtitles🎨 Colorblind

// Accessibility system in Unity XR
using UnityEngine;
using UnityEngine.XR.Interaction.Toolkit;
using UnityEngine.UI;

public class XRAccessibility : MonoBehaviour {
    [Header("Seated Mode")]
    public bool seatedMode = false;
    public Transform xrOrigin;
    public float standingHeight = 1.7f;
    public float seatedHeight = 1.2f;
    
    [Header("One-Handed Mode")]
    public bool oneHandedMode = false;
    public XRBaseController leftController;
    public XRBaseController rightController;
    
    [Header("Subtitles")]
    public bool subtitlesEnabled = true;
    public GameObject subtitlePanel;
    public Text subtitleText;
    
    [Header("Colorblind Mode")]
    public ColorblindMode colorblindMode = ColorblindMode.None;
    public Material[] colorblindMaterials;
    
    [Header("Reduced Motion")]
    public bool reducedMotion = false;
    public VignetteController vignette;
    
    public enum ColorblindMode {
        None,
        Protanopia,    // red-blind
        Deuteranopia,  // green-blind
        Tritanopia     // blue-blind
    }
    
    public void SetSeatedMode(bool enabled) {
        seatedMode = enabled;
        if (enabled) {
            xrOrigin.position = new Vector3(
                xrOrigin.position.x,
                seatedHeight - Camera.main.transform.localPosition.y,
                xrOrigin.position.z
            );
        }
    }
    
    public void SetOneHandedMode(bool enabled) {
        oneHandedMode = enabled;
        if (enabled) {
            // Mirror controls to both hands
            leftController.enabled = true;
            rightController.enabled = true;
        }
    }
    
    public void SetSubtitles(bool enabled) {
        subtitlesEnabled = enabled;
        subtitlePanel.SetActive(enabled);
    }
    
    public void SetColorblindMode(ColorblindMode mode) {
        colorblindMode = mode;
        ApplyColorblindMaterials(mode);
    }
    
    void ApplyColorblindMaterials(ColorblindMode mode) {
        // Apply colorblind-friendly shader
        Shader.SetGlobalInt("_ColorblindMode", (int)mode);
    }
    
    public void SetReducedMotion(bool enabled) {
        reducedMotion = enabled;
        if (enabled) {
            // Disable smooth locomotion, enable teleport
            FindObjectOfType().enabled = false;
            FindObjectOfType().enabled = true;
            
            // Increase vignette
            vignette.enabled = true;
            vignette.SetVignette(0.7f);
        }
    }
}

// Colorblind shader
Shader "XR/Colorblind" {
    Properties {
        _MainTex ("Texture", 2D) = "white" {}
        _Mode ("Mode", Int) = 0
    }
    SubShader {
        Pass {
            CGPROGRAM
            #pragma vertex vert
            #pragma fragment frag
            #include "UnityCG.cginc"
            
            sampler2D _MainTex;
            int _Mode;
            
            fixed4 frag(v2f i) : SV_Target {
                fixed4 color = tex2D(_MainTex, i.uv);
                
                // Apply colorblind correction matrix
                float3x3 matrix;
                if (_Mode == 1) {  // Protanopia
                    matrix = float3x3(
                        0.567, 0.433, 0.0,
                        0.558, 0.442, 0.0,
                        0.0, 0.242, 0.758
                    );
                } else if (_Mode == 2) {  // Deuteranopia
                    matrix = float3x3(
                        0.625, 0.375, 0.0,
                        0.7, 0.3, 0.0,
                        0.0, 0.3, 0.7
                    );
                } else if (_Mode == 3) {  // Tritanopia
                    matrix = float3x3(
                        0.95, 0.05, 0.0,
                        0.0, 0.433, 0.567,
                        0.0, 0.475, 0.525
                    );
                } else {
                    matrix = float3x3(1,0,0, 0,1,0, 0,0,1);
                }
                
                color.rgb = mul(matrix, color.rgb);
                return color;
            }
            ENDCG
        }
    }
}

🧪 User Testing in XR

🎯 Testing with real users

XR user testing is different from 2D — you must observe body movement, comfort, presence, and fatigue. Methods: think-aloud, biometrics, eye tracking, motion tracking, and questionnaires.

XR Testing Methods

Think-aloud: Users verbalize thoughts. Observation: Watch body movement, hesitations. Biometrics: Heart rate, pupil dilation for stress. Eye tracking: Where users look. Motion tracking: How users move. Questionnaires: SSQ (simulator sickness), IPQ (presence), NASA-TLX (workload).

🗣️ Think-aloud👀 Observation❤️ Biometrics📊 SSQ/IPQ

# XR user testing data collection
import time
import json
from dataclasses import dataclass, asdict

@dataclass
class XRTestSession:
    user_id: str
    session_start: float
    events: list
    metrics: dict
    
    def __init__(self, user_id):
        self.user_id = user_id
        self.session_start = time.time()
        self.events = []
        self.metrics = {
            'frame_drops': 0,
            'total_frames': 0,
            'avg_fps': 0,
            'head_movement': 0,
            'hand_movement': 0,
            'interactions': 0,
            'errors': 0
        }
    
    def log_event(self, event_type, data):
        self.events.append({
            'timestamp': time.time() - self.session_start,
            'type': event_type,
            'data': data
        })
    
    def log_interaction(self, interaction_type, target, success):
        self.metrics['interactions'] += 1
        if not success:
            self.metrics['errors'] += 1
        
        self.log_event('interaction', {
            'type': interaction_type,
            'target': target,
            'success': success
        })
    
    def log_frame(self, fps):
        self.metrics['total_frames'] += 1
        if fps < 60:
            self.metrics['frame_drops'] += 1
        
        # Running average
        n = self.metrics['total_frames']
        self.metrics['avg_fps'] = (
            (self.metrics['avg_fps'] * (n - 1) + fps) / n
        )
    
    def log_head_pose(self, position, rotation):
        self.metrics['head_movement'] += (
            abs(position.x) + abs(position.y) + abs(position.z)
        )
    
    def save(self, filepath):
        with open(filepath, 'w') as f:
            json.dump({
                'user_id': self.user_id,
                'duration': time.time() - self.session_start,
                'events': self.events,
                'metrics': self.metrics
            }, f, indent=2)

# Simulator Sickness Questionnaire (SSQ)
SSQ_QUESTIONS = [
    "General discomfort",
    "Fatigue",
    "Headache",
    "Eye strain",
    "Difficulty focusing",
    "Increased salivation",
    "Sweating",
    "Nausea",
    "Difficulty concentrating",
    "Fullness of head",
    "Blurred vision",
    "Dizziness (eyes open)",
    "Dizziness (eyes closed)",
    "Vertigo",
    "Stomach awareness",
    "Burping"
]

def compute_ssq(responses):
    """Compute SSQ score from 0-3 responses."""
    # Weights for nausea, oculomotor, disorientation
    nausea_items = [6, 7, 8, 9, 14, 15]
    oculomotor_items = [0, 1, 2, 3, 4, 10, 11]
    disorientation_items = [5, 12, 13]
    
    nausea = sum(responses[i] for i in nausea_items) * 9.54
    oculomotor = sum(responses[i] for i in oculomotor_items) * 7.58
    disorientation = sum(responses[i] for i in disorientation_items) * 13.92
    
    total = (nausea + oculomotor + disorientation) * 3.74
    return {
        'nausea': nausea,
        'oculomotor': oculomotor,
        'disorientation': disorientation,
        'total': total
    }

# iGroup Presence Questionnaire (IPQ)
IPQ_QUESTIONS = [
    "I felt like I was in the virtual world.",
    "The virtual world seemed real to me.",
    "I felt present in the virtual environment.",
    "I had a sense of being there.",
    "The virtual world felt like a place I visited.",
    "I felt like I was part of the virtual world."
]

def compute_ipq(responses):
    """Compute IPQ presence score (1-7 scale)."""
    return {
        'spatial_presence': sum(responses[0:3]) / 3,
        'involvement': sum(responses[3:5]) / 2,
        'realism': responses[5],
        'total': sum(responses) / len(responses)
    }

⚖️ Ethics & Well-being

🎯 Responsible XR design

XR raises ethical concerns: privacy (eye tracking, spatial mapping), addiction, harassment in social VR, psychological impact, and accessibility. Designers have a responsibility to protect users.

Ethical Considerations

Privacy: Eye tracking, spatial data, biometrics. Safety: Physical boundaries, content warnings. Social: Harassment prevention, moderation. Well-being: Break reminders, time limits. Manipulation: Avoid dark patterns. Inclusion: Diverse avatars, accessible design.

🔒 Privacy🛡️ Safety🤝 Social🧘 Well-being

// Ethical XR design system
using UnityEngine;
using UnityEngine.XR.Interaction.Toolkit;

public class EthicalXRDesign : MonoBehaviour {
    [Header("Well-being")]
    public float sessionTimeLimit = 3600f;  // 1 hour
    public float breakReminderInterval = 1800f;  // 30 min
    private float sessionStartTime;
    private float lastBreakReminder;
    
    [Header("Safety")]
    public GameObject boundaryWarning;
    public float boundaryDistance = 0.5f;  // 50cm from edge
    
    [Header("Privacy")]
    public bool eyeTrackingEnabled = false;
    public bool spatialMappingEnabled = true;
    public GameObject privacyIndicator;
    
    void Start() {
        sessionStartTime = Time.time;
        lastBreakReminder = Time.time;
        
        // Show privacy indicator
        privacyIndicator.SetActive(true);
        
        // Log session start (for well-being)
        Debug.Log("XR session started");
    }
    
    void Update() {
        float sessionDuration = Time.time - sessionStartTime;
        
        // Break reminder
        if (Time.time - lastBreakReminder > breakReminderInterval) {
            ShowBreakReminder();
            lastBreakReminder = Time.time;
        }
        
        // Session time limit
        if (sessionDuration > sessionTimeLimit) {
            ShowSessionEndWarning();
        }
        
        // Boundary warning
        CheckBoundaries();
        
        // Privacy indicator
        UpdatePrivacyIndicator();
    }
    
    void ShowBreakReminder() {
        // Show non-intrusive reminder
        // "You've been in VR for 30 minutes. Consider taking a break."
        ShowNotification("Time for a break", 
            "You've been in VR for 30 minutes. Consider taking a break.");
    }
    
    void ShowSessionEndWarning() {
        ShowNotification("Session ending soon", 
            "Your VR session will end in 5 minutes.");
    }
    
    void CheckBoundaries() {
        // Check if user is near play area boundary
        var origin = FindObjectOfType();
        if (origin != null) {
            // Get boundary from XRInputSubsystem
            // If near edge, show warning
            boundaryWarning.SetActive(false);
        }
    }
    
    void UpdatePrivacyIndicator() {
        // Show when eye tracking or spatial mapping is active
        bool privacyActive = eyeTrackingEnabled || spatialMappingEnabled;
        privacyIndicator.SetActive(privacyActive);
    }
    
    void ShowNotification(string title, string message) {
        // Display notification in XR
        Debug.Log($"{title}: {message}");
    }
    
    // Data privacy: delete user data
    public void DeleteUserData() {
        PlayerPrefs.DeleteAll();
        // Delete spatial maps
        // Delete eye tracking data
        Debug.Log("All user data deleted");
    }
    
    // Consent management
    public void RequestConsent(string feature, System.Action callback) {
        // Show consent dialog
        ShowNotification("Consent required", 
            $"This experience uses {feature}. Do you consent?");
        
        // In real implementation, show UI with Yes/No buttons
        // callback(true) or callback(false)
    }
}

🚀 Future: Adaptive XR UX

🎯 AI-powered personalized experiences

The future of XR UX is adaptive — experiences that learn from user behavior and adjust in real-time. AI will personalize comfort, difficulty, interaction, and content.

Adaptive XR UX

Comfort adaptation: Adjust locomotion based on user's motion sickness. Skill adaptation: Adjust difficulty based on performance. Interaction adaptation: Switch between gaze, hand, controller. Content adaptation: Show relevant content based on attention. Emotion adaptation: Detect stress and adjust pacing. Accessibility: Auto-detect needs and adapt.

🧠 AI🎯 Personalization📊 Real-time♿ Adaptive

# Adaptive XR UX with reinforcement learning
import numpy as np
import torch
import torch.nn as nn

class AdaptiveXRUX:
    """Adapts XR experience based on user behavior."""
    
    def __init__(self):
        self.user_profile = {
            'motion_sickness_sensitivity': 0.5,
            'skill_level': 0.5,
            'preferred_interaction': 'ray',
            'comfort_mode': 'medium',
            'fatigue_level': 0.0
        }
        
        # Neural network for predicting user state
        self.state_predictor = nn.Sequential(
            nn.Linear(10, 64), nn.ReLU(),
            nn.Linear(64, 64), nn.ReLU(),
            nn.Linear(64, 4)  # [sickness, skill, fatigue, engagement]
        )
    
    def update(self, user_data):
        """Update user profile based on real-time data."""
        # Extract features
        features = torch.tensor([
            user_data['head_velocity'],
            user_data['head_acceleration'],
            user_data['hand_velocity'],
            user_data['interaction_success_rate'],
            user_data['time_in_session'],
            user_data['blink_rate'],
            user_data['pupil_dilation'],
            user_data['error_rate'],
            user_data['movement_smoothness'],
            user_data['response_time']
        ])
        
        # Predict user state
        with torch.no_grad():
            state = self.state_predictor(features)
        
        sickness, skill, fatigue, engagement = state.tolist()
        
        # Update profile
        self.user_profile['motion_sickness_sensitivity'] = sickness
        self.user_profile['skill_level'] = skill
        self.user_profile['fatigue_level'] = fatigue
        
        # Adapt experience
        self.adapt_experience()
    
    def adapt_experience(self):
        """Adapt XR experience based on user profile."""
        # 1. Comfort adaptation
        if self.user_profile['motion_sickness_sensitivity'] > 0.7:
            # High sensitivity — use teleport
            self.set_locomotion('teleport')
            self.enable_vignette(True, intensity=0.8)
        elif self.user_profile['motion_sickness_sensitivity'] > 0.4:
            # Medium — smooth with vignette
            self.set_locomotion('smooth')
            self.enable_vignette(True, intensity=0.5)
        else:
            # Low — full smooth
            self.set_locomotion('smooth')
            self.enable_vignette(False)
        
        # 2. Skill adaptation
        if self.user_profile['skill_level'] < 0.3:
            # Beginner — show more hints
            self.show_hints(True)
            self.set_difficulty('easy')
        elif self.user_profile['skill_level'] < 0.7:
            # Intermediate
            self.show_hints(False)
            self.set_difficulty('medium')
        else:
            # Expert
            self.show_hints(False)
            self.set_difficulty('hard')
        
        # 3. Fatigue adaptation
        if self.user_profile['fatigue_level'] > 0.7:
            # High fatigue — suggest break
            self.suggest_break()
            self.reduce_intensity()
        elif self.user_profile['fatigue_level'] > 0.4:
            # Medium — reduce pacing
            self.reduce_pacing()
        
        # 4. Interaction adaptation
        if self.user_profile['skill_level'] > 0.7:
            # Expert — enable advanced interactions
            self.enable_advanced_interactions(True)
        else:
            self.enable_advanced_interactions(False)
    
    def set_locomotion(self, mode):
        print(f"Adaptive UX: Setting locomotion to {mode}")
    
    def enable_vignette(self, enabled, intensity=0.5):
        print(f"Adaptive UX: Vignette {'enabled' if enabled else 'disabled'}, intensity={intensity}")
    
    def show_hints(self, show):
        print(f"Adaptive UX: Hints {'shown' if show else 'hidden'}")
    
    def set_difficulty(self, level):
        print(f"Adaptive UX: Difficulty set to {level}")
    
    def suggest_break(self):
        print("Adaptive UX: Suggesting a break")
    
    def reduce_intensity(self):
        print("Adaptive UX: Reducing intensity")
    
    def reduce_pacing(self):
        print("Adaptive UX: Reducing pacing")
    
    def enable_advanced_interactions(self, enabled):
        print(f"Adaptive UX: Advanced interactions {'enabled' if enabled else 'disabled'}")

# Usage
adaptive_ux = AdaptiveXRUX()

# Simulated user data (real-time)
user_data = {
    'head_velocity': 0.5,
    'head_acceleration': 0.2,
    'hand_velocity': 0.3,
    'interaction_success_rate': 0.85,
    'time_in_session': 1800,  # 30 minutes
    'blink_rate': 15,  # blinks per minute
    'pupil_dilation': 4.5,  # mm
    'error_rate': 0.1,
    'movement_smoothness': 0.8,
    'response_time': 0.5  # seconds
}

adaptive_ux.update(user_data)
print(f"Updated profile: {adaptive_ux.user_profile}")