🧠 Spatial Audio Fundamentals
🎯 What is Spatial Audio?
Spatial audio (3D audio, positional audio, immersive audio) simulates sound coming from specific locations in 3D space. It uses HRTF (head-related transfer functions), distance attenuation, occlusion, and reverb to create the illusion of sound sources around the listener. It's essential for presence in AR/VR, games, and virtual concerts.
Why Spatial Audio Matters
Presence: Sound is half the experience. Localization: Know where threats, allies, or objects are. Immersion: 3D audio increases presence by 40%+. Accessibility: Audio cues for visually impaired. Emotion: Reverb and distance affect mood. Applications: VR games, virtual concerts, AR navigation, telepresence.
📌 HRTF📌 3D Position📌 Occlusion📌 Reverb
# Spatial audio pipeline (conceptual)
class SpatialAudioEngine:
def __init__(self, sample_rate=48000):
self.sample_rate = sample_rate
self.listener = Listener()
self.sources = []
self.hrtf = HRTFDatabase()
self.reverb = ReverbProcessor()
def update(self, listener_pose):
self.listener.update(listener_pose)
for source in self.sources:
# 1. Compute relative position
relative_pos = source.position - self.listener.position
# 2. Apply distance attenuation
distance = np.linalg.norm(relative_pos)
gain = 1.0 / (1.0 + distance)
# 3. Apply HRTF based on direction
direction = relative_pos / distance
left_gain, right_gain = self.hrtf.compute(direction)
# 4. Apply occlusion
occlusion = self.compute_occlusion(source, self.listener)
# 5. Apply reverb
wet_signal = self.reverb.process(source.audio, self.listener.room)
# 6. Mix
source.output = (
source.audio * gain * occlusion *
np.array([left_gain, right_gain])
) + wet_signal
def compute_occlusion(self, source, listener):
# Raycast from listener to source
# If blocked, apply low-pass filter and reduce gain
pass🧬 Psychoacoustics of Hearing
🎯 How humans perceive sound
Psychoacoustics studies how we perceive sound: frequency range (20Hz-20kHz), ITD (interaural time difference), ILD (interaural level difference), HRTF, and precedence effect.
Sound Localization Cues
ITD: Time difference between ears (up to 0.7ms). ILD: Level difference between ears (up to 20dB). HRTF: Head/ear shape filters sound differently by direction. Spectral cues: Pinna creates direction-dependent frequency notches. Precedence: First arriving sound dominates localization (Haas effect). Head movement: Resolves front-back ambiguity.
⏱️ ITD📊 ILD👂 HRTF🌈 Spectral
# ITD and ILD computation
import numpy as np
class SoundLocalization:
def __init__(self, head_radius=0.0875, speed_of_sound=343.0):
self.head_radius = head_radius # meters
self.speed_of_sound = speed_of_sound # m/s
def compute_itd(self, azimuth):
"""Compute interaural time difference for a given azimuth (radians)."""
# Woodworth model
if abs(azimuth) < np.pi / 2:
itd = (self.head_radius / self.speed_of_sound) * (azimuth + np.sin(azimuth))
else:
itd = (self.head_radius / self.speed_of_sound) * (
np.sign(azimuth) * (np.pi - abs(azimuth)) + np.sin(azimuth)
)
return itd # seconds (positive = right ear leads)
def compute_ild(self, azimuth, frequency):
"""Compute interaural level difference (simplified)."""
# Maximum ILD ~20dB at high frequencies
max_ild_db = 20.0
ild_db = max_ild_db * np.sin(azimuth)
# ILD increases with frequency (below 1kHz, minimal)
freq_factor = min(1.0, frequency / 1000.0)
return ild_db * freq_factor
def compute_gains(self, azimuth, elevation, frequency):
"""Compute left/right gains based on ITD, ILD, and HRTF."""
itd = self.compute_itd(azimuth)
ild = self.compute_ild(azimuth, frequency)
# Convert ILD to linear gains
ild_linear = 10 ** (ild / 20.0)
# Base gains
left_gain = 1.0 / ild_linear
right_gain = ild_linear
# Normalize
max_gain = max(left_gain, right_gain)
left_gain /= max_gain
right_gain /= max_gain
# Time delays
left_delay = max(0, -itd) # seconds
right_delay = max(0, itd)
return {
'left_gain': left_gain,
'right_gain': right_gain,
'left_delay': left_delay,
'right_delay': right_delay
}
# Usage
localizer = SoundLocalization()
azimuth = np.radians(45) # 45 degrees to the right
gains = localizer.compute_gains(azimuth, 0, 1000)
print(f"Left gain: {gains['left_gain']:.3f}, Right gain: {gains['right_gain']:.3f}")
print(f"Left delay: {gains['left_delay']*1000:.3f}ms, Right delay: {gains['right_delay']*1000:.3f}ms")👂 HRTF & Binaural Audio
🎯 Head-Related Transfer Function
HRTF describes how sound from a specific direction is filtered by the head, pinna, and torso before reaching the eardrum. It's the key to binaural audio — 3D sound over headphones.
HRTF Explained
Measurement: Record impulse responses in an anechoic chamber with microphones in ears. Dataset: CIPIC, IRCAM, MIT KEMAR. Personalization: HRTFs vary by person — generic HRTFs cause front-back confusion. Interpolation: Blend HRTFs for smooth movement. Rendering: Convolve audio with HRTF impulse responses.
🎧 Binaural📊 Impulse Response👤 Personalization🔄 Interpolation
# HRTF convolution for binaural rendering
import numpy as np
from scipy.signal import fftconvolve
class BinauralRenderer:
def __init__(self, hrtf_database):
self.hrtf_db = hrtf_database # dict: (azimuth, elevation) -> (left_ir, right_ir)
def render(self, audio, azimuth, elevation):
"""Render audio binaurally for given direction."""
# Get HRTF impulse responses
left_ir, right_ir = self.hrtf_db.get_closest(azimuth, elevation)
# Convolve audio with HRTFs
left_output = fftconvolve(audio, left_ir, mode='same')
right_output = fftconvolve(audio, right_ir, mode='same')
# Interleave for stereo output
stereo = np.column_stack((left_output, right_output))
return stereo
def render_moving(self, audio, positions, sample_rate):
"""Render audio for a moving source."""
output = np.zeros((len(audio), 2))
samples_per_position = len(audio) // len(positions)
for i, (azimuth, elevation) in enumerate(positions):
start = i * samples_per_position
end = start + samples_per_position
segment = audio[start:end]
rendered = self.render(segment, azimuth, elevation)
output[start:end] = rendered
return output
# HRTF interpolation for smooth movement
def interpolate_hrtf(hrtf1, hrtf2, t):
"""Interpolate between two HRTFs."""
left_ir = (1 - t) * hrtf1[0] + t * hrtf2[0]
right_ir = (1 - t) * hrtf1[1] + t * hrtf2[1]
return left_ir, right_ir
# Spherical HRTF interpolation
def spherical_interpolation(azimuth, elevation, hrtf_db):
"""Interpolate HRTF from nearby measured directions."""
# Find 4 nearest measured directions
neighbors = hrtf_db.find_neighbors(azimuth, elevation, k=4)
# Compute weights based on angular distance
weights = []
for (az, el) in neighbors:
dist = angular_distance(azimuth, elevation, az, el)
weights.append(1.0 / (dist + 1e-6))
# Normalize
total = sum(weights)
weights = [w / total for w in weights]
# Weighted sum of HRTFs
left_ir = sum(w * hrtf_db[(az, el)][0] for w, (az, el) in zip(weights, neighbors))
right_ir = sum(w * hrtf_db[(az, el)][1] for w, (az, el) in zip(weights, neighbors))
return left_ir, right_ir📍 Positional Audio
🎯 Sound from a point in 3D space
Positional audio places sound sources at 3D coordinates. As the listener moves, the audio updates: direction, distance, and occlusion change in real-time.
Positional Audio Parameters
Position: (x, y, z) in world space. Velocity: For Doppler effect. Spread: Point source vs area source. Directivity: Sound emitted in a cone. Min/max distance: Attenuation range. Rolloff: Linear, logarithmic, custom.
📍 3D Position🏃 Doppler📡 Directivity📉 Rolloff
// Positional audio in Unity
using UnityEngine;
public class PositionalAudio : MonoBehaviour {
public AudioSource audioSource;
public float minDistance = 1.0f;
public float maxDistance = 50.0f;
public AudioRolloffMode rolloffMode = AudioRolloffMode.Logarithmic;
void Start() {
audioSource.spatialBlend = 1.0f; // Full 3D
audioSource.minDistance = minDistance;
audioSource.maxDistance = maxDistance;
audioSource.rolloffMode = rolloffMode;
audioSource.dopplerLevel = 1.0f;
audioSource.spread = 0.0f; // Point source
}
// Update position (e.g., moving source)
void Update() {
// Position is set by transform
// Audio engine handles direction, distance, Doppler
}
}
// Custom rolloff curve
public class CustomRolloff : MonoBehaviour {
public AudioSource audioSource;
void Start() {
audioSource.rolloffMode = AudioRolloffMode.Custom;
AnimationCurve curve = new AnimationCurve();
// Volume vs normalized distance (0 = min, 1 = max)
curve.AddKey(0.0f, 1.0f); // full volume at min distance
curve.AddKey(0.2f, 0.8f); // slight drop
curve.AddKey(0.5f, 0.4f); // half
curve.AddKey(0.8f, 0.1f); // quiet
curve.AddKey(1.0f, 0.0f); // silent at max
audioSource.SetCustomCurve(AudioSourceCurveType.CustomRolloff, curve);
}
}
// Directivity (cone of sound)
public class DirectionalAudio : MonoBehaviour {
public AudioSource audioSource;
public float innerAngle = 30.0f; // full volume within this angle
public float outerAngle = 90.0f; // zero volume beyond this angle
void Start() {
// Set directivity
audioSource.spatialize = true;
// In Wwise/FMOD, use attenuation curves with directivity
}
}📏 Distance & Attenuation
🎯 How volume changes with distance
Sound attenuates with distance. Models: inverse square law (real physics), linear, logarithmic, and custom curves. Distance also affects frequency (air absorption) and reverb ratio.
Attenuation Models
Inverse square: 6dB drop per doubling of distance. Linear: Constant dB drop per meter. Logarithmic: Natural for perception. Air absorption: High frequencies attenuate faster. Near field: No attenuation within min distance. Far field: Silence beyond max distance.
📉 Inverse Square📏 Linear🎵 Logarithmic🌬️ Air Absorption
# Distance attenuation models
import numpy as np
class DistanceAttenuation:
def __init__(self, min_distance=1.0, max_distance=50.0):
self.min_distance = min_distance
self.max_distance = max_distance
def inverse_square(self, distance):
"""Real physics: 1/r^2."""
d = max(distance, self.min_distance)
if d > self.max_distance:
return 0.0
return (self.min_distance / d) ** 2
def linear(self, distance):
"""Linear rolloff."""
if distance <= self.min_distance:
return 1.0
if distance >= self.max_distance:
return 0.0
return 1.0 - (distance - self.min_distance) / (self.max_distance - self.min_distance)
def logarithmic(self, distance):
"""Logarithmic rolloff (natural)."""
d = max(distance, self.min_distance)
if d > self.max_distance:
return 0.0
return self.min_distance / d
def air_absorption(self, distance, frequency):
"""High frequencies attenuate faster (simplified)."""
# ISO 9613-1 model (simplified)
alpha = 0.001 * (frequency / 1000) ** 1.5 # dB/m
attenuation_db = alpha * distance
return 10 ** (-attenuation_db / 20)
def compute_gain(self, distance, frequency=1000):
"""Combined attenuation."""
geometric = self.inverse_square(distance)
air = self.air_absorption(distance, frequency)
return geometric * air
# Usage
atten = DistanceAttenuation(min_distance=1.0, max_distance=50.0)
for d in [0.5, 1, 2, 5, 10, 20, 50]:
gain = atten.compute_gain(d, frequency=1000)
gain_low = atten.compute_gain(d, frequency=100)
print(f"Distance {d}m: 1kHz gain={gain:.4f}, 100Hz gain={gain_low:.4f}")🧱 Occlusion & Obstruction
🎯 Sound behind walls
Occlusion simulates sound being blocked by obstacles. Occlusion = full blockage (muffled, quiet). Obstruction = partial blockage (diffraction around edges). Both use raycasting and low-pass filtering.
Occlusion Techniques
Raycasting: Cast ray from listener to source. Low-pass filter: Muffle high frequencies. Gain reduction: Lower volume. Diffraction: Sound bends around obstacles. Portals: Sound travels through openings. Multi-ray: Cast multiple rays for accuracy.
🎯 Raycast📉 Low-pass🔇 Gain🚪 Portals
// Occlusion in Unity
using UnityEngine;
public class AudioOcclusion : MonoBehaviour {
public AudioSource audioSource;
public Transform listener;
public LayerMask occlusionLayers;
public float occlusionMultiplier = 0.3f;
public float lowPassCutoff = 1000f;
private AudioLowPassFilter lowPassFilter;
void Start() {
lowPassFilter = gameObject.AddComponent();
lowPassFilter.cutoffFrequency = 22000f;
}
void Update() {
// Cast ray from listener to source
Vector3 direction = transform.position - listener.position;
float distance = direction.magnitude;
if (Physics.Raycast(listener.position, direction.normalized, out RaycastHit hit, distance, occlusionLayers)) {
// Occluded
float obstruction = 1.0f - (hit.distance / distance);
// Reduce volume
audioSource.volume = Mathf.Lerp(1.0f, occlusionMultiplier, obstruction);
// Muffle high frequencies
lowPassFilter.cutoffFrequency = Mathf.Lerp(22000f, lowPassCutoff, obstruction);
} else {
// Not occluded
audioSource.volume = 1.0f;
lowPassFilter.cutoffFrequency = 22000f;
}
}
}
// Multi-ray occlusion (more accurate)
public class MultiRayOcclusion : MonoBehaviour {
public AudioSource audioSource;
public Transform listener;
public int rayCount = 8;
public LayerMask occlusionLayers;
private AudioLowPassFilter lowPassFilter;
void Start() {
lowPassFilter = gameObject.AddComponent();
}
void Update() {
int blockedRays = 0;
for (int i = 0; i < rayCount; i++) {
// Cast rays in a small cone
Vector3 direction = (transform.position - listener.position).normalized;
float angle = (i / (float)rayCount) * 2 * Mathf.PI;
float radius = 0.2f;
Vector3 offset = new Vector3(
Mathf.Cos(angle) * radius,
Mathf.Sin(angle) * radius,
0
);
Vector3 rayDir = (direction + offset).normalized;
float distance = Vector3.Distance(listener.position, transform.position);
if (Physics.Raycast(listener.position, rayDir, distance, occlusionLayers)) {
blockedRays++;
}
}
float occlusion = blockedRays / (float)rayCount;
// Apply occlusion
audioSource.volume = Mathf.Lerp(1.0f, 0.2f, occlusion);
lowPassFilter.cutoffFrequency = Mathf.Lerp(22000f, 800f, occlusion);
}
} 🏛️ Reverb & Room Acoustics
🎯 The sound of space
Reverb simulates sound reflections in a space. It's the difference between a small room, a cathedral, and an outdoor space. Parameters: decay time, pre-delay, early reflections, diffusion, wet/dry mix.
Reverb Parameters
Decay time (RT60): Time for sound to drop 60dB. Pre-delay: Time before first reflection. Early reflections: First bounces from walls. Diffusion: Density of reflections. Wet/dry: Ratio of reverberant to direct sound. Reverb zones: Different rooms have different reverbs.
⏱️ RT60⏳ Pre-delay🌊 Diffusion🏠 Zones
// Reverb zones in Unity
using UnityEngine;
public class ReverbZones : MonoBehaviour {
public AudioReverbZone[] reverbZones;
public Transform listener;
void Update() {
// Find nearest reverb zone
float minDistance = float.MaxValue;
AudioReverbZone nearestZone = null;
foreach (var zone in reverbZones) {
float distance = Vector3.Distance(
listener.position,
zone.transform.position
);
if (distance < minDistance) {
minDistance = distance;
nearestZone = zone;
}
}
// Apply reverb
if (nearestZone != null) {
AudioReverbFilter reverbFilter = listener.GetComponent();
if (reverbFilter == null) {
reverbFilter = listener.gameObject.AddComponent();
}
reverbFilter.reverbPreset = nearestZone.reverbPreset;
reverbFilter.dryLevel = nearestZone.dryLevel;
reverbFilter.room = nearestZone.room;
reverbFilter.roomHF = nearestZone.roomHF;
reverbFilter.decayTime = nearestZone.decayTime;
reverbFilter.decayHFRatio = nearestZone.decayHFRatio;
}
}
}
// Custom reverb impulse response
public class CustomReverb : MonoBehaviour {
public AudioSource audioSource;
public AudioClip impulseResponse; // recorded or synthesized IR
void Start() {
// Create convolution reverb
AudioConvolutionReverb reverb = gameObject.AddComponent();
reverb.impulseResponse = impulseResponse;
reverb.wetMix = 0.5f;
reverb.dryMix = 0.5f;
}
}
// Procedural reverb (ray tracing)
public class ReverbRayTracer : MonoBehaviour {
public int numRays = 1000;
public int maxBounces = 10;
public LayerMask roomLayers;
public float[] ComputeReverbIR(Vector3 source, Vector3 listener) {
// Ray trace from source
float[] impulseResponse = new float[48000]; // 1 second at 48kHz
for (int i = 0; i < numRays; i++) {
Vector3 direction = Random.onUnitSphere;
Ray ray = new Ray(source, direction);
float amplitude = 1.0f;
float time = 0.0f;
for (int bounce = 0; bounce < maxBounces; bounce++) {
if (Physics.Raycast(ray, out RaycastHit hit, 100f, roomLayers)) {
// Check if ray reaches listener
Vector3 toListener = listener - hit.point;
if (Vector3.Distance(hit.point, listener) < 0.5f) {
int sampleIndex = (int)(time * 48000);
if (sampleIndex < impulseResponse.Length) {
impulseResponse[sampleIndex] += amplitude * 0.01f;
}
}
// Reflect
ray = new Ray(hit.point, Vector3.Reflect(ray.direction, hit.normal));
amplitude *= 0.9f; // absorption
time += hit.distance / 343.0f; // speed of sound
}
}
}
return impulseResponse;
}
} 🌐 Ambisonics
🎯 Full-sphere audio
Ambisonics is a full-sphere surround sound format. It represents the sound field as spherical harmonics. First-order (B-format: W, X, Y, Z) and higher-order (HOA) ambisonics provide increasingly precise spatial resolution.
Ambisonics Orders
1st order (FOA): 4 channels (W, X, Y, Z), low spatial resolution. 2nd order: 9 channels. 3rd order: 16 channels. Higher order (HOA): More channels, better localization. B-format: Standard for 1st order. AmbiX: Modern format (ACN/SN3D). Binaural decoding: Ambisonics → HRTF → headphones.
🌐 Spherical Harmonics📊 B-format🔊 HOA🎧 Binaural Decode
# Ambisonics encoding and decoding
import numpy as np
class Ambisonics:
def __init__(self, order=1):
self.order = order
self.num_channels = (order + 1) ** 2
def encode(self, audio, azimuth, elevation):
"""Encode mono audio to ambisonics B-format."""
# Spherical harmonics
if self.order == 1:
# First-order (AmbiX format: ACN order)
W = audio * 0.707 # omnidirectional
X = audio * np.cos(azimuth) * np.cos(elevation) # front-back
Y = audio * np.sin(azimuth) * np.cos(elevation) # left-right
Z = audio * np.sin(elevation) # up-down
return np.column_stack((W, Y, Z, X)) # ACN order
elif self.order == 2:
# Second-order (9 channels)
W = audio * 0.707
Y = audio * np.sin(azimuth) * np.cos(elevation)
Z = audio * np.sin(elevation)
X = audio * np.cos(azimuth) * np.cos(elevation)
# Second-order terms
V = audio * np.sin(2 * azimuth) * np.cos(elevation) ** 2
T = audio * np.sin(azimuth) * np.sin(2 * elevation)
R = audio * (3 * np.sin(elevation) ** 2 - 1) / 2
S = audio * np.cos(azimuth) * np.sin(2 * elevation)
U = audio * np.cos(2 * azimuth) * np.cos(elevation) ** 2
return np.column_stack((W, Y, Z, X, V, T, R, S, U))
return None
def decode_binaural(self, ambisonics, hrtf_db):
"""Decode ambisonics to binaural (headphones)."""
# Get HRTF for each ambisonic channel
# (simplified — real implementation uses virtual loudspeakers)
left = np.zeros(len(ambisonics))
right = np.zeros(len(ambisonics))
for i, channel in enumerate(ambisonics.T):
# Get HRTF for this channel's direction
left_ir, right_ir = hrtf_db.get_channel_hrtf(i)
left += np.convolve(channel, left_ir, mode='same')
right += np.convolve(channel, right_ir, mode='same')
return np.column_stack((left, right))
def rotate(self, ambisonics, yaw, pitch, roll):
"""Rotate the sound field (for head tracking)."""
# Rotation matrix in ambisonic domain
if self.order == 1:
# 4x4 rotation matrix
rotation_matrix = self.compute_rotation_matrix(yaw, pitch, roll)
rotated = ambisonics @ rotation_matrix.T
return rotated
return ambisonics
def compute_rotation_matrix(self, yaw, pitch, roll):
"""Compute 1st-order ambisonic rotation matrix."""
cy, sy = np.cos(yaw), np.sin(yaw)
cp, sp = np.cos(pitch), np.sin(pitch)
cr, sr = np.cos(roll), np.sin(roll)
# Combined rotation matrix
R = np.array([
[1, 0, 0, 0],
[0, cy*cp, cy*sp*sr - sy*cr, cy*sp*cr + sy*sr],
[0, sy*cp, sy*sp*sr + cy*cr, sy*sp*cr - cy*sr],
[0, -sp, cp*sr, cp*cr]
])
return R
# Usage
ambi = Ambisonics(order=1)
audio = np.random.randn(48000) # 1 second of audio
azimuth = np.radians(45)
elevation = np.radians(30)
encoded = ambi.encode(audio, azimuth, elevation)
print(f"Encoded shape: {encoded.shape}") # (48000, 4)
# Rotate for head tracking
yaw = np.radians(10)
rotated = ambi.rotate(encoded, yaw, 0, 0)
print(f"Rotated shape: {rotated.shape}")🎯 Object-Based Audio
🎯 Metadata-driven audio
Object-based audio treats each sound as an independent object with metadata (position, size, directivity). The renderer adapts to the playback system (headphones, speakers, soundbar). Standards: Dolby Atmos, DTS:X, MPEG-H.
Object vs Channel-Based
Channel-based: Fixed to speaker layout (5.1, 7.1). Object-based: Sound objects with metadata, rendered to any system. Metadata: Position, size, gain, directivity, divergence. Renderer: Maps objects to available speakers/headphones. Benefits: Scalable, immersive, interactive.
🎯 Objects📊 Metadata🔄 Adaptive🎬 Atmos
// Object-based audio with Wwise
// Sound object metadata
struct SoundObject {
Vector3 position;
float gain;
float spread; // 0 = point, 1 = omnidirectional
float directivity; // 0 = omnidirectional, 1 = focused
float maxDistance;
float attenuationCurve;
};
// Wwise: Set object position
AK::SoundEngine::SetObjectPosition(
gameObjectID,
position.x, position.y, position.z
);
// Set object spread
AkReal32 spread = 0.5f;
AK::SoundEngine::SetObjectAkAudioObjectID(
gameObjectID,
AK_INVALID_AUDIO_OBJECT_ID
);
AK::SoundEngine::SetObjectSpread(gameObjectID, spread);
// Set attenuation
AK::SoundEngine::SetGameObjectAuxSendValues(
gameObjectID,
auxSendValues,
numAuxSends
);
// FMOD: Object-based audio
FMOD::Studio::EventInstance* event;
FMOD_3D_ATTRIBUTES attributes;
attributes.position = { x, y, z };
attributes.velocity = { vx, vy, vz };
attributes.forward = { fx, fy, fz };
attributes.up = { ux, uy, uz };
event->set3DAttributes(&attributes);
// Dolby Atmos object metadata
// Each object has:
// - position (x, y, z)
// - size (spread)
// - gain
// - snap to speaker (optional)
// Renderer maps objects to 7.1.4 or binaural⚙️ Audio Engines (Wwise, FMOD)
🎯 Professional audio middleware
Wwise and FMOD are industry-standard audio middleware for games and XR. They provide advanced spatial audio: HRTF, occlusion, reverb zones, ambisonics, and object-based audio.
Wwise vs FMOD
Wwise: Industry leader, deep spatial audio, extensive profiling. FMOD: Easier learning curve, great for indie, strong Unity/Unreal integration. Both: Support HRTF, occlusion, reverb, ambisonics, object audio. Choosing: Wwise for AAA, FMOD for smaller teams.
🎮 Wwise🎵 FMOD🎧 HRTF🧱 Occlusion
// Wwise spatial audio setup
using UnityEngine;
using AK.Wwise;
public class WwiseSpatialAudio : MonoBehaviour {
public AkSpatialAudioListener listener;
public AkRoomPortal portal;
public AkRoom room;
void Start() {
// Set up spatial audio listener
listener.transform.position = Camera.main.transform.position;
// Create room
room.SetRoomTransform(transform);
room.SetRoomBox(10, 5, 10); // width, height, depth
// Create portal (door/window)
portal.SetPortalTransform(portalTransform);
portal.SetPortalBox(1, 2, 0.1f); // door size
// Set reverb per room
AkRoomParams roomParams = new AkRoomParams();
roomParams.ReverbAuxBus = reverbAuxBus;
roomParams.RoomTone = 0.5f;
room.SetRoomParams(roomParams);
}
void Update() {
// Update listener position
listener.transform.position = Camera.main.transform.position;
listener.transform.rotation = Camera.main.transform.rotation;
}
}
// FMOD spatial audio setup
using FMODUnity;
using FMOD.Studio;
public class FmodSpatialAudio : MonoBehaviour {
public StudioEventEmitter emitter;
public StudioListener listener;
void Start() {
// Configure 3D attributes
emitter.EventInstance.set3DAttributes(
RuntimeUtils.To3DAttributes(transform)
);
// Set occlusion
emitter.EventInstance.setParameterByName("Occlusion", 0.5f);
// Set reverb
emitter.EventInstance.setParameterByName("Reverb", 0.3f);
}
void Update() {
// Update 3D attributes
emitter.EventInstance.set3DAttributes(
RuntimeUtils.To3DAttributes(transform)
);
// Raycast for occlusion
if (Physics.Linecast(listener.transform.position, transform.position, out RaycastHit hit)) {
float occlusion = 1.0f - (hit.distance / Vector3.Distance(listener.transform.position, transform.position));
emitter.EventInstance.setParameterByName("Occlusion", occlusion);
}
}
}🎮 Unity & Unreal Audio
🎯 Built-in audio engines
Unity and Unreal have built-in spatial audio, but often integrate Wwise/FMOD for advanced features. Both support HRTF, occlusion, reverb, and ambisonics via plugins.
Unity & Unreal Audio Features
Unity: AudioSource, AudioListener, AudioMixer, Reverb Zones, Spatializer plugins (Oculus, Steam Audio, Resonance). Unreal: Audio Components, Attenuation, Reverb, Steam Audio, MetaSounds (procedural audio). XR: Both support HRTF, ambisonics, and object-based audio.
🎮 Unity Audio🎬 Unreal MetaSounds🎧 Steam Audio🌐 Resonance
// Unity: Steam Audio spatializer
using UnityEngine;
using SteamAudio;
public class SteamAudioSpatializer : MonoBehaviour {
public AudioSource audioSource;
public SteamAudioSource steamSource;
void Start() {
// Set up Steam Audio
steamSource = gameObject.AddComponent();
steamSource.occlusion = true;
steamSource.transmission = true;
steamSource.reflections = true;
steamSource.pathing = true;
// Set HRTF
audioSource.spatialize = true;
audioSource.spatializePostEffects = true;
}
}
// Unity: Oculus Spatializer
public class OculusSpatializer : MonoBehaviour {
public AudioSource audioSource;
void Start() {
// Enable Oculus spatializer
audioSource.spatialize = true;
audioSource.spatializePostEffects = true;
// Set spatializer plugin
// AudioSettings.SetSpatializerPluginName("OculusSpatializer");
}
}
// Unreal: MetaSounds
// MetaSounds is a node-based procedural audio system
// Example: Create a spatialized sound with distance attenuation
/*
[Source] -> [Attenuation] -> [Spatialization] -> [Output]
| | |
+-- [Oscillator] -- [Filter] -- [Reverb]
*/
// Unreal: Steam Audio
// Install Steam Audio plugin
// Add Steam Audio Source component
// Configure occlusion, reflections, pathing
// Unreal: Attenuation Settings
// In Sound Attenuation asset:
// - Distance Algorithm: Natural Sound
// - Attenuation Function: Logarithmic
// - Air Absorption: Enabled
// - Spatialization: HRTF (if plugin installed)
// - Occlusion: Enabled
// - Reverb: Send to reverb effect 🌐 Web Audio & WebXR
🎯 Spatial audio in the browser
The Web Audio API provides PannerNode for 3D audio, HRTF for binaural, and Ambisonics support. WebXR integrates with Web Audio for immersive browser experiences.
Web Audio Spatial Features
PannerNode: 3D positional audio. HRTF: Binaural rendering. Distance models: Linear, inverse, exponential. ConvolverNode: Reverb via impulse responses. Ambisonics: Via AmbisonicAudioNode or libraries. WebXR: Listener follows XR camera.
🌐 PannerNode🎧 HRTF🏛️ Convolver🥽 WebXR
// Web Audio API spatial audio
const audioContext = new AudioContext();
// Create listener
const listener = audioContext.listener;
listener.positionX.value = 0;
listener.positionY.value = 1.6;
listener.positionZ.value = 0;
listener.forwardX.value = 0;
listener.forwardY.value = 0;
listener.forwardZ.value = -1;
listener.upX.value = 0;
listener.upY.value = 1;
listener.upZ.value = 0;
// Create panner node (3D audio)
const panner = audioContext.createPanner();
panner.panningModel = 'HRTF'; // or 'equalpower'
panner.distanceModel = 'inverse';
panner.refDistance = 1;
panner.maxDistance = 100;
panner.rolloffFactor = 1;
panner.coneInnerAngle = 360;
panner.coneOuterAngle = 0;
panner.coneOuterGain = 0;
panner.positionX.value = 5;
panner.positionY.value = 0;
panner.positionZ.value = -3;
// Create audio source
const oscillator = audioContext.createOscillator();
oscillator.frequency.value = 440;
oscillator.connect(panner);
panner.connect(audioContext.destination);
oscillator.start();
// Update listener position (from WebXR camera)
function updateListener(xrCamera) {
const position = xrCamera.position;
const orientation = xrCamera.orientation;
listener.positionX.value = position.x;
listener.positionY.value = position.y;
listener.positionZ.value = position.z;
listener.forwardX.value = orientation.x;
listener.forwardY.value = orientation.y;
listener.forwardZ.value = orientation.z;
listener.upX.value = 0;
listener.upY.value = 1;
listener.upZ.value = 0;
}
// WebXR + Web Audio
async function startXRWithAudio() {
const session = await navigator.xr.requestSession('immersive-vr');
// Resume audio context (required for WebXR)
await audioContext.resume();
// In XR frame loop, update listener
function onXRFrame(time, frame) {
session.requestAnimationFrame(onXRFrame);
const pose = frame.getViewerPose(refSpace);
if (pose) {
const view = pose.views[0];
const transform = view.transform;
// Update audio listener
listener.positionX.value = transform.position.x;
listener.positionY.value = transform.position.y;
listener.positionZ.value = transform.position.z;
listener.forwardX.value = transform.orientation.x;
listener.forwardY.value = transform.orientation.y;
listener.forwardZ.value = transform.orientation.z;
}
}
session.requestAnimationFrame(onXRFrame);
}
// Convolution reverb (Web Audio)
async function createReverb(url) {
const response = await fetch(url);
const arrayBuffer = await response.arrayBuffer();
const impulseResponse = await audioContext.decodeAudioData(arrayBuffer);
const convolver = audioContext.createConvolver();
convolver.buffer = impulseResponse;
const wetGain = audioContext.createGain();
wetGain.gain.value = 0.3;
const dryGain = audioContext.createGain();
dryGain.gain.value = 0.7;
// Connect: source -> dry -> destination
// source -> convolver -> wet -> destination
return { convolver, wetGain, dryGain };
}🥽 Spatial Audio in AR/VR
🎯 Sound that completes presence
In AR/VR, spatial audio is critical for presence, localization, and immersion. It must be low-latency, head-tracked, and physically plausible.
XR Spatial Audio Requirements
Head tracking: Audio updates with head movement (low latency). HRTF: Binaural rendering for headphones. Ambisonics: For environmental audio. Occlusion: Realistic blocking by walls. Reverb: Matches virtual space. Object audio: Each sound is an object. Performance: Must run at 60+ fps.
🎧 HRTF🌐 Ambisonics🧱 Occlusion🏛️ Reverb
// VR spatial audio with head tracking (Unity + Oculus)
using UnityEngine;
using Oculus.Audio;
public class VRSpatialAudio : MonoBehaviour {
public AudioSource audioSource;
public OVRHeadsetEmulator headset;
void Start() {
// Set up Oculus spatializer
audioSource.spatialize = true;
audioSource.spatializePostEffects = true;
// Enable HRTF
OVRPlugin.SetAudioSpatialization(true);
// Set audio source to 3D
audioSource.spatialBlend = 1.0f;
}
void Update() {
// Listener automatically follows headset camera
// Oculus spatializer handles HRTF, distance, occlusion
}
}
// Ambisonic audio for VR environments
public class AmbisonicAudio : MonoBehaviour {
public AudioSource ambisonicSource;
public AudioClip ambisonicClip; // 4-channel (FOA) or 16-channel (HOA)
void Start() {
ambisonicSource.clip = ambisonicClip;
ambisonicSource.spatialBlend = 0.0f; // Ambisonics handled separately
// Decode ambisonics to binaural
// Use Oculus Ambisonics or Resonance Audio
}
}
// Resonance Audio (Google) for VR
using Google.ResonanceAudio;
public class ResonanceAudioSpatial : MonoBehaviour {
public ResonanceAudioSource resonanceSource;
void Start() {
resonanceSource = gameObject.AddComponent();
resonanceSource.occlusionEnabled = true;
resonanceSource.directivityEnabled = true;
// Set room properties
ResonanceAudioRoom room = FindObjectOfType();
room.roomProperties = new RoomProperties {
material = RoomMaterial.Plaster,
reverbGain = 0.5f,
reverbTime = 1.5f
};
}
}
// VR audio performance tips:
// 1. Limit active audio sources (e.g., 32 max)
// 2. Use audio LOD (simpler processing at distance)
// 3. Bake reverb where possible
// 4. Use compressed audio formats
// 5. Profile audio CPU usage 🚀 Future: Neural Audio
🎯 AI-powered spatial audio
Neural networks are transforming spatial audio: neural HRTF, learned reverb, generative audio, real-time ray tracing, and personalized HRTF from photos.
Neural Audio Advances
Neural HRTF: Predict personalized HRTF from ear photos. Neural reverb: Generate impulse responses for any space. Generative: Create 3D audio from text/video. Real-time ray tracing: GPU-accelerated acoustic simulation. Source separation: Isolate and spatialize individual sounds. Neural upmixing: Convert stereo to spatial audio.
🧠 Neural HRTF🏛️ Neural Reverb🎨 Generative⚡ Ray Tracing
# Neural HRTF personalization
import torch
import torch.nn as nn
class NeuralHRTF(nn.Module):
"""Predict personalized HRTF from ear images."""
def __init__(self):
super().__init__()
# Ear image encoder
self.encoder = nn.Sequential(
nn.Conv2d(3, 32, 3, padding=1), nn.ReLU(), nn.MaxPool2d(2),
nn.Conv2d(32, 64, 3, padding=1), nn.ReLU(), nn.MaxPool2d(2),
nn.Conv2d(64, 128, 3, padding=1), nn.ReLU(), nn.MaxPool2d(2),
nn.Flatten()
)
# HRTF decoder
self.decoder = nn.Sequential(
nn.Linear(128 * 16 * 16 + 2, 512), nn.ReLU(),
nn.Linear(512, 512), nn.ReLU(),
nn.Linear(512, 256 * 2) # 256 samples x 2 ears
)
def forward(self, ear_image, direction):
# ear_image: (B, 3, H, W)
# direction: (B, 2) [azimuth, elevation]
features = self.encoder(ear_image)
combined = torch.cat([features, direction], dim=1)
hrtf = self.decoder(combined)
return hrtf.view(-1, 2, 256) # (B, 2 ears, 256 samples)
# Training
model = NeuralHRTF()
optimizer = torch.optim.Adam(model.parameters(), lr=1e-3)
for ear_img, direction, hrtf_gt in dataloader:
hrtf_pred = model(ear_img, direction)
loss = nn.MSELoss()(hrtf_pred, hrtf_gt)
optimizer.zero_grad()
loss.backward()
optimizer.step()
# Neural reverb (generative)
class NeuralReverb(nn.Module):
"""Generate impulse response for a given room."""
def __init__(self):
super().__init__()
self.network = nn.Sequential(
nn.Linear(10, 256), nn.ReLU(), # room params (size, material, etc.)
nn.Linear(256, 512), nn.ReLU(),
nn.Linear(512, 512), nn.ReLU(),
nn.Linear(512, 48000) # 1 second IR at 48kHz
)
def forward(self, room_params):
return self.network(room_params)
# Real-time acoustic ray tracing (GPU)
# Use NVIDIA RTX or AMD Ray Accelerators
# Trace thousands of rays per frame
# Compute impulse response in real-time
# Convolve with audio