abanoub-ashraf/ios-visionos-ar

visionOS & AR Expert

You are an expert in visionOS and AR development. When this skill activates, help create immersive spatial experiences.

visionOS Fundamentals

App Structure

import SwiftUI

@main
struct MyVisionApp: App {
    var body: some Scene {
        // Standard window
        WindowGroup {
            ContentView()
        }

        // Volumetric window (3D content in bounded space)
        WindowGroup(id: "volumetric") {
            VolumetricView()
        }
        .windowStyle(.volumetric)
        .defaultSize(width: 0.5, height: 0.5, depth: 0.5, in: .meters)

        // Full immersive space
        ImmersiveSpace(id: "immersive") {
            ImmersiveView()
        }
        .immersionStyle(selection: .constant(.mixed), in: .mixed, .full)
    }
}

Opening Spaces

struct ContentView: View {
    @Environment(\.openImmersiveSpace) var openImmersiveSpace
    @Environment(\.dismissImmersiveSpace) var dismissImmersiveSpace
    @Environment(\.openWindow) var openWindow

    var body: some View {
        VStack {
            Button("Enter Immersive") {
                Task {
                    await openImmersiveSpace(id: "immersive")
                }
            }

            Button("Open Volumetric") {
                openWindow(id: "volumetric")
            }
        }
    }
}

RealityKit Basics

RealityView

import RealityKit

struct ModelView: View {
    var body: some View {
        RealityView { content in
            // Create entity
            let sphere = MeshResource.generateSphere(radius: 0.1)
            let material = SimpleMaterial(color: .blue, isMetallic: true)
            let entity = ModelEntity(mesh: sphere, materials: [material])

            // Position in space
            entity.position = [0, 1, -1]  // x, y, z in meters

            // Add to scene
            content.add(entity)
        }
    }
}

Loading 3D Models

struct Model3DView: View {
    var body: some View {
        RealityView { content in
            // Load from bundle
            if let model = try? await ModelEntity(named: "Robot") {
                model.scale = [0.5, 0.5, 0.5]
                content.add(model)
            }
        }

        // Or use Model3D for simple display
        Model3D(named: "Robot") { model in
            model
                .resizable()
                .scaledToFit()
        } placeholder: {
            ProgressView()
        }
    }
}

Updating RealityView

struct InteractiveView: View {
    @State private var scale: Float = 1.0

    var body: some View {
        RealityView { content in
            let box = MeshResource.generateBox(size: 0.2)
            let material = SimpleMaterial(color: .green, isMetallic: false)
            let entity = ModelEntity(mesh: box, materials: [material])
            entity.name = "myBox"
            content.add(entity)
        } update: { content in
            // Called when state changes
            if let box = content.entities.first(where: { $0.name == "myBox" }) {
                box.scale = [scale, scale, scale]
            }
        }

        Slider(value: $scale, in: 0.5...2.0)
    }
}

Spatial Interactions

Gestures on Entities

struct TappableModelView: View {
    @State private var tapped = false

    var body: some View {
        RealityView { content in
            let sphere = MeshResource.generateSphere(radius: 0.1)
            let material = SimpleMaterial(color: tapped ? .red : .blue, isMetallic: true)
            let entity = ModelEntity(mesh: sphere, materials: [material])

            // Enable input
            entity.components.set(InputTargetComponent())
            entity.components.set(CollisionComponent(shapes: [.generateSphere(radius: 0.1)]))

            content.add(entity)
        } update: { content in
            if let sphere = content.entities.first as? ModelEntity {
                sphere.model?.materials = [SimpleMaterial(color: tapped ? .red : .blue, isMetallic: true)]
            }
        }
        .gesture(TapGesture().targetedToAnyEntity().onEnded { _ in
            tapped.toggle()
        })
    }
}

Drag Gesture

struct DraggableModelView: View {
    @State private var position: SIMD3<Float> = [0, 1, -1]

    var body: some View {
        RealityView { content in
            let box = MeshResource.generateBox(size: 0.2)
            let entity = ModelEntity(mesh: box, materials: [SimpleMaterial(color: .purple, isMetallic: false)])
            entity.name = "draggable"
            entity.position = position
            entity.components.set(InputTargetComponent())
            entity.components.set(CollisionComponent(shapes: [.generateBox(size: [0.2, 0.2, 0.2])]))
            content.add(entity)
        } update: { content in
            if let entity = content.entities.first(where: { $0.name == "draggable" }) {
                entity.position = position
            }
        }
        .gesture(
            DragGesture()
                .targetedToAnyEntity()
                .onChanged { value in
                    let translation = value.convert(value.translation3D, from: .local, to: .scene)
                    position = [
                        Float(translation.x),
                        Float(translation.y) + 1,
                        Float(translation.z) - 1
                    ]
                }
        )
    }
}

Hover Effects

struct HoverableView: View {
    var body: some View {
        RealityView { content in
            let entity = ModelEntity(
                mesh: .generateSphere(radius: 0.1),
                materials: [SimpleMaterial(color: .cyan, isMetallic: true)]
            )

            // Add hover effect
            entity.components.set(HoverEffectComponent())
            entity.components.set(InputTargetComponent())
            entity.components.set(CollisionComponent(shapes: [.generateSphere(radius: 0.1)]))

            content.add(entity)
        }
    }
}

Immersive Experiences

Mixed Immersion

struct MixedImmersiveView: View {
    var body: some View {
        RealityView { content in
            // Virtual content blends with real world
            let anchor = AnchorEntity(.plane(.horizontal, classification: .floor, minimumBounds: [0.5, 0.5]))

            let robot = try? await ModelEntity(named: "Robot")
            robot?.position = [0, 0, 0]

            if let robot {
                anchor.addChild(robot)
            }

            content.add(anchor)
        }
    }
}

Full Immersion

struct FullImmersiveView: View {
    @Environment(\.dismissImmersiveSpace) var dismiss

    var body: some View {
        RealityView { content in
            // Skybox
            guard let skyboxResource = try? await EnvironmentResource(named: "Starfield") else { return }
            let skybox = Entity()
            skybox.components.set(ImageBasedLightComponent(source: .single(skyboxResource)))
            skybox.components.set(ImageBasedLightReceiverComponent(imageBasedLight: skybox))
            content.add(skybox)

            // 3D content
            let sphere = ModelEntity(
                mesh: .generateSphere(radius: 1),
                materials: [SimpleMaterial(color: .blue, isMetallic: true)]
            )
            sphere.position = [0, 1.5, -3]
            content.add(sphere)
        }
        .gesture(TapGesture().onEnded {
            Task {
                await dismiss()
            }
        })
    }
}

ARKit Integration (iOS)

AR Session Setup

import ARKit
import RealityKit

struct ARContentView: View {
    var body: some View {
        ARViewContainer()
            .ignoresSafeArea()
    }
}

struct ARViewContainer: UIViewRepresentable {
    func makeUIView(context: Context) -> ARView {
        let arView = ARView(frame: .zero)

        // Configure session
        let config = ARWorldTrackingConfiguration()
        config.planeDetection = [.horizontal, .vertical]
        config.environmentTexturing = .automatic

        // Enable scene understanding (LiDAR)
        if ARWorldTrackingConfiguration.supportsSceneReconstruction(.mesh) {
            config.sceneReconstruction = .mesh
        }

        arView.session.run(config)

        return arView
    }

    func updateUIView(_ uiView: ARView, context: Context) {}
}

Placing Objects on Planes

class ARViewController: UIViewController {
    var arView: ARView!

    override func viewDidLoad() {
        super.viewDidLoad()

        arView = ARView(frame: view.bounds)
        view.addSubview(arView)

        // Configure AR
        let config = ARWorldTrackingConfiguration()
        config.planeDetection = [.horizontal]
        arView.session.run(config)

        // Add tap gesture
        let tapGesture = UITapGestureRecognizer(target: self, action: #selector(handleTap))
        arView.addGestureRecognizer(tapGesture)
    }

    @objc func handleTap(_ gesture: UITapGestureRecognizer) {
        let location = gesture.location(in: arView)

        // Raycast to find surface
        if let result = arView.raycast(from: location, allowing: .estimatedPlane, alignment: .horizontal).first {
            // Create anchor
            let anchor = AnchorEntity(world: result.worldTransform)

            // Create model
            let box = MeshResource.generateBox(size: 0.1)
            let material = SimpleMaterial(color: .blue, isMetallic: true)
            let model = ModelEntity(mesh: box, materials: [material])

            anchor.addChild(model)
            arView.scene.addAnchor(anchor)
        }
    }
}

Face Tracking

func setupFaceTracking() {
    guard ARFaceTrackingConfiguration.isSupported else { return }

    let config = ARFaceTrackingConfiguration()
    config.maximumNumberOfTrackedFaces = 1

    arView.session.run(config)
    arView.session.delegate = self
}

extension ARViewController: ARSessionDelegate {
    func session(_ session: ARSession, didUpdate anchors: [ARAnchor]) {
        for anchor in anchors {
            if let faceAnchor = anchor as? ARFaceAnchor {
                // Access blend shapes for expressions
                let smile = faceAnchor.blendShapes[.mouthSmileLeft]?.floatValue ?? 0
                let eyeBlink = faceAnchor.blendShapes[.eyeBlinkLeft]?.floatValue ?? 0

                // Update face mesh
                updateFaceGeometry(with: faceAnchor)
            }
        }
    }
}

Image Tracking

func setupImageTracking() {
    guard let referenceImages = ARReferenceImage.referenceImages(
        inGroupNamed: "AR Resources",
        bundle: nil
    ) else { return }

    let config = ARWorldTrackingConfiguration()
    config.detectionImages = referenceImages
    config.maximumNumberOfTrackedImages = 4

    arView.session.run(config)
}

func session(_ session: ARSession, didAdd anchors: [ARAnchor]) {
    for anchor in anchors {
        if let imageAnchor = anchor as? ARImageAnchor {
            let imageName = imageAnchor.referenceImage.name ?? ""

            // Place content on detected image
            let anchorEntity = AnchorEntity(anchor: imageAnchor)
            let content = createContentForImage(named: imageName)
            anchorEntity.addChild(content)
            arView.scene.addAnchor(anchorEntity)
        }
    }
}

Entity Component System

Custom Components

import RealityKit

// Define component
struct SpinComponent: Component {
    var speed: Float = 1.0
    var axis: SIMD3<Float> = [0, 1, 0]
}

// Register component
extension SpinComponent {
    static func registerComponent() {
        SpinComponent.registerComponent()
    }
}

// Create system
class SpinSystem: System {
    static let query = EntityQuery(where: .has(SpinComponent.self))

    required init(scene: Scene) {}

    func update(context: SceneUpdateContext) {
        for entity in context.entities(matching: Self.query, updatingSystemWhen: .rendering) {
            guard let spin = entity.components[SpinComponent.self] else { continue }

            let rotation = simd_quatf(angle: spin.speed * Float(context.deltaTime), axis: spin.axis)
            entity.orientation *= rotation
        }
    }
}

// Usage
struct SpinningModelView: View {
    var body: some View {
        RealityView { content in
            SpinComponent.registerComponent()
            SpinSystem.registerSystem()

            let sphere = ModelEntity(
                mesh: .generateSphere(radius: 0.1),
                materials: [SimpleMaterial(color: .orange, isMetallic: true)]
            )

            sphere.components.set(SpinComponent(speed: 2.0))
            content.add(sphere)
        }
    }
}

Animations

Transform Animations

func animateEntity(_ entity: Entity) {
    // Move animation
    var transform = entity.transform
    transform.translation = [0, 2, -1]

    entity.move(
        to: transform,
        relativeTo: nil,
        duration: 2.0,
        timingFunction: .easeInOut
    )
}

// Animation sequence
func animateSequence(_ entity: Entity) async {
    // Move up
    var upTransform = entity.transform
    upTransform.translation.y += 0.5
    entity.move(to: upTransform, relativeTo: nil, duration: 1.0)

    try? await Task.sleep(for: .seconds(1))

    // Rotate
    var rotatedTransform = entity.transform
    rotatedTransform.rotation = simd_quatf(angle: .pi, axis: [0, 1, 0])
    entity.move(to: rotatedTransform, relativeTo: nil, duration: 0.5)
}

USDZ Animations

RealityView { content in
    if let model = try? await Entity(named: "AnimatedCharacter") {
        // Find animation
        if let animation = model.availableAnimations.first {
            model.playAnimation(animation.repeat())
        }
        content.add(model)
    }
}

Spatial Audio

Adding Audio

RealityView { content in
    let audioEntity = Entity()
    audioEntity.position = [0, 1, -2]

    // Spatial audio
    guard let audioResource = try? await AudioFileResource(named: "ambient.mp3") else { return }

    let audioController = audioEntity.prepareAudio(audioResource)
    audioController.gain = -10  // decibels
    audioController.play()

    content.add(audioEntity)
}

Attachments (visionOS)

SwiftUI in 3D Space

struct AttachmentView: View {
    var body: some View {
        RealityView { content, attachments in
            let sphere = ModelEntity(
                mesh: .generateSphere(radius: 0.1),
                materials: [SimpleMaterial(color: .blue, isMetallic: true)]
            )
            sphere.position = [0, 1.5, -1]
            content.add(sphere)

            // Add SwiftUI attachment
            if let label = attachments.entity(for: "infoLabel") {
                label.position = [0, 1.8, -1]
                content.add(label)
            }
        } attachments: {
            Attachment(id: "infoLabel") {
                VStack {
                    Text("Hello, visionOS!")
                        .font(.title)
                    Text("This is a SwiftUI attachment")
                        .font(.caption)
                }
                .padding()
                .glassBackgroundEffect()
            }
        }
    }
}

Best Practices

✅ Do

// Use lightweight meshes
let lowPolyMesh = MeshResource.generateSphere(radius: 0.1)  // Default segments

// Batch similar entities
let material = SimpleMaterial(color: .blue, isMetallic: true)  // Reuse

// Check device capabilities
if ARWorldTrackingConfiguration.supportsSceneReconstruction(.mesh) {
    config.sceneReconstruction = .mesh
}

// Provide fallbacks
#if os(visionOS)
    ImmersiveView()
#else
    ARViewContainer()
#endif

❌ Don't

// Don't create high-poly meshes unnecessarily
let mesh = MeshResource.generateSphere(radius: 0.1)
// Don't use: segments: 100 unless needed

// Don't load models synchronously
let model = try! Entity.load(named: "Heavy")  // ❌ Blocks main thread

// Don't ignore memory
// Clear unused entities and textures

// Don't skip collision shapes for interactive entities
// They're required for gestures