abanoub-ashraf/ios-coreml-ai

iOS Core ML & AI Expert

You are an expert in on-device machine learning for iOS. When this skill activates, help implement AI features efficiently.

Core ML Fundamentals

Loading Models

import CoreML

class ImageClassifier {
    private let model: VNCoreMLModel

    init() throws {
        // Load compiled model
        let config = MLModelConfiguration()
        config.computeUnits = .all  // Use Neural Engine when available

        let mlModel = try MobileNetV2(configuration: config).model
        model = try VNCoreMLModel(for: mlModel)
    }

    // Async loading for large models
    static func loadAsync() async throws -> ImageClassifier {
        try await Task.detached(priority: .userInitiated) {
            try ImageClassifier()
        }.value
    }
}

Model Configuration

let config = MLModelConfiguration()

// Compute units
config.computeUnits = .all           // CPU + GPU + Neural Engine
config.computeUnits = .cpuAndGPU     // No Neural Engine
config.computeUnits = .cpuOnly       // CPU only
config.computeUnits = .cpuAndNeuralEngine

// Allow low precision
config.allowLowPrecisionAccumulationOnGPU = true

// Model loading
let model = try MyModel(configuration: config)

Vision Framework

Image Classification

import Vision
import CoreML

class ImageClassificationService {
    private var classificationRequest: VNCoreMLRequest?

    init() throws {
        let model = try VNCoreMLModel(for: MobileNetV2().model)

        classificationRequest = VNCoreMLRequest(model: model) { request, error in
            // Handle results in completion
        }

        classificationRequest?.imageCropAndScaleOption = .centerCrop
    }

    func classify(image: CGImage) async throws -> [Classification] {
        guard let request = classificationRequest else {
            throw MLError.modelNotLoaded
        }

        return try await withCheckedThrowingContinuation { continuation in
            let handler = VNImageRequestHandler(cgImage: image, options: [:])

            do {
                try handler.perform([request])

                guard let results = request.results as? [VNClassificationObservation] else {
                    continuation.resume(throwing: MLError.invalidResults)
                    return
                }

                let classifications = results
                    .filter { $0.confidence > 0.1 }
                    .map { Classification(label: $0.identifier, confidence: $0.confidence) }

                continuation.resume(returning: classifications)
            } catch {
                continuation.resume(throwing: error)
            }
        }
    }
}

struct Classification {
    let label: String
    let confidence: Float
}

Object Detection

class ObjectDetector {
    private let model: VNCoreMLModel

    init() throws {
        let mlModel = try YOLOv3(configuration: MLModelConfiguration()).model
        model = try VNCoreMLModel(for: mlModel)
    }

    func detect(in image: CGImage) async throws -> [DetectedObject] {
        let request = VNCoreMLRequest(model: model)
        request.imageCropAndScaleOption = .scaleFill

        let handler = VNImageRequestHandler(cgImage: image, options: [:])
        try handler.perform([request])

        guard let results = request.results as? [VNRecognizedObjectObservation] else {
            return []
        }

        return results.map { observation in
            DetectedObject(
                label: observation.labels.first?.identifier ?? "Unknown",
                confidence: observation.labels.first?.confidence ?? 0,
                boundingBox: observation.boundingBox
            )
        }
    }
}

struct DetectedObject {
    let label: String
    let confidence: Float
    let boundingBox: CGRect  // Normalized coordinates (0-1)
}

Face Detection & Analysis

class FaceAnalyzer {
    func detectFaces(in image: CGImage) async throws -> [FaceObservation] {
        let request = VNDetectFaceLandmarksRequest()

        let handler = VNImageRequestHandler(cgImage: image, options: [:])
        try handler.perform([request])

        guard let results = request.results else { return [] }

        return results.map { face in
            FaceObservation(
                boundingBox: face.boundingBox,
                landmarks: face.landmarks,
                roll: face.roll?.doubleValue,
                yaw: face.yaw?.doubleValue
            )
        }
    }

    // Face quality and attributes
    func analyzeFaceQuality(in image: CGImage) async throws -> [FaceQuality] {
        let request = VNDetectFaceCaptureQualityRequest()

        let handler = VNImageRequestHandler(cgImage: image, options: [:])
        try handler.perform([request])

        guard let results = request.results else { return [] }

        return results.map { face in
            FaceQuality(
                boundingBox: face.boundingBox,
                quality: face.faceCaptureQuality ?? 0
            )
        }
    }
}

Text Recognition (OCR)

class TextRecognizer {
    func recognizeText(in image: CGImage) async throws -> [RecognizedText] {
        let request = VNRecognizeTextRequest()
        request.recognitionLevel = .accurate  // or .fast
        request.recognitionLanguages = ["en-US", "ar"]
        request.usesLanguageCorrection = true

        let handler = VNImageRequestHandler(cgImage: image, options: [:])
        try handler.perform([request])

        guard let results = request.results else { return [] }

        return results.compactMap { observation in
            guard let text = observation.topCandidates(1).first else { return nil }
            return RecognizedText(
                text: text.string,
                confidence: text.confidence,
                boundingBox: observation.boundingBox
            )
        }
    }

    // Document scanning
    func scanDocument(in image: CGImage) async throws -> CGImage? {
        let request = VNDetectDocumentSegmentationRequest()

        let handler = VNImageRequestHandler(cgImage: image, options: [:])
        try handler.perform([request])

        guard let result = request.results?.first,
              let documentImage = perspectiveCorrectedImage(
                from: image,
                using: result
              ) else {
            return nil
        }

        return documentImage
    }
}

Barcode & QR Detection

func detectBarcodes(in image: CGImage) async throws -> [Barcode] {
    let request = VNDetectBarcodesRequest()
    request.symbologies = [.qr, .ean13, .code128]

    let handler = VNImageRequestHandler(cgImage: image, options: [:])
    try handler.perform([request])

    guard let results = request.results else { return [] }

    return results.compactMap { observation in
        Barcode(
            payload: observation.payloadStringValue,
            symbology: observation.symbology,
            boundingBox: observation.boundingBox
        )
    }
}

Natural Language

Text Analysis

import NaturalLanguage

class TextAnalyzer {
    // Sentiment Analysis
    func analyzeSentiment(_ text: String) -> Double {
        let tagger = NLTagger(tagSchemes: [.sentimentScore])
        tagger.string = text

        let (sentiment, _) = tagger.tag(
            at: text.startIndex,
            unit: .paragraph,
            scheme: .sentimentScore
        )

        return Double(sentiment?.rawValue ?? "0") ?? 0
    }

    // Language Detection
    func detectLanguage(_ text: String) -> NLLanguage? {
        let recognizer = NLLanguageRecognizer()
        recognizer.processString(text)
        return recognizer.dominantLanguage
    }

    // Named Entity Recognition
    func extractEntities(_ text: String) -> [Entity] {
        let tagger = NLTagger(tagSchemes: [.nameType])
        tagger.string = text

        var entities: [Entity] = []

        tagger.enumerateTags(
            in: text.startIndex..<text.endIndex,
            unit: .word,
            scheme: .nameType,
            options: [.omitWhitespace, .omitPunctuation, .joinNames]
        ) { tag, range in
            if let tag = tag {
                entities.append(Entity(
                    text: String(text[range]),
                    type: tag
                ))
            }
            return true
        }

        return entities
    }

    // Tokenization
    func tokenize(_ text: String) -> [String] {
        let tokenizer = NLTokenizer(unit: .word)
        tokenizer.string = text

        return tokenizer.tokens(for: text.startIndex..<text.endIndex).map {
            String(text[$0])
        }
    }
}

struct Entity {
    let text: String
    let type: NLTag  // .personalName, .placeName, .organizationName
}

Text Embedding

func findSimilarTexts(query: String, in corpus: [String]) -> [(String, Double)] {
    guard let embedding = NLEmbedding.wordEmbedding(for: .english) else {
        return []
    }

    // Get query vector
    guard let queryVector = embedding.vector(for: query) else {
        return []
    }

    // Compare with corpus
    return corpus.compactMap { text in
        guard let textVector = embedding.vector(for: text) else { return nil }
        let distance = embedding.distance(between: query, and: text)
        return (text, 1.0 - distance)  // Convert distance to similarity
    }
    .sorted { $0.1 > $1.1 }
}

Create ML

Training Image Classifier

import CreateML

func trainImageClassifier() async throws -> MLModel {
    // Load training data
    let trainingData = MLImageClassifier.DataSource.labeledDirectories(
        at: URL(fileURLWithPath: "/path/to/training/data")
    )

    // Configure training
    let parameters = MLImageClassifier.ModelParameters(
        featureExtractor: .scenePrint(revision: 2),
        validationData: nil,
        maxIterations: 25,
        augmentationOptions: [.crop, .rotation, .blur]
    )

    // Train
    let classifier = try MLImageClassifier(
        trainingData: trainingData,
        parameters: parameters
    )

    // Evaluate
    let evaluation = classifier.evaluation(on: trainingData)
    print("Training accuracy: \(evaluation.classificationError)")

    // Save
    try classifier.write(to: URL(fileURLWithPath: "MyClassifier.mlmodel"))

    return classifier.model
}

Training Text Classifier

func trainTextClassifier() async throws -> MLModel {
    // Prepare data
    let data = try MLDataTable(contentsOf: URL(fileURLWithPath: "training.json"))

    // Train
    let classifier = try MLTextClassifier(
        trainingData: data,
        textColumn: "text",
        labelColumn: "label"
    )

    // Test
    let prediction = try classifier.prediction(from: "This is great!")
    print("Predicted: \(prediction)")

    // Save
    try classifier.write(to: URL(fileURLWithPath: "TextClassifier.mlmodel"))

    return classifier.model
}

Real-Time Processing

Camera + ML Pipeline

import AVFoundation
import Vision

class CameraMLProcessor: NSObject {
    private var captureSession: AVCaptureSession?
    private var classificationRequest: VNCoreMLRequest?

    var onClassification: (([Classification]) -> Void)?

    func setup() throws {
        // Setup model
        let model = try VNCoreMLModel(for: MobileNetV2().model)
        classificationRequest = VNCoreMLRequest(model: model) { [weak self] request, _ in
            self?.processClassifications(request)
        }

        // Setup camera
        captureSession = AVCaptureSession()
        captureSession?.sessionPreset = .high

        guard let camera = AVCaptureDevice.default(.builtInWideAngleCamera, for: .video, position: .back),
              let input = try? AVCaptureDeviceInput(device: camera) else {
            throw CameraError.noCameraAvailable
        }

        captureSession?.addInput(input)

        // Add video output
        let videoOutput = AVCaptureVideoDataOutput()
        videoOutput.setSampleBufferDelegate(self, queue: DispatchQueue(label: "ml.processing"))
        captureSession?.addOutput(videoOutput)
    }

    private func processClassifications(_ request: VNRequest) {
        guard let results = request.results as? [VNClassificationObservation] else { return }

        let classifications = results
            .prefix(5)
            .map { Classification(label: $0.identifier, confidence: $0.confidence) }

        DispatchQueue.main.async {
            self.onClassification?(classifications)
        }
    }
}

extension CameraMLProcessor: AVCaptureVideoDataOutputSampleBufferDelegate {
    func captureOutput(_ output: AVCaptureOutput, didOutput sampleBuffer: CMSampleBuffer, from connection: AVCaptureConnection) {
        guard let pixelBuffer = CMSampleBufferGetImageBuffer(sampleBuffer),
              let request = classificationRequest else { return }

        let handler = VNImageRequestHandler(cvPixelBuffer: pixelBuffer, options: [:])
        try? handler.perform([request])
    }
}

SwiftUI Integration

import SwiftUI

struct MLCameraView: View {
    @StateObject private var viewModel = MLCameraViewModel()

    var body: some View {
        ZStack {
            CameraPreviewView(session: viewModel.captureSession)
                .ignoresSafeArea()

            VStack {
                Spacer()

                // Results overlay
                VStack(alignment: .leading, spacing: 4) {
                    ForEach(viewModel.classifications, id: \.label) { classification in
                        HStack {
                            Text(classification.label)
                            Spacer()
                            Text("\(Int(classification.confidence * 100))%")
                        }
                        .font(.caption)
                    }
                }
                .padding()
                .background(.ultraThinMaterial)
                .cornerRadius(12)
                .padding()
            }
        }
        .onAppear {
            viewModel.start()
        }
        .onDisappear {
            viewModel.stop()
        }
    }
}

@MainActor
@Observable
class MLCameraViewModel {
    var classifications: [Classification] = []
    let captureSession: AVCaptureSession

    private let processor: CameraMLProcessor

    init() {
        processor = CameraMLProcessor()
        captureSession = processor.captureSession ?? AVCaptureSession()

        processor.onClassification = { [weak self] results in
            self?.classifications = results
        }
    }

    func start() {
        DispatchQueue.global(qos: .userInitiated).async {
            self.captureSession.startRunning()
        }
    }

    func stop() {
        captureSession.stopRunning()
    }
}

Model Optimization

Quantization

// In Create ML or using coremltools (Python)

// Swift: Load with low precision
let config = MLModelConfiguration()
config.allowLowPrecisionAccumulationOnGPU = true

// Python: Quantize model
// import coremltools as ct
// model = ct.models.MLModel("model.mlmodel")
// quantized = ct.models.neural_network.quantization_utils.quantize_weights(model, nbits=16)
// quantized.save("model_quantized.mlmodel")

Model Compilation

// Compile at build time (recommended)
// Add .mlmodel to Xcode project

// Runtime compilation (if needed)
func compileModel(at url: URL) throws -> MLModel {
    let compiledURL = try MLModel.compileModel(at: url)
    return try MLModel(contentsOf: compiledURL)
}

On-Device Training

Updatable Models (iOS 13+)

// Define updatable model in Create ML
// Then update on device:

func updateModel(with samples: [TrainingSample]) async throws {
    let trainingData = try prepareTrainingData(samples)

    let updateTask = try MLUpdateTask(
        forModelAt: modelURL,
        trainingData: trainingData,
        configuration: nil,
        completionHandler: { context in
            // Save updated model
            let updatedModel = context.model
            try? updatedModel.write(to: self.updatedModelURL)
        }
    )

    updateTask.resume()
}

Error Handling

enum MLError: LocalizedError {
    case modelNotLoaded
    case invalidImage
    case invalidResults
    case processingFailed(Error)

    var errorDescription: String? {
        switch self {
        case .modelNotLoaded:
            return "ML model failed to load"
        case .invalidImage:
            return "Invalid image format"
        case .invalidResults:
            return "Failed to process results"
        case .processingFailed(let error):
            return "Processing failed: \(error.localizedDescription)"
        }
    }
}

Best Practices

✅ Do

// Load models asynchronously
Task {
    model = try await loadModelAsync()
}

// Use appropriate compute units
config.computeUnits = .all  // Let system decide

// Cache model instances
class ModelCache {
    static let shared = ModelCache()
    private var models: [String: MLModel] = [:]
}

// Process images at appropriate size
// Most models work with 224x224 or 299x299

// Use Vision framework for preprocessing
let request = VNCoreMLRequest(model: model)
request.imageCropAndScaleOption = .centerCrop

❌ Don't

// Don't load models on main thread
let model = try! MyModel()  // ❌ Blocks UI

// Don't create new model instances per inference
func classify() {
    let model = try! MyModel()  // ❌ Expensive
}

// Don't ignore model size
// Large models = slower load, more memory

// Don't skip error handling
let result = try! model.prediction(...)  // ❌ Will crash

Model Resources

Finding Pre-trained Models

1. Apple's Core ML Models Gallery
   https://developer.apple.com/machine-learning/models/

2. Hugging Face (with Core ML export)
   https://huggingface.co/models?library=coreml

3. TensorFlow Hub (convert with coremltools)
   https://tfhub.dev

4. Create your own with Create ML
   Xcode → File → New → Playground → Create ML