Agent skills for iOS, iPadOS, Swift, SwiftUI, and modern Apple framework development.
71
89%
Does it follow best practices?
Impact
—
No eval scenarios have been run
Advisory
Suggest reviewing before use
Complete implementation patterns for loading, configuring, and running Core ML models in Swift. All patterns target iOS 26+ with Swift 6.3, backward-compatible to iOS 14 unless noted.
Use an actor to manage model lifecycle, prevent concurrent loading, and cache compiled models persistently.
import CoreML
actor ModelManager {
private var loadedModels: [String: MLModel] = [:]
private let cacheDirectory: URL
init() {
let appSupport = FileManager.default.urls(
for: .applicationSupportDirectory, in: .userDomainMask
).first!
cacheDirectory = appSupport.appendingPathComponent("CompiledModels", isDirectory: true)
try? FileManager.default.createDirectory(at: cacheDirectory, withIntermediateDirectories: true)
}
func model(named name: String, configuration: MLModelConfiguration = .init()) async throws -> MLModel {
if let cached = loadedModels[name] {
return cached
}
let compiledURL = try await compiledModelURL(for: name)
let model = try await MLModel.load(contentsOf: compiledURL, configuration: configuration)
loadedModels[name] = model
return model
}
func unloadModel(named name: String) {
loadedModels.removeValue(forKey: name)
}
func unloadAll() {
loadedModels.removeAll()
}
private func compiledModelURL(for name: String) async throws -> URL {
// Check for pre-compiled model in cache
let cachedURL = cacheDirectory.appendingPathComponent("\(name).mlmodelc")
if FileManager.default.fileExists(atPath: cachedURL.path) {
return cachedURL
}
// Check for pre-compiled model in bundle
if let bundledCompiledURL = Bundle.main.url(forResource: name, withExtension: "mlmodelc") {
return bundledCompiledURL
}
// Compile from .mlpackage and cache
guard let packageURL = Bundle.main.url(forResource: name, withExtension: "mlpackage") else {
throw ModelManagerError.modelNotFound(name)
}
let tempCompiledURL = try await MLModel.compileModel(at: packageURL)
// Move compiled model to persistent cache
if FileManager.default.fileExists(atPath: cachedURL.path) {
try FileManager.default.removeItem(at: cachedURL)
}
try FileManager.default.moveItem(at: tempCompiledURL, to: cachedURL)
return cachedURL
}
}
enum ModelManagerError: Error {
case modelNotFound(String)
case predictionFailed(String)
}@MainActor
@Observable
final class ClassifierViewModel {
var classLabel: String = ""
var confidence: Double = 0
var isLoading = false
var errorMessage: String?
private let modelManager = ModelManager()
func classify(image: CGImage) async {
isLoading = true
defer { isLoading = false }
do {
let config = MLModelConfiguration()
config.computeUnits = .all
let model = try await modelManager.model(named: "ImageClassifier", configuration: config)
let pixelBuffer = try image.toPixelBuffer(width: 224, height: 224)
let input = try MLDictionaryFeatureProvider(dictionary: [
"image": MLFeatureValue(pixelBuffer: pixelBuffer),
])
let output = try await model.prediction(from: input)
classLabel = output.featureValue(for: "classLabel")?.stringValue ?? "Unknown"
confidence = output.featureValue(for: "classLabelProbs")?
.dictionaryValue[classLabel]?
.doubleValue ?? 0
} catch {
errorMessage = "Classification failed: \(error.localizedDescription)"
}
}
}When you add a .mlpackage or .mlmodelc to your Xcode project, Xcode
generates a Swift class with typed inputs and outputs.
import CoreML
// Xcode generates: MyImageClassifier, MyImageClassifierInput, MyImageClassifierOutput
// Synchronous prediction with generated types
func classifyWithGeneratedClass(pixelBuffer: CVPixelBuffer) throws -> (label: String, confidence: Double) {
let config = MLModelConfiguration()
config.computeUnits = .all
let classifier = try MyImageClassifier(configuration: config)
let input = MyImageClassifierInput(image: pixelBuffer)
let output = try classifier.prediction(input: input)
let topLabel = output.classLabel
let topConfidence = output.classLabelProbs[topLabel] ?? 0
return (topLabel, topConfidence)
}// Get the underlying MLModel from a generated class
let classifier = try MyImageClassifier(configuration: config)
let mlModel = classifier.model
// Useful for Vision integration
let vnModel = try VNCoreMLModel(for: mlModel)Implement MLFeatureProvider when you need custom input construction.
import CoreML
final class CustomImageInput: MLFeatureProvider {
let image: CVPixelBuffer
let confidenceThreshold: Double
var featureNames: Set<String> {
["image", "confidence_threshold"]
}
func featureValue(for featureName: String) -> MLFeatureValue? {
switch featureName {
case "image":
return MLFeatureValue(pixelBuffer: image)
case "confidence_threshold":
return MLFeatureValue(double: confidenceThreshold)
default:
return nil
}
}
init(image: CVPixelBuffer, confidenceThreshold: Double = 0.5) {
self.image = image
self.confidenceThreshold = confidenceThreshold
}
}
// Usage
let input = CustomImageInput(image: pixelBuffer, confidenceThreshold: 0.7)
let output = try model.prediction(from: input)actor PredictionService {
private let model: MLModel
init(model: MLModel) {
self.model = model
}
func predict(input: MLFeatureProvider) async throws -> MLFeatureProvider {
try await model.prediction(from: input)
}
}func classifyFrames(_ frames: AsyncStream<CVPixelBuffer>) async throws -> AsyncStream<String> {
let model = try await ModelManager().model(named: "Classifier")
return AsyncStream { continuation in
Task {
for await frame in frames {
let input = try MLDictionaryFeatureProvider(dictionary: [
"image": MLFeatureValue(pixelBuffer: frame),
])
let output = try await model.prediction(from: input)
let label = output.featureValue(for: "classLabel")?.stringValue ?? "unknown"
continuation.yield(label)
}
continuation.finish()
}
}
}import CoreML
func batchClassify(images: [CVPixelBuffer], model: MLModel) throws -> [(label: String, confidence: Double)] {
let batchInputs = try MLArrayBatchProvider(array: images.map { buffer in
try MLDictionaryFeatureProvider(dictionary: [
"image": MLFeatureValue(pixelBuffer: buffer),
])
})
let batchOutput = try model.predictions(from: batchInputs)
var results: [(String, Double)] = []
for i in 0..<batchOutput.count {
let features = batchOutput.features(at: i)
let label = features.featureValue(for: "classLabel")?.stringValue ?? "unknown"
let probs = features.featureValue(for: "classLabelProbs")?.dictionaryValue ?? [:]
let confidence = (probs[label] as? NSNumber)?.doubleValue ?? 0
results.append((label, confidence))
}
return results
}MLState enables models that maintain internal state across predictions. This is
essential for sequence models (text generation, audio classification, time-series)
where each prediction depends on previous context.
import CoreML
/// Audio classification that accumulates context over time
actor AudioClassifier {
private let model: MLModel
private var state: MLState?
init(model: MLModel) {
self.model = model
}
/// Start a new classification session
func beginSession() {
state = model.makeState()
}
/// Classify the next audio frame using accumulated state
func classify(audioFeatures: MLMultiArray) async throws -> String {
guard let state else {
throw ClassifierError.noActiveSession
}
let input = try MLDictionaryFeatureProvider(dictionary: [
"audio_features": MLFeatureValue(multiArray: audioFeatures)
])
let output = try await model.prediction(from: input, using: state)
return output.featureValue(for: "label")?.stringValue ?? "unknown"
}
/// End the session and release state
func endSession() {
state = nil
}
}
enum ClassifierError: Error {
case noActiveSession
}MLState from a single actor or task. Do not share
across concurrency domains.model.makeState() per stream when processing
multiple concurrent sequences (e.g., multiple audio channels).import CoreVideo
import CoreGraphics
func createPixelBuffer(from cgImage: CGImage, width: Int, height: Int) throws -> CVPixelBuffer {
let attrs: [CFString: Any] = [
kCVPixelBufferCGImageCompatibilityKey: true,
kCVPixelBufferCGBitmapContextCompatibilityKey: true,
]
var pixelBuffer: CVPixelBuffer?
let status = CVPixelBufferCreate(
kCFAllocatorDefault,
width, height,
kCVPixelFormatType_32ARGB,
attrs as CFDictionary,
&pixelBuffer
)
guard status == kCVReturnSuccess, let buffer = pixelBuffer else {
throw ImageError.pixelBufferCreationFailed
}
CVPixelBufferLockBaseAddress(buffer, [])
defer { CVPixelBufferUnlockBaseAddress(buffer, []) }
guard let context = CGContext(
data: CVPixelBufferGetBaseAddress(buffer),
width: width,
height: height,
bitsPerComponent: 8,
bytesPerRow: CVPixelBufferGetBytesPerRow(buffer),
space: CGColorSpaceCreateDeviceRGB(),
bitmapInfo: CGImageAlphaInfo.noneSkipFirst.rawValue
) else {
throw ImageError.contextCreationFailed
}
context.draw(cgImage, in: CGRect(x: 0, y: 0, width: width, height: height))
return buffer
}
enum ImageError: Error {
case pixelBufferCreationFailed
case contextCreationFailed
}For CIImage sources, use CIContext.render(_:to:) into a CVPixelBuffer
created with kCVPixelFormatType_32BGRA.
import CoreML
let array = try MLMultiArray(shape: [1, 3, 224, 224], dataType: .float32)
for i in 0..<array.count { array[i] = NSNumber(value: Float.random(in: 0...1)) }
let values: [Float] = [1.0, 2.0, 3.0, 4.0, 5.0]
let mlArray = try MLMultiArray(values)
// Access elements
let element = array[[0, 0, 112, 112] as [NSNumber]].floatValue
// Convert to Swift array
func toFloatArray(_ multiArray: MLMultiArray) -> [Float] {
let pointer = multiArray.dataPointer.assumingMemoryBound(to: Float.self)
return Array(UnsafeBufferPointer(start: pointer, count: multiArray.count))
}
let featureValue = MLFeatureValue(multiArray: array)import CoreML
// Creation patterns
let tensor1D = MLTensor([1.0, 2.0, 3.0, 4.0])
let zeros = MLTensor(zeros: [3, 224, 224], scalarType: Float.self)
let ones = MLTensor(ones: [2, 2], scalarType: Float.self)
// Reshaping
let reshaped = tensor1D.reshaped(to: [2, 2])
let expanded = tensor1D.expandingShape(at: 0) // [1, 4]
// Arithmetic and reduction
let sum = tensor1D + ones.reshaped(to: [4])
let mean = tensor1D.mean()
let argmax = tensor1D.argmax()
// Activation functions
let softmaxed = tensor1D.softmax()
// Interop with MLMultiArray
let multiArray = try MLMultiArray([1.0, 2.0, 3.0])
let fromMultiArray = MLTensor(multiArray)
let shaped = tensor1D.shapedArray(of: Float.self) // MLShapedArray
// Concatenation
let a = MLTensor([1.0, 2.0])
let b = MLTensor([3.0, 4.0])
let concatenated = MLTensor(concatenating: [a, b], alongAxis: 0)
// Normalization pattern (e.g., ImageNet preprocessing)
func normalize(_ tensor: MLTensor, mean: [Float], std: [Float]) -> MLTensor {
let meanTensor = MLTensor(mean)
let stdTensor = MLTensor(std)
return (tensor - meanTensor) / stdTensor
}import Vision
import CoreML
@MainActor
@Observable
final class ObjectDetectionViewModel {
var detections: [Detection] = []
var isProcessing = false
struct Detection: Identifiable {
let id = UUID()
let label: String
let confidence: Float
let boundingBox: CGRect
}
func detect(in image: CGImage) async {
isProcessing = true
defer { isProcessing = false }
do {
let config = MLModelConfiguration()
config.computeUnits = .all
let detector = try MyObjectDetector(configuration: config)
let request = CoreMLRequest(model: .init(detector.model))
let results = try await request.perform(on: image)
detections = results.compactMap { observation in
guard let object = observation as? RecognizedObjectObservation,
let topLabel = object.labels.first else { return nil }
return Detection(
label: topLabel.identifier,
confidence: topLabel.confidence,
boundingBox: object.boundingBox
)
}
} catch {
detections = []
}
}
}func classifyImage(_ image: CGImage) async throws -> [(label: String, confidence: Float)] {
let classifier = try MyImageClassifier(configuration: .init())
let request = CoreMLRequest(model: .init(classifier.model))
let results = try await request.perform(on: image)
return results.compactMap { observation in
guard let classification = observation as? ClassificationObservation else { return nil }
return (classification.identifier, classification.confidence)
}
}import Vision
import CoreML
func detectLegacy(in image: CGImage) async throws -> [VNRecognizedObjectObservation] {
let config = MLModelConfiguration()
config.computeUnits = .all
let detector = try MyObjectDetector(configuration: config)
let vnModel = try VNCoreMLModel(for: detector.model)
let request = VNCoreMLRequest(model: vnModel)
request.imageCropAndScaleOption = .scaleFill
let handler = VNImageRequestHandler(cgImage: image)
return try await Task.detached {
try handler.perform([request])
return request.results as? [VNRecognizedObjectObservation] ?? []
}.value
}func classifyImageLegacy(_ image: CGImage) async throws -> [(label: String, confidence: Float)] {
let classifier = try MyImageClassifier(configuration: .init())
let vnModel = try VNCoreMLModel(for: classifier.model)
let request = VNCoreMLRequest(model: vnModel)
request.imageCropAndScaleOption = .centerCrop
let handler = VNImageRequestHandler(cgImage: image)
return try await Task.detached {
try handler.perform([request])
guard let results = request.results as? [VNClassificationObservation] else { return [] }
return results.prefix(5).map { ($0.identifier, $0.confidence) }
}.value
}Use NLModel to load Core ML models trained for NLP tasks.
import NaturalLanguage
func analyzeSentiment(text: String) throws -> (label: String, confidence: Double)? {
let modelURL = Bundle.main.url(forResource: "SentimentClassifier", withExtension: "mlmodelc")!
let nlModel = try NLModel(contentsOf: modelURL)
guard let label = nlModel.predictedLabel(for: text) else { return nil }
let hypotheses = nlModel.predictedLabelHypotheses(for: text, maximumCount: 1)
let confidence = hypotheses[label] ?? 0
return (label, confidence)
}import CoreML
func profileModel(at url: URL) async throws {
let config = MLModelConfiguration()
config.computeUnits = .all
let computePlan = try await MLComputePlan.load(contentsOf: url, configuration: config)
guard case let .program(program) = computePlan.modelStructure,
let mainFunction = program.functions["main"] else {
print("Model is not an ML program or has no main function")
return
}
for operation in mainFunction.block.operations {
let opName = operation.operatorName
if let deviceUsage = computePlan.deviceUsage(for: operation) {
print(" \(opName): \(deviceUsage.preferredComputeDevice)")
}
if let cost = computePlan.estimatedCost(of: operation) {
print(" Estimated weight: \(cost.weight)")
}
}
}| Device | Meaning | Action |
|---|---|---|
| Neural Engine | Best efficiency and speed for supported ops | Ideal -- no changes needed |
| GPU | Runs on Metal GPU | Good for large matrix ops |
| CPU | Fallback for unsupported operations | Investigate if many ops fall here |
If many critical operations fall back to CPU, try .cpuAndGPU compute units,
check for unsupported ANE operations, or re-convert with a different deployment target.
Manage model loading and memory across app lifecycle transitions.
@MainActor
@Observable
final class AppModelState {
var isModelReady = false
private let modelManager = ModelManager()
func warmup() async {
do {
let config = MLModelConfiguration()
config.computeUnits = .all
_ = try await modelManager.model(named: "MainClassifier", configuration: config)
isModelReady = true
} catch {
isModelReady = false
}
}
func handleBackground() async {
await modelManager.unloadAll()
isModelReady = false
}
}
// SwiftUI integration
@main
struct MyApp: App {
@State private var modelState = AppModelState()
@Environment(\.scenePhase) private var scenePhase
var body: some Scene {
WindowGroup {
ContentView()
.environment(modelState)
.task { await modelState.warmup() }
.onChange(of: scenePhase) { _, newPhase in
Task {
if newPhase == .background {
await modelState.handleBackground()
} else if newPhase == .active {
await modelState.warmup()
}
}
}
}
}
}import CoreML
func loadAndPredict(modelName: String, input: MLFeatureProvider) async -> MLFeatureProvider? {
let config = MLModelConfiguration()
config.computeUnits = .all
do {
guard let url = Bundle.main.url(forResource: modelName, withExtension: "mlmodelc") else {
print("Model \(modelName) not found in bundle")
return nil
}
let model = try await MLModel.load(contentsOf: url, configuration: config)
return try await model.prediction(from: input)
} catch {
print("Model error: \(error)")
return nil
}
}| Error | Cause | Fix |
|---|---|---|
MLModel file not found | Wrong bundle path or missing target membership | Verify file is in correct target |
| Compilation failure | Corrupted .mlpackage or unsupported ops | Re-export from coremltools |
| Input shape mismatch | Wrong image dimensions or tensor shape | Match model's expected input shape |
| Out of memory | Model too large for device | Use smaller model or .cpuOnly compute |
| Compute unit fallback | Ops unsupported on requested device | Use .all or check MLComputePlan |
import Testing
import CoreML
struct ModelLoadingTests {
@Test func loadModelSucceeds() async throws {
let config = MLModelConfiguration()
config.computeUnits = .cpuOnly // CPU for test stability
let model = try MyImageClassifier(configuration: config)
#expect(model.model.modelDescription.inputDescriptionsByName.count > 0)
}
@Test func predictionReturnsValidOutput() async throws {
let config = MLModelConfiguration()
config.computeUnits = .cpuOnly
let model = try MyImageClassifier(configuration: config)
let input = try createTestInput(width: 224, height: 224)
let output = try model.prediction(input: input)
#expect(!output.classLabel.isEmpty)
#expect(output.classLabelProbs.values.allSatisfy { $0 >= 0 && $0 <= 1 })
}
@Test func predictionLatencyUnderThreshold() async throws {
let config = MLModelConfiguration()
config.computeUnits = .all
let model = try MyImageClassifier(configuration: config)
let input = try createTestInput(width: 224, height: 224)
_ = try model.prediction(input: input) // Warm up
let start = ContinuousClock.now
for _ in 0..<10 {
_ = try model.prediction(input: input)
}
let avgMs = (ContinuousClock.now - start) / 10
#expect(avgMs < .milliseconds(50), "Average prediction time \(avgMs) exceeds 50ms")
}
}
private func createTestPixelBuffer(width: Int, height: Int) throws -> CVPixelBuffer {
var pixelBuffer: CVPixelBuffer?
let status = CVPixelBufferCreate(
kCFAllocatorDefault, width, height,
kCVPixelFormatType_32ARGB, nil, &pixelBuffer
)
guard status == kCVReturnSuccess, let buffer = pixelBuffer else {
throw ImageError.pixelBufferCreationFailed
}
return buffer
}scenePhase == .background
and reload on return to foreground. iOS reclaims memory aggressively..cpuOnly for background processing. GPU and Neural Engine may not
be available in background execution contexts..mlmodelc loads faster and uses less transient
memory than compiling .mlpackage at runtime.ModelManager above) to
ensure only one instance of each model exists.MLArrayBatchProvider instances
hold references to all input data.skills
accessorysetupkit
references
activitykit
references
adattributionkit
references
alarmkit
references
app-clips
app-intents
references
app-store-optimization
app-store-review
apple-on-device-ai
appmigrationkit
references
audioaccessorykit
references
authentication
references
avkit
references
background-processing
references
browserenginekit
references
callkit
references
carplay
references
cloudkit
references
contacts-framework
references
core-bluetooth
references
core-data
core-motion
references
core-nfc
references
coreml
references
cryptokit
references
cryptotokenkit
references
debugging-instruments
device-integrity
references
dockkit
references
energykit
references
eventkit
references
financekit
references
focus-engine
gamekit
references
healthkit
references
homekit
references
ios-accessibility
ios-localization
ios-networking
ios-simulator
references
mapkit
metrickit
references
musickit
references
natural-language
references
paperkit
references
passkit
references
pdfkit
references
pencilkit
references
permissionkit
references
photokit
push-notifications
realitykit
references
relevancekit
references
scenekit
references
sensorkit
references
speech-recognition
spritekit
references
storekit
swift-api-design-guidelines
swift-architecture
swift-charts
references
swift-codable
swift-concurrency
swift-formatstyle
swift-language
swift-security
references
swift-testing
swiftdata
swiftlint
swiftui-animation
swiftui-gestures
references
swiftui-layout-components
swiftui-liquid-glass
references
swiftui-patterns
swiftui-performance
swiftui-uikit-interop
swiftui-webkit
tabletopkit
references
tipkit
references
vision-framework
weatherkit
references
widgetkit
references