0
# AI & Machine Learning APIs
1

2
Comprehensive AI and Machine Learning capabilities for WeChat Mini Programs, including local ONNX model inference, cloud AI services, and AI-powered features.
3

4
## Capabilities
5

6
### Local AI Inference
7

8
Local machine learning model inference using ONNX runtime for privacy-preserving AI applications.
9

10
```typescript { .api }
11
/**
12
 * Create AI inference session for ONNX models
13
 * @param option - Inference session configuration
14
 */
15
function createInferenceSession(option: CreateInferenceSessionOption): InferenceSession;
16

17
/**
18
 * Get AI inference environment information
19
 * @param option - Environment info configuration
20
 */
21
function getInferenceEnvInfo(option?: GetInferenceEnvInfoOption): void;
22

23
interface CreateInferenceSessionOption {
24
  /** ONNX model file path (supports code package path and local file system path) */
25
  model: string;
26
  /** Whether to use NPU inference (iOS only) */
27
  allowNPU?: boolean;
28
  /** Whether to generate quantized model inference */
29
  allowQuantize?: boolean;
30
  /** Precision level for computation */
31
  precisionLevel?: 1 | 2 | 3 | 4;
32
  /** Typical input shapes for dynamic axis models */
33
  typicalShape?: Record<string, number[]>;
34
}
35

36
interface GetInferenceEnvInfoOption {
37
  /** Success callback */
38
  success?(res: GetInferenceEnvInfoSuccessCallbackResult): void;
39
  /** Failure callback */
40
  fail?(res: any): void;
41
  /** Completion callback */
42
  complete?(res: any): void;
43
}
44

45
interface GetInferenceEnvInfoSuccessCallbackResult {
46
  /** AI inference engine version */
47
  ver: string;
48
  /** Success message */
49
  errMsg: string;
50
}
51
```
52

53
**Usage Examples:**
54

55
```typescript
56
// Check AI inference environment
57
wx.getInferenceEnvInfo({
58
  success(res) {
59
    console.log('AI Engine Version:', res.ver);
60
  },
61
  fail(err) {
62
    console.error('AI inference not supported:', err);
63
  }
64
});
65

66
// Create inference session
67
const session = wx.createInferenceSession({
68
  model: `${wx.env.USER_DATA_PATH}/image_classifier.onnx`,
69
  precisionLevel: 4,
70
  allowNPU: true,
71
  allowQuantize: false,
72
  typicalShape: {
73
    input: [1, 3, 224, 224], // Batch, Channels, Height, Width
74
    mask: [1, 1, 224, 224]   // Optional mask input
75
  }
76
});
77

78
// Handle session load events
79
session.onLoad(() => {
80
  console.log('Model loaded successfully');
81
  
82
  // Model is ready for inference
83
  runInference();
84
});
85

86
session.onError((err) => {
87
  console.error('Model loading failed:', err);
88
});
89

90
async function runInference() {
91
  try {
92
    // Prepare input tensors
93
    const inputTensor = new Float32Array(1 * 3 * 224 * 224);
94
    // ... fill tensor with image data
95
    
96
    const inputTensors = {
97
      input: {
98
        data: inputTensor,
99
        shape: [1, 3, 224, 224],
100
        type: 'float32'
101
      }
102
    };
103
    
104
    // Run inference
105
    const results = await session.run(inputTensors);
106
    
107
    console.log('Inference results:', results);
108
    
109
    // Process results
110
    const predictions = results.output.data;
111
    const topClass = Array.from(predictions).indexOf(Math.max(...predictions));
112
    console.log('Predicted class:', topClass);
113
    
114
  } catch (error) {
115
    console.error('Inference failed:', error);
116
  }
117
}
118

119
// Clean up when done
120
function cleanup() {
121
  session.destroy();
122
  console.log('Inference session destroyed');
123
}
124
```
125

126
### InferenceSession Interface
127

128
Complete interface for managing AI inference sessions.
129

130
```typescript { .api }
131
interface InferenceSession {
132
  /**
133
   * Run inference on input tensors
134
   * @param tensors - Input tensor data
135
   * @returns Promise resolving to output tensors
136
   */
137
  run(tensors: Tensors): Promise<Tensors>;
138
  
139
  /**
140
   * Destroy inference session and free resources
141
   */
142
  destroy(): void;
143
  
144
  /**
145
   * Listen for session load completion
146
   * @param callback - Load completion callback
147
   */
148
  onLoad(callback: () => void): void;
149
  
150
  /**
151
   * Remove load event listener
152
   * @param callback - Callback to remove
153
   */
154
  offLoad(callback: () => void): void;
155
  
156
  /**
157
   * Listen for session errors
158
   * @param callback - Error callback
159
   */
160
  onError(callback: (error: any) => void): void;
161
  
162
  /**
163
   * Remove error event listener
164
   * @param callback - Callback to remove
165
   */
166
  offError(callback: (error: any) => void): void;
167
}
168

169
interface Tensors {
170
  [key: string]: Tensor;
171
}
172

173
interface Tensor {
174
  /** Tensor data as typed array */
175
  data: Float32Array | Int32Array | Uint8Array;
176
  /** Tensor shape dimensions */
177
  shape: number[];
178
  /** Data type */
179
  type: 'float32' | 'int32' | 'uint8' | 'int64' | 'float64';
180
}
181
```
182

183
**Advanced Usage Examples:**
184

185
```typescript
186
// Multi-input model inference
187
const multiInputSession = wx.createInferenceSession({
188
  model: `${wx.env.USER_DATA_PATH}/multi_input_model.onnx`,
189
  precisionLevel: 3,
190
  typicalShape: {
191
    image: [1, 3, 224, 224],
192
    features: [1, 100],
193
    metadata: [1, 10]
194
  }
195
});
196

197
multiInputSession.onLoad(async () => {
198
  // Prepare multiple inputs
199
  const imageData = new Float32Array(1 * 3 * 224 * 224);
200
  const featureData = new Float32Array(1 * 100);
201
  const metadataData = new Float32Array(1 * 10);
202
  
203
  // Fill with actual data...
204
  
205
  const inputs = {
206
    image: {
207
      data: imageData,
208
      shape: [1, 3, 224, 224],
209
      type: 'float32'
210
    },
211
    features: {
212
      data: featureData,
213
      shape: [1, 100],
214
      type: 'float32'
215
    },
216
    metadata: {
217
      data: metadataData,
218
      shape: [1, 10],
219
      type: 'float32'
220
    }
221
  };
222
  
223
  try {
224
    const results = await multiInputSession.run(inputs);
225
    
226
    // Process multiple outputs
227
    console.log('Classification:', results.classification.data);
228
    console.log('Confidence:', results.confidence.data);
229
    console.log('Features:', results.extracted_features.data);
230
    
231
  } catch (error) {
232
    console.error('Multi-input inference failed:', error);
233
  }
234
});
235

236
// Batch inference for multiple samples
237
async function batchInference(samples: Float32Array[]) {
238
  const batchSize = samples.length;
239
  const batchData = new Float32Array(batchSize * 3 * 224 * 224);
240
  
241
  // Concatenate samples into batch
242
  samples.forEach((sample, index) => {
243
    const offset = index * 3 * 224 * 224;
244
    batchData.set(sample, offset);
245
  });
246
  
247
  const batchInputs = {
248
    input: {
249
      data: batchData,
250
      shape: [batchSize, 3, 224, 224],
251
      type: 'float32'
252
    }
253
  };
254
  
255
  const results = await session.run(batchInputs);
256
  
257
  // Split batch results
258
  const predictions = [];
259
  const outputSize = results.output.data.length / batchSize;
260
  
261
  for (let i = 0; i < batchSize; i++) {
262
    const start = i * outputSize;
263
    const end = start + outputSize;
264
    predictions.push(Array.from(results.output.data.slice(start, end)));
265
  }
266
  
267
  return predictions;
268
}
269
```
270

271
### Face Detection APIs
272

273
Built-in face detection and analysis capabilities.
274

275
```typescript { .api }
276
/**
277
 * Initialize face detection
278
 * @param option - Face detection initialization
279
 */
280
function initFaceDetect(option: InitFaceDetectOption): void;
281

282
/**
283
 * Detect faces in image
284
 * @param option - Face detection configuration
285
 */
286
function faceDetect(option: FaceDetectOption): void;
287

288
/**
289
 * Stop face detection
290
 * @param option - Stop face detection configuration
291
 */
292
function stopFaceDetect(option?: StopFaceDetectOption): void;
293

294
interface InitFaceDetectOption {
295
  /** Success callback */
296
  success?(res: any): void;
297
  /** Failure callback */
298
  fail?(res: any): void;
299
  /** Completion callback */
300
  complete?(res: any): void;
301
}
302

303
interface FaceDetectOption {
304
  /** Image file path */
305
  frameBuffer: ArrayBuffer;
306
  /** Image width */
307
  width: number;
308
  /** Image height */
309
  height: number;
310
  /** Enable angle detection */
311
  enableAngle?: boolean;
312
  /** Enable point detection */
313
  enablePoint?: boolean;
314
  /** Enable confidence score */
315
  enableConf?: boolean;
316
  /** Success callback */
317
  success?(res: FaceDetectSuccessCallbackResult): void;
318
  /** Failure callback */
319
  fail?(res: any): void;
320
  /** Completion callback */
321
  complete?(res: any): void;
322
}
323

324
interface FaceDetectSuccessCallbackResult {
325
  /** Detected faces */
326
  faceInfo: FaceInfo[];
327
  /** Error message */
328
  errMsg: string;
329
}
330

331
interface FaceInfo {
332
  /** Face bounding box */
333
  x: number;
334
  y: number;
335
  width: number;
336
  height: number;
337
  /** Face angle (if enabled) */
338
  angleArray?: {
339
    pitch: number;
340
    roll: number;
341
    yaw: number;
342
  };
343
  /** Facial landmarks (if enabled) */
344
  pointArray?: FacePoint[];
345
  /** Confidence score (if enabled) */
346
  confArray?: number[];
347
}
348

349
interface FacePoint {
350
  /** Point x coordinate */
351
  x: number;
352
  /** Point y coordinate */
353
  y: number;
354
}
355
```
356

357
**Usage Examples:**
358

359
```typescript
360
// Initialize face detection
361
wx.initFaceDetect({
362
  success() {
363
    console.log('Face detection initialized');
364
  },
365
  fail(err) {
366
    console.error('Face detection init failed:', err);
367
  }
368
});
369

370
// Capture image and detect faces
371
wx.createCameraContext('cameraId').takePhoto({
372
  quality: 'high',
373
  success(res) {
374
    // Convert image to ArrayBuffer
375
    wx.getFileSystemManager().readFile({
376
      filePath: res.tempImagePath,
377
      success(fileRes) {
378
        // Perform face detection
379
        wx.faceDetect({
380
          frameBuffer: fileRes.data,
381
          width: 640,
382
          height: 480,
383
          enableAngle: true,
384
          enablePoint: true,
385
          enableConf: true,
386
          success(detectRes) {
387
            console.log(`Detected ${detectRes.faceInfo.length} faces`);
388
            
389
            detectRes.faceInfo.forEach((face, index) => {
390
              console.log(`Face ${index + 1}:`);
391
              console.log(`  Position: (${face.x}, ${face.y})`);
392
              console.log(`  Size: ${face.width}x${face.height}`);
393
              
394
              if (face.angleArray) {
395
                console.log(`  Angles: pitch=${face.angleArray.pitch}, roll=${face.angleArray.roll}, yaw=${face.angleArray.yaw}`);
396
              }
397
              
398
              if (face.pointArray) {
399
                console.log(`  Landmarks: ${face.pointArray.length} points`);
400
              }
401
              
402
              if (face.confArray) {
403
                console.log(`  Confidence: ${face.confArray.join(', ')}`);
404
              }
405
            });
406
          },
407
          fail(err) {
408
            console.error('Face detection failed:', err);
409
          }
410
        });
411
      }
412
    });
413
  }
414
});
415

416
// Stop face detection when done
417
wx.stopFaceDetect({
418
  success() {
419
    console.log('Face detection stopped');
420
  }
421
});
422
```
423

424
### Vision Kit (VK) Session
425

426
Advanced computer vision capabilities for AR and object recognition.
427

428
```typescript { .api }
429
/**
430
 * Create Vision Kit session for AR and computer vision
431
 * @param option - VK session configuration
432
 */
433
function createVKSession(option: CreateVKSessionOption): VKSession;
434

435
interface CreateVKSessionOption {
436
  /** VK session track */
437
  track: VKTrack;
438
  /** VK session version */
439
  version?: 'v1' | 'v2';
440
  /** Success callback */
441
  success?(res: any): void;
442
  /** Failure callback */
443
  fail?(res: any): void;
444
  /** Completion callback */
445
  complete?(res: any): void;
446
}
447

448
interface VKSession {
449
  /**
450
   * Start VK session
451
   * @param callback - Start callback
452
   */
453
  start(callback?: (res: any) => void): void;
454
  
455
  /**
456
   * Stop VK session
457
   * @param callback - Stop callback
458
   */
459
  stop(callback?: (res: any) => void): void;
460
  
461
  /**
462
   * Destroy VK session
463
   */
464
  destroy(): void;
465
  
466
  /**
467
   * Detect anchor points
468
   * @param option - Detection configuration
469
   */
470
  detectAnchors(option: VKDetectAnchorsOption): void;
471
  
472
  /**
473
   * Get camera texture
474
   */
475
  getCameraTexture(): any;
476
}
477

478
interface VKTrack {
479
  /** Plane tracking */
480
  plane?: {
481
    mode?: 1 | 3; // 1 for horizontal, 3 for both
482
  };
483
  /** Face tracking */
484
  face?: {
485
    mode?: 1 | 2; // 1 for single face, 2 for multiple faces
486
  };
487
  /** Object tracking */
488
  osd?: {
489
    mode?: 1;
490
  };
491
  /** Hand tracking */
492
  hand?: {
493
    mode?: 1;
494
  };
495
}
496
```
497

498
### Cloud AI Services
499

500
Server-side AI capabilities through WeChat Cloud functions.
501

502
```typescript { .api }
503
interface WxCloud {
504
  /**
505
   * Generate text using cloud AI models
506
   * @param options - Text generation configuration
507
   */
508
  generateText(options: ICloud.GenerateTextOptions): Promise<ICloud.GenerateTextResult>;
509
  
510
  /**
511
   * Analyze image content
512
   * @param options - Image analysis configuration
513
   */
514
  analyzeImage(options: ICloud.AnalyzeImageOptions): Promise<ICloud.AnalyzeImageResult>;
515
  
516
  /**
517
   * Convert speech to text
518
   * @param options - Speech recognition configuration
519
   */
520
  speechToText(options: ICloud.SpeechToTextOptions): Promise<ICloud.SpeechToTextResult>;
521
  
522
  /**
523
   * Convert text to speech
524
   * @param options - Text-to-speech configuration
525
   */
526
  textToSpeech(options: ICloud.TextToSpeechOptions): Promise<ICloud.TextToSpeechResult>;
527
}
528

529
interface ICloud.GenerateTextOptions {
530
  /** AI model identifier */
531
  modelId: string;
532
  /** Input prompt */
533
  prompt: string;
534
  /** Generation parameters */
535
  parameters?: {
536
    /** Maximum tokens to generate */
537
    maxTokens?: number;
538
    /** Sampling temperature */
539
    temperature?: number;
540
    /** Top-p sampling */
541
    topP?: number;
542
    /** Frequency penalty */
543
    frequencyPenalty?: number;
544
    /** Presence penalty */
545
    presencePenalty?: number;
546
  };
547
}
548

549
interface ICloud.GenerateTextResult {
550
  /** Generated text */
551
  text: string;
552
  /** Token usage statistics */
553
  usage: {
554
    promptTokens: number;
555
    completionTokens: number;
556
    totalTokens: number;
557
  };
558
}
559

560
interface ICloud.AnalyzeImageOptions {
561
  /** Image file ID or URL */
562
  imageUrl: string;
563
  /** Analysis features to enable */
564
  features: ('labels' | 'faces' | 'text' | 'objects')[];
565
  /** Maximum results per feature */
566
  maxResults?: number;
567
}
568

569
interface ICloud.AnalyzeImageResult {
570
  /** Detected labels */
571
  labels?: ImageLabel[];
572
  /** Detected faces */
573
  faces?: ImageFace[];
574
  /** Detected text */
575
  text?: ImageText;
576
  /** Detected objects */
577
  objects?: ImageObject[];
578
}
579
```
580

581
**Cloud AI Usage Examples:**
582

583
```typescript
584
// Text generation
585
wx.cloud.generateText({
586
  modelId: 'gpt-3.5-turbo',
587
  prompt: 'Write a product description for a smartphone',
588
  parameters: {
589
    maxTokens: 150,
590
    temperature: 0.7
591
  }
592
}).then(result => {
593
  console.log('Generated text:', result.text);
594
  console.log('Tokens used:', result.usage.totalTokens);
595
});
596

597
// Image analysis
598
wx.cloud.analyzeImage({
599
  imageUrl: 'cloud://my-env.my-bucket/image.jpg',
600
  features: ['labels', 'faces', 'text'],
601
  maxResults: 10
602
}).then(result => {
603
  if (result.labels) {
604
    console.log('Image labels:', result.labels);
605
  }
606
  if (result.faces) {
607
    console.log('Detected faces:', result.faces.length);
608
  }
609
  if (result.text) {
610
    console.log('Detected text:', result.text.content);
611
  }
612
});
613
```
614

615
## Types
616

617
```typescript { .api }
618
// AI inference types
619
interface ImageLabel {
620
  /** Label name */
621
  name: string;
622
  /** Confidence score */
623
  confidence: number;
624
  /** Label category */
625
  category?: string;
626
}
627

628
interface ImageFace {
629
  /** Face bounding box */
630
  boundingBox: {
631
    x: number;
632
    y: number;
633
    width: number;
634
    height: number;
635
  };
636
  /** Face attributes */
637
  attributes?: {
638
    age?: number;
639
    gender?: 'male' | 'female';
640
    emotion?: string;
641
  };
642
  /** Facial landmarks */
643
  landmarks?: FacePoint[];
644
}
645

646
interface ImageText {
647
  /** Detected text content */
648
  content: string;
649
  /** Text regions */
650
  regions: TextRegion[];
651
}
652

653
interface TextRegion {
654
  /** Text content */
655
  text: string;
656
  /** Bounding box */
657
  boundingBox: {
658
    x: number;
659
    y: number;
660
    width: number;
661
    height: number;
662
  };
663
  /** Confidence score */
664
  confidence: number;
665
}
666

667
interface ImageObject {
668
  /** Object name */
669
  name: string;
670
  /** Confidence score */
671
  confidence: number;
672
  /** Bounding box */
673
  boundingBox: {
674
    x: number;
675
    y: number;
676
    width: number;
677
    height: number;
678
  };
679
}
680

681
// VK session types
682
interface VKDetectAnchorsOption {
683
  /** Success callback */
684
  success?(res: VKDetectAnchorsResult): void;
685
  /** Failure callback */
686
  fail?(res: any): void;
687
}
688

689
interface VKDetectAnchorsResult {
690
  /** Detected anchors */
691
  anchors: VKAnchor[];
692
}
693

694
interface VKAnchor {
695
  /** Anchor ID */
696
  id: string;
697
  /** Anchor type */
698
  type: 'plane' | 'face' | 'hand';
699
  /** Transform matrix */
700
  transform: number[];
701
  /** Size (for plane anchors) */
702
  size?: { width: number; height: number };
703
}
704
```