0
# Utility Types
1

2
Additional utility types for payload manipulation, filtering, value expressions, and type transformations to enhance developer experience and type safety within the XYO Protocol 2.0 ecosystem.
3

4
## Capabilities
5

6
### Payload Value Expressions
7

8
Type-safe property extraction and manipulation utilities for working with payload data properties.
9

10
```typescript { .api }
11
/**
12
 * Type representing all property keys of a payload
13
 */
14
type PayloadProperty<T extends Payload = Payload> = keyof T;
15

16
/**
17
 * Type representing the value of a specific payload property
18
 */
19
type PayloadValue<
20
  T extends Payload = Payload,
21
  Key extends PayloadProperty<T> = PayloadProperty<T>
22
> = T[Key];
23

24
/**
25
 * Function type for extracting property values from payloads
26
 */
27
type PayloadValueExpression<
28
  T extends Payload = Payload,
29
  Key extends PayloadProperty<T> = PayloadProperty<T>,
30
  TValue = PayloadValue<T, Key>
31
> = (payload: T) => TValue;
32
```
33

34
**Usage Examples:**
35

36
```typescript
37
import { 
38
  PayloadProperty,
39
  PayloadValue,
40
  PayloadValueExpression,
41
  Payload
42
} from "@xyo-network/payload-model";
43

44
// Define custom payload type
45
interface UserPayload extends Payload {
46
  schema: "network.example.user";
47
  name: string;
48
  email: string;
49
  age: number;
50
  preferences: {
51
    theme: "light" | "dark";
52
    notifications: boolean;
53
  };
54
}
55

56
// Extract property types
57
type UserProperties = PayloadProperty<UserPayload>; 
58
// "schema" | "name" | "email" | "age" | "preferences"
59

60
type NameValue = PayloadValue<UserPayload, "name">; // string
61
type PreferencesValue = PayloadValue<UserPayload, "preferences">; 
62
// { theme: "light" | "dark"; notifications: boolean; }
63

64
// Create value expression functions
65
const getName: PayloadValueExpression<UserPayload, "name"> = (payload) => payload.name;
66
const getAge: PayloadValueExpression<UserPayload, "age"> = (payload) => payload.age;
67

68
// Generic value extractor
69
function createValueExtractor<T extends Payload, K extends PayloadProperty<T>>(
70
  key: K
71
): PayloadValueExpression<T, K> {
72
  return (payload) => payload[key];
73
}
74

75
// Use value extractors
76
const userPayload: UserPayload = {
77
  schema: "network.example.user",
78
  name: "Alice",
79
  email: "alice@example.com",
80
  age: 25,
81
  preferences: {
82
    theme: "dark",
83
    notifications: true
84
  }
85
};
86

87
const nameExtractor = createValueExtractor<UserPayload, "name">("name");
88
const emailExtractor = createValueExtractor<UserPayload, "email">("email");
89

90
console.log("Name:", nameExtractor(userPayload)); // "Alice"
91
console.log("Email:", emailExtractor(userPayload)); // "alice@example.com"
92

93
// Complex value expressions
94
const getDisplayName: PayloadValueExpression<UserPayload, "name", string> = (payload) => {
95
  return `${payload.name} (${payload.age})`;
96
};
97

98
const getTheme: PayloadValueExpression<UserPayload, "preferences", string> = (payload) => {
99
  return payload.preferences.theme;
100
};
101

102
console.log("Display:", getDisplayName(userPayload)); // "Alice (25)"
103
console.log("Theme:", getTheme(userPayload)); // "dark"
104

105
// Array processing with value expressions
106
const users: UserPayload[] = [
107
  { schema: "network.example.user", name: "Alice", email: "alice@example.com", age: 25, preferences: { theme: "dark", notifications: true } },
108
  { schema: "network.example.user", name: "Bob", email: "bob@example.com", age: 30, preferences: { theme: "light", notifications: false } }
109
];
110

111
const names = users.map(getName);
112
const ages = users.map(getAge);
113
const themes = users.map(getTheme);
114

115
console.log("Names:", names); // ["Alice", "Bob"]
116
console.log("Ages:", ages); // [25, 30]
117
console.log("Themes:", themes); // ["dark", "light"]
118
```
119

120
### Payload Collection Utilities
121

122
Interfaces and types for filtering, organizing, and managing collections of payloads.
123

124
```typescript { .api }
125
/**
126
 * Filter interface for payload discovery and search operations
127
 */
128
interface PayloadFindFilter {
129
  /** Maximum number of results to return */
130
  limit?: number;
131
  
132
  /** Sort order for results */
133
  order?: 'desc' | 'asc';
134
  
135
  /** Schema filter - single schema or array of schemas to match */
136
  schema?: string | string[];
137
}
138

139
/**
140
 * Hash map interface for efficient payload storage and lookup
141
 */
142
interface PayloadHashMap<
143
  TPayload extends Payload = Payload,
144
  TId extends string | number | symbol = Hash
145
> {
146
  /** 
147
   * Data hash mappings - maps root hashes to data hashes
148
   * Multiple root hashes may reference the same data hash
149
   */
150
  dataHash: Record<TId, TId>;
151
  
152
  /** 
153
   * Complete payload mappings - maps hashes to full payload objects
154
   */
155
  hash: Record<TId, TPayload>;
156
}
157
```
158

159
**Usage Examples:**
160

161
```typescript
162
import { 
163
  PayloadFindFilter,
164
  PayloadHashMap,
165
  Payload,
166
  Hash
167
} from "@xyo-network/payload-model";
168

169
// Payload filtering utilities
170
class PayloadFilter {
171
  static applyFilter<T extends Payload>(
172
    payloads: T[],
173
    filter: PayloadFindFilter
174
  ): T[] {
175
    let result = [...payloads];
176

177
    // Schema filtering
178
    if (filter.schema) {
179
      const schemas = Array.isArray(filter.schema) ? filter.schema : [filter.schema];
180
      result = result.filter(payload => schemas.includes(payload.schema));
181
    }
182

183
    // Sorting (simplified example)
184
    if (filter.order) {
185
      result.sort((a, b) => {
186
        const aKey = a.schema;
187
        const bKey = b.schema;
188
        const comparison = aKey.localeCompare(bKey);
189
        return filter.order === 'desc' ? -comparison : comparison;
190
      });
191
    }
192

193
    // Limit results
194
    if (filter.limit && filter.limit > 0) {
195
      result = result.slice(0, filter.limit);
196
    }
197

198
    return result;
199
  }
200

201
  // Create common filters
202
  static createSchemaFilter(schema: string | string[]): PayloadFindFilter {
203
    return { schema };
204
  }
205

206
  static createLimitFilter(limit: number, order?: 'desc' | 'asc'): PayloadFindFilter {
207
    return { limit, order };
208
  }
209

210
  static createCompleteFilter(
211
    schema?: string | string[],
212
    limit?: number,
213
    order?: 'desc' | 'asc'
214
  ): PayloadFindFilter {
215
    return { schema, limit, order };
216
  }
217
}
218

219
// Example payloads
220
const payloads: Payload[] = [
221
  { schema: "network.example.user", name: "Alice", email: "alice@example.com" },
222
  { schema: "network.example.product", name: "Laptop", price: 999 },
223
  { schema: "network.example.user", name: "Bob", email: "bob@example.com" },
224
  { schema: "network.example.order", userId: "alice", productId: "laptop" },
225
  { schema: "network.example.user", name: "Charlie", email: "charlie@example.com" }
226
];
227

228
// Apply various filters
229
const userFilter = PayloadFilter.createSchemaFilter("network.example.user");
230
const limitFilter = PayloadFilter.createLimitFilter(3, "asc");
231
const complexFilter = PayloadFilter.createCompleteFilter(
232
  ["network.example.user", "network.example.product"],
233
  2,
234
  "desc"
235
);
236

237
const userPayloads = PayloadFilter.applyFilter(payloads, userFilter);
238
const limitedPayloads = PayloadFilter.applyFilter(payloads, limitFilter);
239
const complexResults = PayloadFilter.applyFilter(payloads, complexFilter);
240

241
console.log("Users:", userPayloads.length); // 3
242
console.log("Limited:", limitedPayloads.length); // 3
243
console.log("Complex:", complexResults.length); // 2
244

245
// Hash map utilities
246
class PayloadHashMapManager<T extends Payload> {
247
  private hashMap: PayloadHashMap<T> = {
248
    dataHash: {},
249
    hash: {}
250
  };
251

252
  // Add payload to hash map
253
  addPayload(payload: T, hash: string, dataHash?: string): void {
254
    this.hashMap.hash[hash] = payload;
255
    this.hashMap.dataHash[hash] = dataHash || hash;
256
  }
257

258
  // Get payload by hash
259
  getPayload(hash: string): T | undefined {
260
    return this.hashMap.hash[hash];
261
  }
262

263
  // Get all payloads with same data hash
264
  getPayloadsByDataHash(dataHash: string): T[] {
265
    const hashes = Object.entries(this.hashMap.dataHash)
266
      .filter(([_, dHash]) => dHash === dataHash)
267
      .map(([hash, _]) => hash);
268

269
    return hashes
270
      .map(hash => this.hashMap.hash[hash])
271
      .filter((payload): payload is T => payload !== undefined);
272
  }
273

274
  // Get statistics
275
  getStats(): {
276
    totalPayloads: number;
277
    uniqueDataHashes: number;
278
    duplicateDataHashes: number;
279
  } {
280
    const totalPayloads = Object.keys(this.hashMap.hash).length;
281
    const dataHashes = Object.values(this.hashMap.dataHash);
282
    const uniqueDataHashes = new Set(dataHashes).size;
283
    const duplicateDataHashes = dataHashes.length - uniqueDataHashes;
284

285
    return {
286
      totalPayloads,
287
      uniqueDataHashes,
288
      duplicateDataHashes
289
    };
290
  }
291

292
  // Export hash map
293
  export(): PayloadHashMap<T> {
294
    return { ...this.hashMap };
295
  }
296
}
297

298
// Use hash map manager
299
interface UserPayload extends Payload {
300
  schema: "network.example.user";
301
  name: string;
302
  email: string;
303
}
304

305
const hashMapManager = new PayloadHashMapManager<UserPayload>();
306

307
const user1: UserPayload = {
308
  schema: "network.example.user",
309
  name: "Alice",
310
  email: "alice@example.com"
311
};
312

313
const user2: UserPayload = {
314
  schema: "network.example.user", 
315
  name: "Alice",
316
  email: "alice@example.com"
317
}; // Same data, different hash
318

319
hashMapManager.addPayload(user1, "0x123...", "0xabc...");
320
hashMapManager.addPayload(user2, "0x456...", "0xabc..."); // Same data hash
321

322
const stats = hashMapManager.getStats();
323
console.log("Stats:", stats); // { totalPayloads: 2, uniqueDataHashes: 1, duplicateDataHashes: 1 }
324

325
const duplicates = hashMapManager.getPayloadsByDataHash("0xabc...");
326
console.log("Duplicates:", duplicates.length); // 2
327
```
328

329
### PayloadSet Types
330

331
Types and interfaces for working with payload sets and collections of schema requirements.
332

333
```typescript { .api }
334
/**
335
 * PayloadSet schema constant
336
 */
337
const PayloadSetSchema = "network.xyo.payload.set" as const;
338
type PayloadSetSchema = typeof PayloadSetSchema;
339

340
/**
341
 * Interface defining required and optional schema counts
342
 */
343
interface PayloadSet {
344
  /** Optional schemas with their minimum required counts */  
345
  optional?: Record<string, number>;
346
  /** Required schemas with their minimum required counts */
347
  required?: Record<string, number>;
348
}
349

350
/**
351
 * PayloadSet payload type combining schema requirements with payload structure
352
 */
353
type PayloadSetPayload = Payload<PayloadSet, PayloadSetSchema>;
354
```
355

356
**Usage Examples:**
357

358
```typescript
359
import { 
360
  PayloadSet,
361
  PayloadSetPayload,
362
  PayloadSetSchema,
363
  Payload
364
} from "@xyo-network/payload-model";
365

366
// Define a payload set requirement
367
const apiPayloadSet: PayloadSetPayload = {
368
  schema: PayloadSetSchema,
369
  required: {
370
    "network.example.user": 1,        // At least 1 user payload required
371
    "network.example.session": 1      // At least 1 session payload required
372
  },
373
  optional: {
374
    "network.example.metadata": 2,    // Up to 2 metadata payloads optional
375
    "network.example.config": 1       // Up to 1 config payload optional
376
  }
377
};
378

379
// Utility class for working with payload sets
380
class PayloadSetValidator {
381
  // Validate if a collection of payloads meets the requirements
382
  static validate(payloads: Payload[], requirements: PayloadSet): {
383
    valid: boolean;
384
    missing: string[];
385
    satisfied: string[];
386
    counts: Record<string, number>;
387
  } {
388
    const counts: Record<string, number> = {};
389
    const missing: string[] = [];
390
    const satisfied: string[] = [];
391

392
    // Count payloads by schema
393
    payloads.forEach(payload => {
394
      counts[payload.schema] = (counts[payload.schema] || 0) + 1;
395
    });
396

397
    // Check required schemas
398
    if (requirements.required) {
399
      Object.entries(requirements.required).forEach(([schema, required]) => {
400
        const actual = counts[schema] || 0;
401
        if (actual >= required) {
402
          satisfied.push(schema);
403
        } else {
404
          missing.push(`${schema} (need ${required}, have ${actual})`);
405
        }
406
      });
407
    }
408

409
    // Check optional schemas (they're always satisfied if present)
410
    if (requirements.optional) {
411
      Object.entries(requirements.optional).forEach(([schema, _]) => {
412
        if (counts[schema] > 0) {
413
          satisfied.push(schema);
414
        }
415
      });
416
    }
417

418
    return {
419
      valid: missing.length === 0,
420
      missing,
421
      satisfied,
422
      counts
423
    };
424
  }
425

426
  // Generate a report for payload set validation
427
  static generateReport(payloads: Payload[], requirements: PayloadSet): string {
428
    const result = this.validate(payloads, requirements);
429
    
430
    let report = `Payload Set Validation Report\n`;
431
    report += `================================\n`;
432
    report += `Status: ${result.valid ? 'VALID' : 'INVALID'}\n\n`;
433
    
434
    report += `Schema Counts:\n`;
435
    Object.entries(result.counts).forEach(([schema, count]) => {
436
      report += `  ${schema}: ${count}\n`;
437
    });
438
    
439
    if (result.satisfied.length > 0) {
440
      report += `\nSatisfied Requirements:\n`;
441
      result.satisfied.forEach(schema => {
442
        report += `  ✓ ${schema}\n`;
443
      });
444
    }
445
    
446
    if (result.missing.length > 0) {
447
      report += `\nMissing Requirements:\n`;
448
      result.missing.forEach(requirement => {
449
        report += `  ✗ ${requirement}\n`;
450
      });
451
    }
452
    
453
    return report;
454
  }
455

456
  // Filter payloads that match the set requirements
457
  static filterMatchingPayloads(
458
    allPayloads: Payload[],
459
    requirements: PayloadSet
460
  ): Payload[] {
461
    const requiredSchemas = Object.keys(requirements.required || {});
462
    const optionalSchemas = Object.keys(requirements.optional || {});
463
    const allowedSchemas = new Set([...requiredSchemas, ...optionalSchemas]);
464
    
465
    return allPayloads.filter(payload => 
466
      allowedSchemas.has(payload.schema)
467
    );
468
  }
469
}
470

471
// Usage examples
472
const testPayloads: Payload[] = [
473
  { schema: "network.example.user", name: "Alice", email: "alice@example.com" },
474
  { schema: "network.example.user", name: "Bob", email: "bob@example.com" },
475
  { schema: "network.example.session", sessionId: "sess123", userId: "alice" },
476
  { schema: "network.example.metadata", type: "analytics", data: {} },
477
  { schema: "network.example.config", theme: "dark", lang: "en" },
478
  { schema: "network.example.unknown", data: "should be filtered" }
479
];
480

481
// Validate against payload set requirements
482
const validation = PayloadSetValidator.validate(testPayloads, apiPayloadSet);
483
console.log("Validation result:", validation.valid);
484
console.log("Missing:", validation.missing);
485

486
// Generate detailed report
487
const report = PayloadSetValidator.generateReport(testPayloads, apiPayloadSet);
488
console.log(report);
489

490
// Filter matching payloads
491
const matchingPayloads = PayloadSetValidator.filterMatchingPayloads(
492
  testPayloads, 
493
  apiPayloadSet
494
);
495
console.log("Matching payloads:", matchingPayloads.length); // Excludes unknown schema
496
```
497

498
### Timestamp Utilities
499

500
Utility types and interfaces for working with temporal data in payloads.
501

502
```typescript { .api }
503
/**
504
 * Interface for objects with timestamp information
505
 */
506
interface Timestamp {
507
  timestamp: number;
508
}
509

510
/**
511
 * Utility type to add timestamp field to any object
512
 */
513
type WithTimestamp<T extends EmptyObject = EmptyObject> = T & Timestamp;
514
```
515

516
**Usage Examples:**
517

518
```typescript
519
import { 
520
  Timestamp,
521
  WithTimestamp,
522
  Payload
523
} from "@xyo-network/payload-model";
524

525
// Add timestamps to payload types
526
interface TimestampedUserPayload extends WithTimestamp<Payload> {
527
  schema: "network.example.user";
528
  name: string;
529
  email: string;
530
}
531

532
// Create timestamped payload
533
const timestampedUser: TimestampedUserPayload = {
534
  schema: "network.example.user",
535
  name: "Alice",
536
  email: "alice@example.com",
537
  timestamp: Date.now()
538
};
539

540
// Utility functions for timestamps
541
class TimestampUtils {
542
  // Add timestamp to existing payload
543
  static addTimestamp<T extends Payload>(payload: T): WithTimestamp<T> {
544
    return {
545
      ...payload,
546
      timestamp: Date.now()
547
    };
548
  }
549

550
  // Check if payload has timestamp
551
  static hasTimestamp(payload: unknown): payload is Timestamp {
552
    return typeof payload === 'object' && 
553
           payload !== null && 
554
           'timestamp' in payload && 
555
           typeof (payload as any).timestamp === 'number';
556
  }
557

558
  // Sort payloads by timestamp
559
  static sortByTimestamp<T extends Timestamp>(
560
    payloads: T[],
561
    order: 'asc' | 'desc' = 'desc'
562
  ): T[] {
563
    return [...payloads].sort((a, b) => {
564
      return order === 'desc' 
565
        ? b.timestamp - a.timestamp 
566
        : a.timestamp - b.timestamp;
567
    });
568
  }
569

570
  // Filter payloads by time range
571
  static filterByTimeRange<T extends Timestamp>(
572
    payloads: T[],
573
    startTime: number,
574
    endTime: number
575
  ): T[] {
576
    return payloads.filter(payload => 
577
      payload.timestamp >= startTime && payload.timestamp <= endTime
578
    );
579
  }
580

581
  // Group payloads by time period
582
  static groupByTimePeriod<T extends Timestamp>(
583
    payloads: T[],
584
    periodMs: number
585
  ): Record<string, T[]> {
586
    const groups: Record<string, T[]> = {};
587

588
    payloads.forEach(payload => {
589
      const periodStart = Math.floor(payload.timestamp / periodMs) * periodMs;
590
      const key = periodStart.toString();
591
      
592
      if (!groups[key]) {
593
        groups[key] = [];
594
      }
595
      groups[key].push(payload);
596
    });
597

598
    return groups;
599
  }
600
}
601

602
// Usage examples
603
const users: WithTimestamp<Payload>[] = [
604
  { schema: "network.example.user", name: "Alice", timestamp: Date.now() - 3600000 }, // 1 hour ago
605
  { schema: "network.example.user", name: "Bob", timestamp: Date.now() - 1800000 },   // 30 min ago
606
  { schema: "network.example.user", name: "Charlie", timestamp: Date.now() - 900000 } // 15 min ago
607
];
608

609
// Sort by timestamp
610
const sortedUsers = TimestampUtils.sortByTimestamp(users, 'asc');
611
console.log("Sorted users:", sortedUsers.map(u => u.name)); // ["Alice", "Bob", "Charlie"]
612

613
// Filter recent users (last 45 minutes)
614
const recentUsers = TimestampUtils.filterByTimeRange(
615
  users,
616
  Date.now() - 45 * 60 * 1000,
617
  Date.now()
618
);
619
console.log("Recent users:", recentUsers.length); // 2
620

621
// Group by hour
622
const hourlyGroups = TimestampUtils.groupByTimePeriod(users, 60 * 60 * 1000);
623
console.log("Hourly groups:", Object.keys(hourlyGroups));
624

625
// Add timestamp to existing payload
626
const existingPayload: Payload = {
627
  schema: "network.example.data",
628
  data: "some data"
629
};
630

631
const timestampedPayload = TimestampUtils.addTimestamp(existingPayload);
632
console.log("Timestamped:", timestampedPayload.timestamp);
633
```
634

635
### Advanced Type Manipulation
636

637
Advanced utility types for complex payload type transformations and manipulations.
638

639
```typescript { .api }
640
/**
641
 * Utility types for deep property manipulation (from @xylabs/object)
642
 */
643
type DeepOmitStartsWith<T, Prefix extends string> = T; // Simplified representation
644
type DeepPickStartsWith<T, Prefix extends string> = T; // Simplified representation
645

646
/**
647
 * Empty object type for type safety
648
 */
649
type EmptyObject = Record<string, never>;
650

651
/**
652
 * JSON value type for flexible data storage
653
 */
654
type JsonValue = string | number | boolean | null | JsonValue[] | { [key: string]: JsonValue };
655
```
656

657
**Usage Examples:**
658

659
```typescript
660
import { 
661
  Payload,
662
  WithoutMeta,
663
  WithoutClientMeta,
664
  WithoutStorageMeta,
665
  JsonValue
666
} from "@xyo-network/payload-model";
667

668
// Complex payload with various metadata
669
interface ComplexPayload extends Payload {
670
  schema: "network.example.complex";
671
  data: JsonValue;
672
  
673
  // Client metadata (starts with $)
674
  $userId?: string;
675
  $sessionId?: string;
676
  $clientVersion?: string;
677
  
678
  // Storage metadata (starts with _)
679
  _hash?: string;
680
  _dataHash?: string;
681
  _sequence?: string;
682
  
683
  // Private storage metadata (starts with __)
684
  __internalId?: string;
685
  __debugInfo?: JsonValue;
686
}
687

688
// Create utility class for payload transformation
689
class PayloadTransformer {
690
  // Remove all metadata for external API
691
  static toExternalFormat<T extends Payload>(payload: T): WithoutMeta<T> {
692
    const cleaned = { ...payload };
693
    
694
    // Remove client metadata
695
    Object.keys(cleaned).forEach(key => {
696
      if (key.startsWith('$') || key.startsWith('_')) {
697
        delete (cleaned as any)[key];
698
      }
699
    });
700
    
701
    return cleaned as WithoutMeta<T>;
702
  }
703

704
  // Remove only client metadata
705
  static removeClientMeta<T extends Payload>(payload: T): WithoutClientMeta<T> {
706
    const cleaned = { ...payload };
707
    
708
    Object.keys(cleaned).forEach(key => {
709
      if (key.startsWith('$')) {
710
        delete (cleaned as any)[key];
711
      }
712
    });
713
    
714
    return cleaned as WithoutClientMeta<T>;
715
  }
716

717
  // Extract only metadata
718
  static extractMetadata<T extends Payload>(payload: T): {
719
    client: Record<string, any>;
720
    storage: Record<string, any>;
721
    private: Record<string, any>;
722
  } {
723
    const client: Record<string, any> = {};
724
    const storage: Record<string, any> = {};
725
    const private: Record<string, any> = {};
726

727
    Object.entries(payload).forEach(([key, value]) => {
728
      if (key.startsWith('__')) {
729
        private[key] = value;
730
      } else if (key.startsWith('_')) {
731
        storage[key] = value;
732
      } else if (key.startsWith('$')) {
733
        client[key] = value;
734
      }
735
    });
736

737
    return { client, storage, private };
738
  }
739

740
  // Merge metadata back into payload
741
  static mergeMetadata<T extends Payload>(
742
    payload: T,
743
    metadata: {
744
      client?: Record<string, any>;
745
      storage?: Record<string, any>;
746
      private?: Record<string, any>;
747
    }
748
  ): T {
749
    return {
750
      ...payload,
751
      ...metadata.client,
752
      ...metadata.storage,
753
      ...metadata.private
754
    };
755
  }
756
}
757

758
// Usage examples
759
const complexPayload: ComplexPayload = {
760
  schema: "network.example.complex",
761
  data: { message: "Hello World", count: 42 },
762
  $userId: "user123",
763
  $sessionId: "session456", 
764
  $clientVersion: "1.2.3",
765
  _hash: "0x123...",
766
  _dataHash: "0x456...",
767
  _sequence: "seq789",
768
  __internalId: "internal123",
769
  __debugInfo: { trace: true, level: "debug" }
770
};
771

772
// Transform for external API
773
const externalPayload = PayloadTransformer.toExternalFormat(complexPayload);
774
console.log("External payload keys:", Object.keys(externalPayload));
775
// Only: ["schema", "data"]
776

777
// Remove client metadata only
778
const withoutClient = PayloadTransformer.removeClientMeta(complexPayload);
779
console.log("Without client metadata:", Object.keys(withoutClient));
780
// Includes storage metadata but not client metadata
781

782
// Extract metadata
783
const metadata = PayloadTransformer.extractMetadata(complexPayload);
784
console.log("Client metadata:", metadata.client);
785
console.log("Storage metadata:", metadata.storage);
786
console.log("Private metadata:", metadata.private);
787

788
// Work with JsonValue data
789
function processJsonData(data: JsonValue): string {
790
  if (typeof data === 'string') return data;
791
  if (typeof data === 'number') return data.toString();
792
  if (typeof data === 'boolean') return data ? 'true' : 'false';  
793
  if (data === null) return 'null';
794
  if (Array.isArray(data)) return `[${data.length} items]`;
795
  if (typeof data === 'object') return `{${Object.keys(data).length} props}`;
796
  return 'unknown';
797
}
798

799
const processed = processJsonData(complexPayload.data);
800
console.log("Processed data:", processed); // "{2 props}"
801
```
802

803
## Advanced Usage Patterns
804

805
### Payload Processing Pipeline
806

807
```typescript
808
import { 
809
  Payload,
810
  PayloadFindFilter,
811
  PayloadValueExpression,
812
  WithTimestamp
813
} from "@xyo-network/payload-model";
814

815
// Create a comprehensive payload processing pipeline
816
class PayloadPipeline<T extends Payload> {
817
  private processors: Array<(payload: T) => T> = [];
818
  private validators: Array<(payload: T) => boolean> = [];
819
  private extractors: Map<string, PayloadValueExpression<T, any, any>> = new Map();
820

821
  // Add processing step
822
  addProcessor(processor: (payload: T) => T): this {
823
    this.processors.push(processor);
824
    return this;
825
  }
826

827
  // Add validation step
828
  addValidator(validator: (payload: T) => boolean): this {
829
    this.validators.push(validator);
830
    return this;
831
  }
832

833
  // Add value extractor
834
  addExtractor<K extends keyof T>(
835
    name: string, 
836
    extractor: PayloadValueExpression<T, K, any>
837
  ): this {
838
    this.extractors.set(name, extractor);
839
    return this;
840
  }
841

842
  // Process single payload
843
  process(payload: T): {
844
    success: boolean;
845
    payload?: T;
846
    extracted: Record<string, any>;
847
    errors: string[];
848
  } {
849
    const errors: string[] = [];
850
    const extracted: Record<string, any> = {};
851
    let currentPayload = payload;
852

853
    // Apply processors
854
    try {
855
      for (const processor of this.processors) {
856
        currentPayload = processor(currentPayload);
857
      }
858
    } catch (error) {
859
      errors.push(`Processing error: ${error}`);
860
      return { success: false, extracted, errors };
861
    }
862

863
    // Apply validators
864
    for (const validator of this.validators) {
865
      if (!validator(currentPayload)) {
866
        errors.push("Validation failed");
867
      }
868
    }
869

870
    // Extract values
871
    for (const [name, extractor] of this.extractors) {
872
      try {
873
        extracted[name] = extractor(currentPayload);
874
      } catch (error) {
875
        errors.push(`Extraction error for ${name}: ${error}`);
876
      }
877
    }
878

879
    return {
880
      success: errors.length === 0,
881
      payload: errors.length === 0 ? currentPayload : undefined,
882
      extracted,
883
      errors
884
    };
885
  }
886

887
  // Process batch of payloads
888
  processBatch(payloads: T[]): {
889
    successful: Array<{ payload: T; extracted: Record<string, any> }>;
890
    failed: Array<{ original: T; errors: string[] }>;
891
  } {
892
    const successful: Array<{ payload: T; extracted: Record<string, any> }> = [];
893
    const failed: Array<{ original: T; errors: string[] }> = [];
894

895
    for (const payload of payloads) {
896
      const result = this.process(payload);
897
      if (result.success && result.payload) {
898
        successful.push({ payload: result.payload, extracted: result.extracted });
899
      } else {
900
        failed.push({ original: payload, errors: result.errors });
901
      }
902
    }
903

904
    return { successful, failed };
905
  }
906
}
907

908
// Usage example
909
interface UserPayload extends WithTimestamp<Payload> {
910
  schema: "network.example.user";
911
  name: string;
912
  email: string;
913
  age: number;
914
}
915

916
const userPipeline = new PayloadPipeline<UserPayload>()
917
  .addProcessor((payload) => ({
918
    ...payload,
919
    name: payload.name.trim(),
920
    email: payload.email.toLowerCase()
921
  }))
922
  .addValidator((payload) => payload.age >= 18)
923
  .addValidator((payload) => payload.email.includes('@'))
924
  .addExtractor('displayName', (payload) => `${payload.name} (${payload.age})`)
925
  .addExtractor('domain', (payload) => payload.email.split('@')[1]);
926

927
const testUsers: UserPayload[] = [
928
  {
929
    schema: "network.example.user",
930
    name: " Alice ",
931
    email: "ALICE@EXAMPLE.COM",
932
    age: 25,
933
    timestamp: Date.now()
934
  },
935
  {
936
    schema: "network.example.user", 
937
    name: "Bob",
938
    email: "bob@test",
939
    age: 16,
940
    timestamp: Date.now()
941
  }
942
];
943

944
const results = userPipeline.processBatch(testUsers);
945
console.log("Successful:", results.successful.length);
946
console.log("Failed:", results.failed.length);
947

948
results.successful.forEach(({ payload, extracted }) => {
949
  console.log(`Processed: ${extracted.displayName} from ${extracted.domain}`);
950
});
951
```
952

953
## Types Reference
954

955
### Value Expression Types
956

957
- **`PayloadProperty<T>`**: Type representing all property keys of a payload
958
- **`PayloadValue<T, Key>`**: Type representing the value of a specific payload property
959
- **`PayloadValueExpression<T, Key, TValue>`**: Function type for extracting property values
960

961
### Collection Types
962

963
- **`PayloadFindFilter`**: Interface for payload filtering operations
964
- **`PayloadHashMap<TPayload, TId>`**: Interface for efficient payload storage and lookup
965

966
### Timestamp Types
967

968
- **`Timestamp`**: Interface for objects with timestamp information
969
- **`WithTimestamp<T>`**: Utility type to add timestamp field to any object
970

971
### Utility Types
972

973
- **`EmptyObject`**: Empty object type for type safety
974
- **`JsonValue`**: JSON value type for flexible data storage
975
- **`DeepOmitStartsWith<T, Prefix>`**: Remove properties starting with prefix
976
- **`DeepPickStartsWith<T, Prefix>`**: Select properties starting with prefix
977

978
### Filter Options
979

980
- **`PayloadFindFilter.limit`**: Maximum number of results
981
- **`PayloadFindFilter.order`**: Sort order ('desc' | 'asc')
982
- **`PayloadFindFilter.schema`**: Schema filter (string or string array)
983

984
### Hash Map Structure
985

986
- **`PayloadHashMap.dataHash`**: Data hash mappings (Record<TId, TId>)
987
- **`PayloadHashMap.hash`**: Complete payload mappings (Record<TId, TPayload>)