0
# Traffic Analysis and Detection
1

2
Core functionality for analyzing API traffic, detecting sensitive data, identifying security vulnerabilities, and processing GraphQL operations. The analysis engine provides real-time security insights and automated vulnerability detection.
3

4
## Capabilities
5

6
### Traffic Processing
7

8
#### Processed Trace Data
9

10
Comprehensive security analysis results for API traces including vulnerability detection and data classification.
11

12
```typescript { .api }
13
interface ProcessedTraceData {
14
  /** Whether the request should be blocked */
15
  block: boolean;
16
  /** XSS vulnerability detection results */
17
  xssDetected: Record<string, string>;
18
  /** SQL injection vulnerability detection results */
19
  sqliDetected: Record<string, [string, string]>;
20
  /** Sensitive data detection results by field path */
21
  sensitiveDataDetected: Record<string, string[]>;
22
  /** Data type classifications by field path */
23
  dataTypes: Record<string, string[]>;
24
  /** Request content type */
25
  requestContentType: string;
26
  /** Response content type */
27
  responseContentType: string;
28
  /** OpenAPI specification validation errors */
29
  validationErrors: Record<string, string[]>;
30
  /** GraphQL operation analysis results */
31
  graphQlData: GraphQlData[];
32
}
33
```
34

35
**Usage Examples:**
36

37
```typescript
38
import { ProcessedTraceData } from "@metlo/common";
39

40
// Analyze processed trace data
41
function analyzeTraceResults(processed: ProcessedTraceData) {
42
  // Check for blocking conditions
43
  if (processed.block) {
44
    console.log("Request blocked due to security policy");
45
  }
46
  
47
  // Review XSS detections
48
  Object.entries(processed.xssDetected).forEach(([field, pattern]) => {
49
    console.log(`XSS detected in ${field}: ${pattern}`);
50
  });
51
  
52
  // Review SQL injection detections
53
  Object.entries(processed.sqliDetected).forEach(([field, [query, type]]) => {
54
    console.log(`SQL injection in ${field}: ${query} (type: ${type})`);
55
  });
56
  
57
  // Review sensitive data
58
  Object.entries(processed.sensitiveDataDetected).forEach(([path, dataClasses]) => {
59
    console.log(`Sensitive data at ${path}:`, dataClasses);
60
  });
61
  
62
  // Check validation errors
63
  Object.entries(processed.validationErrors).forEach(([field, errors]) => {
64
    console.log(`Validation errors in ${field}:`, errors);
65
  });
66
}
67
```
68

69
### Sensitive Data Detection
70

71
#### Data Field Classification
72

73
Detailed information about detected sensitive data fields with classification and metadata.
74

75
```typescript { .api }
76
interface DataField {
77
  /** Unique identifier for the data field */
78
  uuid: string;
79
  /** Classified sensitive data types (e.g., "Email", "Credit Card Number") */
80
  dataClasses: string[];
81
  /** JSON path to the field (e.g., "request.body.user.email") */
82
  dataPath: string;
83
  /** Section of HTTP message containing the field */
84
  dataSection: DataSection;
85
  /** Detected data type */
86
  dataType: DataType;
87
  /** Sensitivity classification tag */
88
  dataTag: DataTag;
89
  /** Fields manually marked as false positives */
90
  falsePositives: string[];
91
  /** Data classes identified by automated scanners */
92
  scannerIdentified: string[];
93
  /** Pattern matches used for classification */
94
  matches: Record<string, string[]>;
95
  /** Associated API endpoint UUID */
96
  apiEndpointUuid: string;
97
  /** HTTP status code where field was detected */
98
  statusCode: number;
99
  /** Content type of the HTTP message */
100
  contentType: string;
101
  /** Whether the field can contain null values */
102
  isNullable: boolean;
103
  /** Business entity associated with the field */
104
  entity: string;
105
  /** When the field was first detected */
106
  createdAt: Date;
107
  /** When the field was last updated */
108
  updatedAt: Date;
109
}
110
```
111

112
**Usage Examples:**
113

114
```typescript
115
import { DataField, DataSection, DataType, DataTag } from "@metlo/common";
116

117
// Analyze data fields for PII exposure
118
function analyzePIIExposure(dataFields: DataField[]) {
119
  const piiFields = dataFields.filter(field => 
120
    field.dataTag === DataTag.PII && 
121
    field.dataSection === DataSection.RESPONSE_BODY
122
  );
123
  
124
  piiFields.forEach(field => {
125
    console.log(`PII exposure detected:`);
126
    console.log(`  Path: ${field.dataPath}`);
127
    console.log(`  Data Classes: ${field.dataClasses.join(", ")}`);
128
    console.log(`  Endpoint: ${field.apiEndpointUuid}`);
129
    console.log(`  Entity: ${field.entity}`);
130
  });
131
}
132

133
// Check for credit card data in requests
134
function findCreditCardData(dataFields: DataField[]) {
135
  return dataFields.filter(field => 
136
    field.dataClasses.includes("Credit Card Number") &&
137
    field.dataSection === DataSection.REQUEST_BODY
138
  );
139
}
140
```
141

142
#### Data Classification Enums
143

144
Built-in sensitive data classifications and categories.
145

146
```typescript { .api }
147
enum __DataClass_INTERNAL__ {
148
  EMAIL = "Email",
149
  CREDIT_CARD = "Credit Card Number", 
150
  SSN = "Social Security Number",
151
  PHONE_NUMBER = "Phone Number",
152
  IP_ADDRESS = "IP Address",
153
  COORDINATE = "Geographic Coordinates",
154
  VIN = "Vehicle Identification Number", 
155
  ADDRESS = "Address",
156
  DOB = "Date of Birth",
157
  DL_NUMBER = "Driver License Number",
158
  AADHAR_NUMBER = "Aadhar Number",
159
  BRAZIL_CPF = "Brazil CPF"
160
}
161

162
enum DataTag {
163
  PII = "PII"
164
}
165

166
enum DataType {
167
  INTEGER = "integer",
168
  NUMBER = "number",
169
  STRING = "string", 
170
  BOOLEAN = "boolean",
171
  OBJECT = "object",
172
  ARRAY = "array",
173
  UNKNOWN = "unknown"
174
}
175

176
enum DataSection {
177
  REQUEST_PATH = "reqPath",
178
  REQUEST_QUERY = "reqQuery",
179
  REQUEST_HEADER = "reqHeaders", 
180
  REQUEST_BODY = "reqBody",
181
  RESPONSE_HEADER = "resHeaders",
182
  RESPONSE_BODY = "resBody"
183
}
184
```
185

186
### GraphQL Analysis
187

188
#### GraphQL Operation Data
189

190
Analysis results for GraphQL operations including queries, mutations, and subscriptions.
191

192
```typescript { .api }
193
interface GraphQlData {
194
  /** Name of the GraphQL operation */
195
  operationName: string;
196
  /** Parsed GraphQL operations */
197
  operations: Operation[];
198
}
199

200
interface Operation {
201
  /** Operation name */
202
  operationName: string;
203
  /** Type of GraphQL operation */
204
  operationType: GraphQlOperation;
205
  /** Selected fields and nested items */
206
  items: OperationItem[];
207
  /** Operation variables */
208
  variables: Variable[];
209
}
210

211
interface OperationItem {
212
  /** Field name */
213
  name: string;
214
  /** Field alias if specified */
215
  alias: string;
216
  /** Field arguments */
217
  arguments: string[];
218
  /** Nested selection items */
219
  items: OperationItem[];
220
}
221

222
interface Variable {
223
  /** Variable name */
224
  name: string;
225
  /** Variable type definition */
226
  varType: string;
227
}
228

229
enum GraphQlOperation {
230
  QUERY = "query",
231
  MUTATION = "mutation", 
232
  SUBSCRIPTION = "subscription"
233
}
234
```
235

236
**Usage Examples:**
237

238
```typescript
239
import { GraphQlData, GraphQlOperation } from "@metlo/common";
240

241
// Analyze GraphQL operations for security concerns
242
function analyzeGraphQLOperations(graphqlData: GraphQlData[]) {
243
  graphqlData.forEach(data => {
244
    data.operations.forEach(operation => {
245
      console.log(`Operation: ${operation.operationName} (${operation.operationType})`);
246
      
247
      // Check for potentially dangerous mutations
248
      if (operation.operationType === GraphQlOperation.MUTATION) {
249
        console.log("Mutation detected - reviewing for authorization");
250
        analyzeOperationItems(operation.items);
251
      }
252
      
253
      // Analyze variables for injection risks
254
      operation.variables.forEach(variable => {
255
        console.log(`Variable: ${variable.name} (${variable.varType})`);
256
      });
257
    });
258
  });
259
}
260

261
function analyzeOperationItems(items: OperationItem[], depth = 0) {
262
  items.forEach(item => {
263
    const indent = "  ".repeat(depth);
264
    console.log(`${indent}Field: ${item.name} (alias: ${item.alias})`);
265
    
266
    if (item.arguments.length > 0) {
267
      console.log(`${indent}Arguments:`, item.arguments);
268
    }
269
    
270
    if (item.items.length > 0) {
271
      analyzeOperationItems(item.items, depth + 1);
272
    }
273
  });
274
}
275
```
276

277
### Encryption and Security
278

279
#### Encryption Configuration
280

281
Configuration for encrypting sensitive data in traces and responses.
282

283
```typescript { .api }
284
interface Encryption {
285
  /** Encryption key identifier */
286
  key: string;
287
  /** Generated initialization vectors by field */
288
  generatedIvs: Record<string, number[]>;
289
}
290
```
291

292
**Usage Examples:**
293

294
```typescript
295
import { Encryption } from "@metlo/common";
296

297
// Configure encryption for sensitive fields
298
function setupFieldEncryption(sensitiveFields: string[]): Encryption {
299
  const encryption: Encryption = {
300
    key: "encryption-key-id-123",
301
    generatedIvs: {}
302
  };
303
  
304
  // Generate IVs for each sensitive field
305
  sensitiveFields.forEach(field => {
306
    encryption.generatedIvs[field] = generateRandomIV();
307
  });
308
  
309
  return encryption;
310
}
311

312
function generateRandomIV(): number[] {
313
  return Array.from({ length: 16 }, () => Math.floor(Math.random() * 256));
314
}
315
```
316

317
### Analysis Types and Configuration
318

319
#### Analysis Type
320

321
Configuration for different levels of traffic analysis.
322

323
```typescript { .api }
324
enum AnalysisType {
325
  /** Full comprehensive analysis including content inspection */
326
  FULL = "full",
327
  /** Partial analysis for performance optimization */
328
  PARTIAL = "partial"
329
}
330
```
331

332
#### Trace Parameters
333

334
Complete parameter set for trace analysis including request, response, and metadata.
335

336
```typescript { .api }
337
interface TraceParams {
338
  /** HTTP request data */
339
  request: Request;
340
  /** HTTP response data */
341
  response: Response;
342
  /** Network traffic metadata */
343
  meta: Meta;
344
  /** Security analysis results */
345
  processedTraceData?: ProcessedTraceData;
346
  /** Whether sensitive data was redacted */
347
  redacted?: boolean;
348
  /** Authentication session information */
349
  sessionMeta?: SessionMeta;
350
  /** Encryption configuration for sensitive data */
351
  encryption?: Encryption;
352
  /** Level of analysis to perform */
353
  analysisType?: AnalysisType;
354
  /** GraphQL-specific request paths */
355
  graphqlPaths?: string[];
356
}
357
```
358

359
**Usage Examples:**
360

361
```typescript
362
import { TraceParams, AnalysisType, RestMethod } from "@metlo/common";
363

364
// Prepare trace for analysis
365
function prepareTraceAnalysis(
366
  requestData: any,
367
  responseData: any, 
368
  networkMeta: any
369
): TraceParams {
370
  return {
371
    request: {
372
      url: {
373
        host: requestData.host,
374
        path: requestData.path,
375
        parameters: requestData.queryParams
376
      },
377
      headers: requestData.headers,
378
      body: requestData.body,
379
      method: requestData.method as RestMethod
380
    },
381
    response: {
382
      status: responseData.statusCode,
383
      headers: responseData.headers,
384
      body: responseData.body
385
    },
386
    meta: {
387
      incoming: true,
388
      source: networkMeta.sourceIP,
389
      sourcePort: networkMeta.sourcePort,
390
      destination: networkMeta.destIP,
391
      destinationPort: networkMeta.destPort
392
    },
393
    analysisType: AnalysisType.FULL,
394
    redacted: false
395
  };
396
}
397
```
398

399
### Vulnerability Detection
400

401
#### Security Alert Generation
402

403
Analysis results are used to generate security alerts for detected vulnerabilities.
404

405
```typescript { .api }
406
// Alert types generated from analysis
407
enum AlertType {
408
  NEW_ENDPOINT = "New Endpoint Detected",
409
  PII_DATA_DETECTED = "PII Data Detected", 
410
  OPEN_API_SPEC_DIFF = "Open API Spec Diff",
411
  QUERY_SENSITIVE_DATA = "Sensitive Data in Query Params",
412
  PATH_SENSITIVE_DATA = "Sensitive Data in Path Params",
413
  BASIC_AUTHENTICATION_DETECTED = "Basic Authentication Detected",
414
  UNSECURED_ENDPOINT_DETECTED = "Endpoint not secured by SSL",
415
  UNAUTHENTICATED_ENDPOINT_SENSITIVE_DATA = "Unauthenticated Endpoint returning Sensitive Data"
416
}
417

418
// Vulnerability alert types for security issues
419
const VULNERABILITY_ALERT_TYPES = [
420
  AlertType.OPEN_API_SPEC_DIFF,
421
  AlertType.QUERY_SENSITIVE_DATA, 
422
  AlertType.PATH_SENSITIVE_DATA,
423
  AlertType.BASIC_AUTHENTICATION_DETECTED,
424
  AlertType.UNSECURED_ENDPOINT_DETECTED,
425
  AlertType.UNAUTHENTICATED_ENDPOINT_SENSITIVE_DATA
426
];
427
```
428

429
**Usage Examples:**
430

431
```typescript
432
import { AlertType, VULNERABILITY_ALERT_TYPES, ProcessedTraceData } from "@metlo/common";
433

434
// Generate alerts based on analysis results
435
function generateSecurityAlerts(processed: ProcessedTraceData, endpoint: ApiEndpoint): AlertType[] {
436
  const alerts: AlertType[] = [];
437
  
438
  // Check for PII data exposure
439
  if (Object.keys(processed.sensitiveDataDetected).length > 0) {
440
    alerts.push(AlertType.PII_DATA_DETECTED);
441
    
442
    // Check if unauthenticated endpoint returns PII
443
    if (!endpoint.isAuthenticatedDetected) {
444
      alerts.push(AlertType.UNAUTHENTICATED_ENDPOINT_SENSITIVE_DATA);
445
    }
446
  }
447
  
448
  // Check for sensitive data in query parameters
449
  Object.entries(processed.sensitiveDataDetected).forEach(([path, dataClasses]) => {
450
    if (path.includes("reqQuery")) {
451
      alerts.push(AlertType.QUERY_SENSITIVE_DATA);
452
    }
453
    if (path.includes("reqPath")) {
454
      alerts.push(AlertType.PATH_SENSITIVE_DATA);
455
    }
456
  });
457
  
458
  return alerts;
459
}
460

461
// Filter vulnerability alerts
462
function getVulnerabilityAlerts(allAlerts: AlertType[]): AlertType[] {
463
  return allAlerts.filter(alert => VULNERABILITY_ALERT_TYPES.includes(alert));
464
}
465
```
466

467
### Data Classification
468

469
#### Custom Data Classes
470

471
Define custom data classification rules beyond built-in types.
472

473
```typescript { .api }
474
interface DataClass {
475
  /** Name of the data classification */
476
  className: string;
477
  /** Risk severity level */
478
  severity: RiskScore;
479
  /** Regular expression pattern for detection */
480
  regex?: string;
481
  /** Key-based regex for field name matching */
482
  keyRegex?: string;
483
  /** Short display name */
484
  shortName?: string;
485
}
486
```
487

488
**Usage Examples:**
489

490
```typescript
491
import { DataClass, RiskScore } from "@metlo/common";
492

493
// Define custom data classes for organization-specific data
494
const customDataClasses: DataClass[] = [
495
  {
496
    className: "Employee ID",
497
    severity: RiskScore.MEDIUM,
498
    regex: "^EMP\\d{6}$",
499
    keyRegex: "(employee|emp)_?(id|number)",
500
    shortName: "EmpID"
501
  },
502
  {
503
    className: "Customer Account Number", 
504
    severity: RiskScore.HIGH,
505
    regex: "^ACCT\\d{10}$",
506
    keyRegex: "(account|acct)_?(number|num|id)",
507
    shortName: "AcctNum"
508
  },
509
  {
510
    className: "Internal API Key",
511
    severity: RiskScore.HIGH, 
512
    regex: "^sk_[a-zA-Z0-9]{32}$",
513
    keyRegex: "(api|secret)_?key",
514
    shortName: "APIKey"
515
  }
516
];
517

518
// Apply custom data classification
519
function classifyCustomData(fieldValue: string, fieldName: string): string[] {
520
  const matches: string[] = [];
521
  
522
  customDataClasses.forEach(dataClass => {
523
    // Check value pattern
524
    if (dataClass.regex && new RegExp(dataClass.regex).test(fieldValue)) {
525
      matches.push(dataClass.className);
526
    }
527
    
528
    // Check field name pattern
529
    if (dataClass.keyRegex && new RegExp(dataClass.keyRegex, 'i').test(fieldName)) {
530
      matches.push(dataClass.className);
531
    }
532
  });
533
  
534
  return matches;
535
}
536
```
537

538
### Performance and Optimization
539

540
#### Field Blocking Configuration
541

542
Configure which fields to block from analysis for performance optimization.
543

544
```typescript { .api }
545
interface DisabledPathSection {
546
  /** Request query parameters to skip */
547
  reqQuery: string[];
548
  /** Request headers to skip */
549
  reqHeaders: string[];
550
  /** Request body paths to skip */
551
  reqBody: string[];
552
  /** Response headers to skip */ 
553
  resHeaders: string[];
554
  /** Response body paths to skip */
555
  resBody: string[];
556
}
557

558
interface BlockFieldEntry {
559
  /** URL path pattern */
560
  path: string;
561
  /** Regex pattern for path matching */
562
  pathRegex: string;
563
  /** HTTP method */
564
  method: DisableRestMethod;
565
  /** Number of path parameters */
566
  numberParams: number;
567
  /** Fields to disable in analysis */
568
  disabledPaths: DisabledPathSection;
569
}
570

571
enum DisableRestMethod {
572
  GET = "GET",
573
  HEAD = "HEAD",
574
  POST = "POST", 
575
  PUT = "PUT",
576
  PATCH = "PATCH",
577
  DELETE = "DELETE",
578
  CONNECT = "CONNECT",
579
  OPTIONS = "OPTIONS",
580
  TRACE = "TRACE",
581
  ALL = "ALL"
582
}
583
```
584

585
**Usage Examples:**
586

587
```typescript
588
import { BlockFieldEntry, DisableRestMethod } from "@metlo/common";
589

590
// Configure analysis exclusions for performance
591
const analysisExclusions: BlockFieldEntry[] = [
592
  {
593
    path: "/api/health",
594
    pathRegex: "^/api/health$",
595
    method: DisableRestMethod.GET,
596
    numberParams: 0,
597
    disabledPaths: {
598
      reqQuery: [],
599
      reqHeaders: ["User-Agent", "Accept"],
600
      reqBody: [],
601
      resHeaders: ["Server", "Date"],
602
      resBody: []
603
    }
604
  },
605
  {
606
    path: "/api/logs",
607
    pathRegex: "^/api/logs.*",
608
    method: DisableRestMethod.ALL,
609
    numberParams: 0,
610
    disabledPaths: {
611
      reqQuery: ["timestamp", "level"],
612
      reqHeaders: [],
613
      reqBody: ["metadata", "context"],
614
      resHeaders: [],
615
      resBody: ["stackTrace", "internalIds"]
616
    }
617
  }
618
];
619
```