0
# Platform Integration and Ingestors
1

2
Traffic collection and platform integration capabilities for various environments and programming languages. The integration layer provides flexible ingestors and configuration options for collecting API traffic from diverse infrastructure setups.
3

4
## Capabilities
5

6
### Traffic Collection Parameters
7

8
#### Trace Parameters
9

10
Complete parameter set for collecting and forwarding API traffic to the Metlo backend.
11

12
```typescript { .api }
13
interface TraceParams {
14
  /** HTTP request data */
15
  request: Request;
16
  /** HTTP response data */
17
  response: Response;
18
  /** Network traffic metadata */
19
  meta: Meta;
20
  /** Pre-processed security analysis results */
21
  processedTraceData?: ProcessedTraceData;
22
  /** Whether sensitive data was redacted from the trace */
23
  redacted?: boolean;
24
  /** Session and authentication metadata */
25
  sessionMeta?: SessionMeta;
26
  /** Encryption configuration for sensitive fields */
27
  encryption?: Encryption;
28
  /** Type of analysis requested */
29
  analysisType?: AnalysisType;
30
  /** GraphQL-specific operation paths */
31
  graphqlPaths?: string[];
32
}
33
```
34

35
**Usage Examples:**
36

37
```typescript
38
import { TraceParams, RestMethod, AnalysisType } from "@metlo/common";
39

40
// Collect trace from Express.js middleware
41
function collectExpressTrace(req: any, res: any, responseBody: string): TraceParams {
42
  return {
43
    request: {
44
      url: {
45
        host: req.get('host'),
46
        path: req.path,
47
        parameters: Object.entries(req.query).map(([name, value]) => ({
48
          name,
49
          value: String(value)
50
        }))
51
      },
52
      headers: Object.entries(req.headers).map(([name, value]) => ({
53
        name,
54
        value: String(value)
55
      })),
56
      body: JSON.stringify(req.body),
57
      method: req.method as RestMethod
58
    },
59
    response: {
60
      status: res.statusCode,
61
      headers: Object.entries(res.getHeaders()).map(([name, value]) => ({
62
        name,
63
        value: String(value)
64
      })),
65
      body: responseBody
66
    },
67
    meta: {
68
      incoming: true,
69
      source: req.ip || req.connection.remoteAddress,
70
      sourcePort: String(req.connection.remotePort),
71
      destination: req.connection.localAddress,
72
      destinationPort: String(req.connection.localPort)
73
    },
74
    analysisType: AnalysisType.FULL
75
  };
76
}
77
```
78

79
### Network Metadata
80

81
#### Traffic Metadata
82

83
Network-level information about API traffic for security analysis and monitoring.
84

85
```typescript { .api }
86
interface Meta {
87
  /** Whether traffic is incoming to the monitored service */
88
  incoming: boolean;
89
  /** Source IP address */
90
  source: string;
91
  /** Source port number as string */
92
  sourcePort: string;
93
  /** Destination IP address */
94
  destination: string;
95
  /** Destination port number as string */
96
  destinationPort: string;
97
  /** Original source IP before any proxy/load balancer */
98
  originalSource?: string;
99
}
100
```
101

102
**Usage Examples:**
103

104
```typescript
105
import { Meta } from "@metlo/common";
106

107
// Extract metadata from various proxy headers
108
function extractNetworkMeta(req: any): Meta {
109
  // Handle X-Forwarded-For and similar proxy headers
110
  const forwardedFor = req.headers['x-forwarded-for'];
111
  const realIP = req.headers['x-real-ip'];
112
  const originalSource = forwardedFor ? forwardedFor.split(',')[0].trim() : realIP;
113
  
114
  return {
115
    incoming: true,
116
    source: req.connection.remoteAddress || req.ip,
117
    sourcePort: String(req.connection.remotePort || 0),
118
    destination: req.connection.localAddress,
119
    destinationPort: String(req.connection.localPort || req.socket.localPort),
120
    originalSource: originalSource || undefined
121
  };
122
}
123

124
// Handle load balancer metadata
125
function extractLoadBalancerMeta(headers: Record<string, string>): Partial<Meta> {
126
  return {
127
    originalSource: headers['x-forwarded-for']?.split(',')[0].trim(),
128
    // AWS ALB specific
129
    ...(headers['x-amzn-trace-id'] && {
130
      source: headers['x-forwarded-for']?.split(',')[0].trim()
131
    })
132
  };
133
}
134
```
135

136
### Session and Authentication
137

138
#### Session Metadata
139

140
Authentication and session information for security context.
141

142
```typescript { .api }
143
interface SessionMeta {
144
  /** Whether any authentication was provided with the request */
145
  authenticationProvided: boolean;
146
  /** Whether the provided authentication was valid */
147
  authenticationSuccessful: boolean;
148
  /** Type of authentication method used */
149
  authType: AuthType;
150
  /** Unique identifier for the user session */
151
  uniqueSessionKey?: string;
152
  /** Authenticated user identifier */
153
  user?: string;
154
}
155

156
enum AuthType {
157
  BASIC = "basic",
158
  HEADER = "header",
159
  JWT = "jwt",
160
  SESSION_COOKIE = "session_cookie"
161
}
162
```
163

164
**Usage Examples:**
165

166
```typescript
167
import { SessionMeta, AuthType } from "@metlo/common";
168

169
// Extract session metadata from different auth types
170
function extractSessionMeta(req: any): SessionMeta {
171
  // Check for JWT token
172
  const authHeader = req.headers.authorization;
173
  if (authHeader?.startsWith('Bearer ')) {
174
    const token = authHeader.substring(7);
175
    try {
176
      const decoded = jwt.decode(token);
177
      return {
178
        authenticationProvided: true,
179
        authenticationSuccessful: !!decoded,
180
        authType: AuthType.JWT,
181
        uniqueSessionKey: decoded?.jti || decoded?.sessionId,
182
        user: decoded?.sub || decoded?.userId
183
      };
184
    } catch (error) {
185
      return {
186
        authenticationProvided: true,
187
        authenticationSuccessful: false,
188
        authType: AuthType.JWT
189
      };
190
    }
191
  }
192
  
193
  // Check for session cookie
194
  const sessionCookie = req.cookies?.sessionId;
195
  if (sessionCookie) {
196
    return {
197
      authenticationProvided: true,
198
      authenticationSuccessful: true,
199
      authType: AuthType.SESSION_COOKIE,
200
      uniqueSessionKey: sessionCookie,
201
      user: req.user?.id
202
    };
203
  }
204
  
205
  // Check for basic auth
206
  if (authHeader?.startsWith('Basic ')) {
207
    return {
208
      authenticationProvided: true,
209
      authenticationSuccessful: !!req.user,
210
      authType: AuthType.BASIC,
211
      user: req.user?.username
212
    };
213
  }
214
  
215
  // No authentication
216
  return {
217
    authenticationProvided: false,
218
    authenticationSuccessful: false,
219
    authType: AuthType.HEADER
220
  };
221
}
222
```
223

224
### Platform Configuration
225

226
#### API Key Management
227

228
API key configuration for accessing Metlo backend services.
229

230
```typescript { .api }
231
interface ApiKey {
232
  /** Human-readable name for the API key */
233
  name: string;
234
  /** Unique identifier for the key */
235
  identifier: string;
236
  /** When the key was created (ISO string) */
237
  created: string;
238
  /** What the key is used for */
239
  for: API_KEY_TYPE;
240
}
241

242
enum API_KEY_TYPE {
243
  GCP = "GCP",
244
  AWS = "AWS", 
245
  GENERIC = "GENERIC",
246
  ONBOARDING = "ONBOARDING"
247
}
248
```
249

250
**Usage Examples:**
251

252
```typescript
253
import { ApiKey, API_KEY_TYPE } from "@metlo/common";
254

255
// Create API keys for different integration types
256
const integrationKeys: ApiKey[] = [
257
  {
258
    name: "Production AWS Integration",
259
    identifier: "sk_live_aws_prod_abc123",
260
    created: new Date().toISOString(),
261
    for: API_KEY_TYPE.AWS
262
  },
263
  {
264
    name: "GCP Staging Environment",
265
    identifier: "sk_test_gcp_staging_def456", 
266
    created: new Date().toISOString(),
267
    for: API_KEY_TYPE.GCP
268
  },
269
  {
270
    name: "Generic Application Key",
271
    identifier: "sk_generic_app_ghi789",
272
    created: new Date().toISOString(),
273
    for: API_KEY_TYPE.GENERIC
274
  }
275
];
276

277
// Validate API key for integration
278
function validateIntegrationKey(keyIdentifier: string, expectedType: API_KEY_TYPE): boolean {
279
  const key = integrationKeys.find(k => k.identifier === keyIdentifier);
280
  return key?.for === expectedType || key?.for === API_KEY_TYPE.GENERIC;
281
}
282
```
283

284
#### Connection Types
285

286
Supported integration platforms and environments.
287

288
```typescript { .api }
289
enum ConnectionType {
290
  AWS = "AWS",
291
  GCP = "GCP",
292
  PYTHON = "PYTHON",
293
  NODE = "NODE", 
294
  JAVA = "JAVA",
295
  GOLANG = "GOLANG",
296
  KUBERNETES = "KUBERNETES",
297
  DOCKERCOMPOSE = "DOCKERCOMPOSE",
298
  BURPSUITE = "BURPSUITE"
299
}
300
```
301

302
### Cloud Provider Integration
303

304
#### AWS Integration
305

306
AWS-specific configuration and metadata for traffic mirroring and collection.
307

308
```typescript { .api }
309
enum AWS_SOURCE_TYPE {
310
  INSTANCE,
311
  NETWORK_INTERFACE
312
}
313

314
// AWS-specific trace collection
315
interface AWSTraceConfig {
316
  sourceType: AWS_SOURCE_TYPE;
317
  instanceId?: string;
318
  networkInterfaceId?: string;
319
  region: string;
320
  vpcId: string;
321
}
322
```
323

324
**Usage Examples:**
325

326
```typescript
327
import { AWS_SOURCE_TYPE, ConnectionType } from "@metlo/common";
328

329
// Configure AWS traffic mirroring
330
function configureAWSMirroring(config: {
331
  sourceInstanceId?: string;
332
  sourceENIId?: string;
333
  targetENIId: string;
334
  region: string;
335
}) {
336
  const sourceType = config.sourceInstanceId 
337
    ? AWS_SOURCE_TYPE.INSTANCE 
338
    : AWS_SOURCE_TYPE.NETWORK_INTERFACE;
339
    
340
  return {
341
    connectionType: ConnectionType.AWS,
342
    sourceType,
343
    source: config.sourceInstanceId || config.sourceENIId,
344
    target: config.targetENIId,
345
    region: config.region
346
  };
347
}
348
```
349

350
#### GCP Integration
351

352
Google Cloud Platform-specific configuration for traffic collection.
353

354
```typescript { .api }
355
enum GCP_SOURCE_TYPE {
356
  INSTANCE,
357
  SUBNET,
358
  TAG
359
}
360

361
// GCP-specific trace collection
362
interface GCPTraceConfig {
363
  sourceType: GCP_SOURCE_TYPE;
364
  instanceName?: string;
365
  subnetName?: string;
366
  networkTag?: string;
367
  zone: string;
368
  project: string;
369
}
370
```
371

372
**Usage Examples:**
373

374
```typescript
375
import { GCP_SOURCE_TYPE, ConnectionType } from "@metlo/common";
376

377
// Configure GCP Packet Mirroring
378
function configureGCPMirroring(config: {
379
  sourceInstance?: string;
380
  sourceSubnet?: string;
381
  sourceTag?: string;
382
  zone: string;
383
  project: string;
384
}) {
385
  let sourceType: GCP_SOURCE_TYPE;
386
  let source: string;
387
  
388
  if (config.sourceInstance) {
389
    sourceType = GCP_SOURCE_TYPE.INSTANCE;
390
    source = config.sourceInstance;
391
  } else if (config.sourceSubnet) {
392
    sourceType = GCP_SOURCE_TYPE.SUBNET;
393
    source = config.sourceSubnet;
394
  } else if (config.sourceTag) {
395
    sourceType = GCP_SOURCE_TYPE.TAG;
396
    source = config.sourceTag;
397
  } else {
398
    throw new Error("Must specify source instance, subnet, or tag");
399
  }
400
  
401
  return {
402
    connectionType: ConnectionType.GCP,
403
    sourceType,
404
    source,
405
    zone: config.zone,
406
    project: config.project
407
  };
408
}
409
```
410

411
### Ingestor Implementation Examples
412

413
#### Node.js/Express Middleware
414

415
```typescript
416
import express from 'express';
417
import { TraceParams, RestMethod, AnalysisType } from '@metlo/common';
418

419
// Metlo middleware for Express.js
420
function metloMiddleware(options: { 
421
  backendUrl: string;
422
  apiKey: string;
423
  redactSensitive?: boolean;
424
}) {
425
  return (req: express.Request, res: express.Response, next: express.NextFunction) => {
426
    const originalSend = res.send;
427
    let responseBody = '';
428
    
429
    // Capture response body
430
    res.send = function(body: any) {
431
      responseBody = typeof body === 'string' ? body : JSON.stringify(body);
432
      return originalSend.call(this, body);
433
    };
434
    
435
    // Capture request completion
436
    res.on('finish', async () => {
437
      const traceParams: TraceParams = {
438
        request: {
439
          url: {
440
            host: req.get('host') || 'unknown',
441
            path: req.path,
442
            parameters: Object.entries(req.query).map(([name, value]) => ({
443
              name,
444
              value: String(value)
445
            }))
446
          },
447
          headers: Object.entries(req.headers).map(([name, value]) => ({
448
            name,
449
            value: Array.isArray(value) ? value.join(', ') : String(value)
450
          })),
451
          body: req.method !== 'GET' ? JSON.stringify(req.body) : '',
452
          method: req.method as RestMethod
453
        },
454
        response: {
455
          status: res.statusCode,
456
          headers: Object.entries(res.getHeaders()).map(([name, value]) => ({
457
            name,
458
            value: String(value)
459
          })),
460
          body: responseBody
461
        },
462
        meta: {
463
          incoming: true,
464
          source: req.ip || req.socket.remoteAddress || 'unknown',
465
          sourcePort: String(req.socket.remotePort || 0),
466
          destination: req.socket.localAddress || 'unknown',
467
          destinationPort: String(req.socket.localPort || 0),
468
          originalSource: req.headers['x-forwarded-for'] as string || undefined
469
        },
470
        analysisType: AnalysisType.FULL,
471
        redacted: options.redactSensitive
472
      };
473
      
474
      // Send to Metlo backend
475
      try {
476
        await sendTraceToMetlo(traceParams, options);
477
      } catch (error) {
478
        console.error('Failed to send trace to Metlo:', error);
479
      }
480
    });
481
    
482
    next();
483
  };
484
}
485

486
async function sendTraceToMetlo(trace: TraceParams, options: { backendUrl: string; apiKey: string }) {
487
  const response = await fetch(`${options.backendUrl}/api/v1/log-request`, {
488
    method: 'POST',
489
    headers: {
490
      'Content-Type': 'application/json',
491
      'Authorization': `Bearer ${options.apiKey}`
492
    },
493
    body: JSON.stringify(trace)
494
  });
495
  
496
  if (!response.ok) {
497
    throw new Error(`HTTP ${response.status}: ${response.statusText}`);
498
  }
499
}
500
```
501

502
#### Python Flask Integration
503

504
```python
505
from flask import Flask, request, g
506
import json
507
import time
508
import requests
509
from typing import Dict, Any, List
510

511
class MetloFlaskIntegration:
512
    def __init__(self, app: Flask, backend_url: str, api_key: str):
513
        self.app = app
514
        self.backend_url = backend_url
515
        self.api_key = api_key
516
        
517
        # Install hooks
518
        app.before_request(self.before_request)
519
        app.after_request(self.after_request)
520
    
521
    def before_request(self):
522
        g.metlo_start_time = time.time()
523
        g.metlo_request_data = {
524
            'url': {
525
                'host': request.host,
526
                'path': request.path,
527
                'parameters': [
528
                    {'name': k, 'value': v} 
529
                    for k, v in request.args.items()
530
                ]
531
            },
532
            'headers': [
533
                {'name': k, 'value': v}
534
                for k, v in request.headers.items()
535
            ],
536
            'body': request.get_data(as_text=True) if request.data else '',
537
            'method': request.method
538
        }
539
    
540
    def after_request(self, response):
541
        if hasattr(g, 'metlo_request_data'):
542
            trace_params = {
543
                'request': g.metlo_request_data,
544
                'response': {
545
                    'status': response.status_code,
546
                    'headers': [
547
                        {'name': k, 'value': v}
548
                        for k, v in response.headers.items()
549
                    ],
550
                    'body': response.get_data(as_text=True)
551
                },
552
                'meta': {
553
                    'incoming': True,
554
                    'source': request.remote_addr or 'unknown',
555
                    'sourcePort': str(request.environ.get('REMOTE_PORT', 0)),
556
                    'destination': request.environ.get('SERVER_NAME', 'unknown'),
557
                    'destinationPort': str(request.environ.get('SERVER_PORT', 0)),
558
                    'originalSource': request.headers.get('X-Forwarded-For')
559
                },
560
                'analysisType': 'full'
561
            }
562
            
563
            # Send asynchronously to avoid blocking response
564
            self._send_trace_async(trace_params)
565
        
566
        return response
567
    
568
    def _send_trace_async(self, trace_params: Dict[str, Any]):
569
        try:
570
            requests.post(
571
                f"{self.backend_url}/api/v1/log-request",
572
                json=trace_params,
573
                headers={
574
                    'Content-Type': 'application/json',
575
                    'Authorization': f'Bearer {self.api_key}'
576
                },
577
                timeout=5
578
            )
579
        except Exception as e:
580
            print(f"Failed to send trace to Metlo: {e}")
581

582
# Usage
583
app = Flask(__name__)
584
metlo = MetloFlaskIntegration(
585
    app, 
586
    backend_url="https://your-metlo-backend.com",
587
    api_key="your-api-key"
588
)
589
```
590

591
### Configuration Management
592

593
#### Webhook Configuration
594

595
Configure webhooks for alert notifications and integrations.
596

597
```typescript { .api }
598
interface WebhookResp {
599
  /** Webhook configuration UUID */
600
  uuid: string;
601
  /** When the webhook was created */
602
  createdAt: Date;
603
  /** Webhook endpoint URL */
604
  url: string;
605
  /** Maximum retry attempts for failed deliveries */
606
  maxRetries: number;
607
  /** Alert types that trigger this webhook */
608
  alertTypes: AlertType[];
609
  /** Hosts that trigger this webhook */
610
  hosts: string[];
611
  /** Recent webhook execution results */
612
  runs: WebhookRun[];
613
}
614

615
interface WebhookRun {
616
  /** Whether the webhook delivery succeeded */
617
  ok: boolean;
618
  /** Result message */
619
  msg: string;
620
  /** Alert payload that was sent */
621
  payload: Alert;
622
}
623
```
624

625
**Usage Examples:**
626

627
```typescript
628
import { WebhookResp, AlertType } from "@metlo/common";
629

630
// Configure webhook for critical security alerts
631
const securityWebhook: Partial<WebhookResp> = {
632
  url: "https://your-security-system.com/webhook/metlo",
633
  maxRetries: 3,
634
  alertTypes: [
635
    AlertType.UNAUTHENTICATED_ENDPOINT_SENSITIVE_DATA,
636
    AlertType.PII_DATA_DETECTED,
637
    AlertType.BASIC_AUTHENTICATION_DETECTED
638
  ],
639
  hosts: ["api.production.com", "secure.internal.com"]
640
};
641

642
// Configure Slack webhook for team notifications
643
const slackWebhook: Partial<WebhookResp> = {
644
  url: "https://hooks.slack.com/services/YOUR/SLACK/WEBHOOK",
645
  maxRetries: 2,
646
  alertTypes: [AlertType.NEW_ENDPOINT, AlertType.OPEN_API_SPEC_DIFF],
647
  hosts: [] // All hosts
648
};
649
```
650

651
#### Instance Settings
652

653
Global platform configuration and settings.
654

655
```typescript { .api }
656
interface InstanceSettings {
657
  /** Settings UUID */
658
  uuid: string;
659
  /** Email for update notifications */
660
  updateEmail: string;
661
  /** Whether update email notifications are skipped */
662
  skippedUpdateEmail: boolean;
663
}
664
```
665

666
### Performance Optimization
667

668
#### Test Generation Parameters
669

670
Configure test generation for API endpoints.
671

672
```typescript { .api }
673
interface GenerateTestParams {
674
  /** Type of test to generate */
675
  type: string;
676
  /** Target endpoint path */
677
  endpoint: string;
678
  /** Template version to use */
679
  version?: number;
680
  /** Host for the endpoint */
681
  host?: string;
682
}
683

684
interface GenerateTestRes {
685
  /** Whether test generation succeeded */
686
  success: boolean;
687
  /** Generated template name */
688
  templateName?: string;
689
  /** Template version used */
690
  templateVersion?: number;
691
  /** Error message if generation failed */
692
  msg?: string;
693
  /** Generated test configuration */
694
  test?: TestConfig;
695
}
696
```
697

698
**Usage Examples:**
699

700
```typescript
701
import { GenerateTestParams, GenerateTestRes } from "@metlo/common";
702

703
// Generate authentication tests for critical endpoints
704
async function generateEndpointTests(endpoints: string[]) {
705
  const testResults: GenerateTestRes[] = [];
706
  
707
  for (const endpoint of endpoints) {
708
    const params: GenerateTestParams = {
709
      type: "auth",
710
      endpoint,
711
      host: "api.production.com",
712
      version: 2
713
    };
714
    
715
    try {
716
      const result = await generateTest(params);
717
      testResults.push(result);
718
      
719
      if (result.success && result.test) {
720
        console.log(`Generated test: ${result.templateName} v${result.templateVersion}`);
721
      } else {
722
        console.error(`Failed to generate test for ${endpoint}: ${result.msg}`);
723
      }
724
    } catch (error) {
725
      console.error(`Error generating test for ${endpoint}:`, error);
726
    }
727
  }
728
  
729
  return testResults;
730
}
731

732
// Batch generate different test types
733
async function generateComprehensiveTests(endpoint: string, host: string) {
734
  const testTypes = ["auth", "sqli", "xss", "bola"];
735
  
736
  for (const type of testTypes) {
737
    await generateTest({
738
      type,
739
      endpoint,
740
      host,
741
      version: 1
742
    });
743
  }
744
}
745
```