0
# Usage Analytics
1
2
Monitor API usage, credits, tokens, and queue status for billing optimization and performance monitoring.
3
4
## Core Usage Methods
5
6
```typescript { .api }
7
/**
8
* Get current concurrency usage
9
* @returns Promise resolving to concurrency information
10
*/
11
getConcurrency(): Promise<ConcurrencyCheck>;
12
13
/**
14
* Get current credit usage and billing period
15
* @returns Promise resolving to credit usage information
16
*/
17
getCreditUsage(): Promise<CreditUsage>;
18
19
/**
20
* Get recent token usage and billing period
21
* @returns Promise resolving to token usage information
22
*/
23
getTokenUsage(): Promise<TokenUsage>;
24
25
/**
26
* Get historical credit usage by month
27
* @param byApiKey - Break down usage by API key
28
* @returns Promise resolving to historical credit data
29
*/
30
getCreditUsageHistorical(byApiKey?: boolean): Promise<CreditUsageHistoricalResponse>;
31
32
/**
33
* Get historical token usage by month
34
* @param byApiKey - Break down usage by API key
35
* @returns Promise resolving to historical token data
36
*/
37
getTokenUsageHistorical(byApiKey?: boolean): Promise<TokenUsageHistoricalResponse>;
38
39
/**
40
* Get metrics about team's scrape queue
41
* @returns Promise resolving to queue status information
42
*/
43
getQueueStatus(): Promise<QueueStatusResponse>;
44
```
45
46
## Usage Response Types
47
48
```typescript { .api }
49
// Current concurrency information
50
interface ConcurrencyCheck {
51
concurrency: number;
52
maxConcurrency: number;
53
}
54
55
// Current credit usage
56
interface CreditUsage {
57
remainingCredits: number;
58
planCredits?: number;
59
billingPeriodStart?: string | null;
60
billingPeriodEnd?: string | null;
61
}
62
63
// Current token usage
64
interface TokenUsage {
65
remainingTokens: number;
66
planTokens?: number;
67
billingPeriodStart?: string | null;
68
billingPeriodEnd?: string | null;
69
}
70
71
// Historical credit usage period
72
interface CreditUsageHistoricalPeriod {
73
startDate: string | null;
74
endDate: string | null;
75
apiKey?: string;
76
creditsUsed: number;
77
}
78
79
// Historical credit usage response
80
interface CreditUsageHistoricalResponse {
81
success: boolean;
82
periods: CreditUsageHistoricalPeriod[];
83
}
84
85
// Historical token usage period
86
interface TokenUsageHistoricalPeriod {
87
startDate: string | null;
88
endDate: string | null;
89
apiKey?: string;
90
tokensUsed: number;
91
}
92
93
// Historical token usage response
94
interface TokenUsageHistoricalResponse {
95
success: boolean;
96
periods: TokenUsageHistoricalPeriod[];
97
}
98
99
// Queue status information
100
interface QueueStatusResponse {
101
success: boolean;
102
jobsInQueue: number;
103
activeJobsInQueue: number;
104
waitingJobsInQueue: number;
105
maxConcurrency: number;
106
mostRecentSuccess: string | null;
107
}
108
```
109
110
## Usage Examples
111
112
### Current Usage Monitoring
113
114
```typescript
115
// Check current usage across all metrics
116
async function checkCurrentUsage() {
117
const [concurrency, credits, tokens, queue] = await Promise.all([
118
app.getConcurrency(),
119
app.getCreditUsage(),
120
app.getTokenUsage(),
121
app.getQueueStatus()
122
]);
123
124
console.log('=== Current Usage Status ===');
125
126
// Concurrency status
127
console.log(`Concurrency: ${concurrency.concurrency}/${concurrency.maxConcurrency}`);
128
const concurrencyUsed = (concurrency.concurrency / concurrency.maxConcurrency) * 100;
129
console.log(`Concurrency utilization: ${concurrencyUsed.toFixed(1)}%`);
130
131
// Credit status
132
console.log(`\nCredits remaining: ${credits.remainingCredits.toLocaleString()}`);
133
if (credits.planCredits) {
134
const creditsUsed = credits.planCredits - credits.remainingCredits;
135
const creditUsage = (creditsUsed / credits.planCredits) * 100;
136
console.log(`Credits used: ${creditsUsed.toLocaleString()}/${credits.planCredits.toLocaleString()} (${creditUsage.toFixed(1)}%)`);
137
}
138
139
// Token status
140
console.log(`\nTokens remaining: ${tokens.remainingTokens.toLocaleString()}`);
141
if (tokens.planTokens) {
142
const tokensUsed = tokens.planTokens - tokens.remainingTokens;
143
const tokenUsage = (tokensUsed / tokens.planTokens) * 100;
144
console.log(`Tokens used: ${tokensUsed.toLocaleString()}/${tokens.planTokens.toLocaleString()} (${tokenUsage.toFixed(1)}%)`);
145
}
146
147
// Queue status
148
console.log(`\nQueue status:`);
149
console.log(`- Total jobs in queue: ${queue.jobsInQueue}`);
150
console.log(`- Active jobs: ${queue.activeJobsInQueue}`);
151
console.log(`- Waiting jobs: ${queue.waitingJobsInQueue}`);
152
console.log(`- Max concurrency: ${queue.maxConcurrency}`);
153
154
if (queue.mostRecentSuccess) {
155
const lastSuccess = new Date(queue.mostRecentSuccess);
156
console.log(`- Last successful job: ${lastSuccess.toLocaleString()}`);
157
}
158
159
return {
160
concurrency,
161
credits,
162
tokens,
163
queue
164
};
165
}
166
167
// Usage
168
const usage = await checkCurrentUsage();
169
```
170
171
### Usage Alerts and Monitoring
172
173
```typescript
174
class UsageMonitor {
175
private warningThresholds = {
176
credits: 0.8, // 80%
177
tokens: 0.8, // 80%
178
concurrency: 0.9 // 90%
179
};
180
181
private criticalThresholds = {
182
credits: 0.95, // 95%
183
tokens: 0.95, // 95%
184
concurrency: 1.0 // 100%
185
};
186
187
async checkThresholds() {
188
const [concurrency, credits, tokens] = await Promise.all([
189
app.getConcurrency(),
190
app.getCreditUsage(),
191
app.getTokenUsage()
192
]);
193
194
const alerts = [];
195
196
// Check concurrency
197
const concurrencyRatio = concurrency.concurrency / concurrency.maxConcurrency;
198
if (concurrencyRatio >= this.criticalThresholds.concurrency) {
199
alerts.push({
200
type: 'critical',
201
metric: 'concurrency',
202
message: `Concurrency at maximum: ${concurrency.concurrency}/${concurrency.maxConcurrency}`,
203
value: concurrencyRatio
204
});
205
} else if (concurrencyRatio >= this.warningThresholds.concurrency) {
206
alerts.push({
207
type: 'warning',
208
metric: 'concurrency',
209
message: `High concurrency usage: ${(concurrencyRatio * 100).toFixed(1)}%`,
210
value: concurrencyRatio
211
});
212
}
213
214
// Check credits
215
if (credits.planCredits) {
216
const creditRatio = (credits.planCredits - credits.remainingCredits) / credits.planCredits;
217
if (creditRatio >= this.criticalThresholds.credits) {
218
alerts.push({
219
type: 'critical',
220
metric: 'credits',
221
message: `Credits nearly exhausted: ${credits.remainingCredits} remaining`,
222
value: creditRatio
223
});
224
} else if (creditRatio >= this.warningThresholds.credits) {
225
alerts.push({
226
type: 'warning',
227
metric: 'credits',
228
message: `Credit usage high: ${(creditRatio * 100).toFixed(1)}%`,
229
value: creditRatio
230
});
231
}
232
}
233
234
// Check tokens
235
if (tokens.planTokens) {
236
const tokenRatio = (tokens.planTokens - tokens.remainingTokens) / tokens.planTokens;
237
if (tokenRatio >= this.criticalThresholds.tokens) {
238
alerts.push({
239
type: 'critical',
240
metric: 'tokens',
241
message: `Tokens nearly exhausted: ${tokens.remainingTokens} remaining`,
242
value: tokenRatio
243
});
244
} else if (tokenRatio >= this.warningThresholds.tokens) {
245
alerts.push({
246
type: 'warning',
247
metric: 'tokens',
248
message: `Token usage high: ${(tokenRatio * 100).toFixed(1)}%`,
249
value: tokenRatio
250
});
251
}
252
}
253
254
return alerts;
255
}
256
257
async startMonitoring(intervalMinutes: number = 5) {
258
const checkInterval = setInterval(async () => {
259
try {
260
const alerts = await this.checkThresholds();
261
262
if (alerts.length > 0) {
263
console.log('\n🚨 Usage Alerts:');
264
alerts.forEach(alert => {
265
const emoji = alert.type === 'critical' ? '🔴' : '⚠️';
266
console.log(`${emoji} ${alert.message}`);
267
});
268
console.log('');
269
}
270
} catch (error) {
271
console.error('Error checking usage thresholds:', error);
272
}
273
}, intervalMinutes * 60 * 1000);
274
275
return () => clearInterval(checkInterval);
276
}
277
}
278
279
// Usage
280
const monitor = new UsageMonitor();
281
const stopMonitoring = await monitor.startMonitoring(5); // Check every 5 minutes
282
283
// Stop monitoring when done
284
// stopMonitoring();
285
```
286
287
### Historical Usage Analysis
288
289
```typescript
290
async function analyzeHistoricalUsage() {
291
const [creditHistory, tokenHistory] = await Promise.all([
292
app.getCreditUsageHistorical(),
293
app.getTokenUsageHistorical()
294
]);
295
296
console.log('=== Historical Usage Analysis ===');
297
298
// Credit usage trends
299
if (creditHistory.success && creditHistory.periods.length > 0) {
300
console.log('\nCredit Usage by Month:');
301
const sortedCredits = creditHistory.periods
302
.sort((a, b) => (a.startDate || '').localeCompare(b.startDate || ''));
303
304
sortedCredits.forEach(period => {
305
const startDate = period.startDate ? new Date(period.startDate).toLocaleDateString() : 'Unknown';
306
const endDate = period.endDate ? new Date(period.endDate).toLocaleDateString() : 'Unknown';
307
console.log(`${startDate} - ${endDate}: ${period.creditsUsed.toLocaleString()} credits`);
308
});
309
310
// Calculate trends
311
if (sortedCredits.length >= 2) {
312
const recent = sortedCredits[sortedCredits.length - 1];
313
const previous = sortedCredits[sortedCredits.length - 2];
314
const change = recent.creditsUsed - previous.creditsUsed;
315
const changePercent = (change / previous.creditsUsed) * 100;
316
317
console.log(`\nMonth-over-month change: ${change > 0 ? '+' : ''}${change.toLocaleString()} credits (${changePercent.toFixed(1)}%)`);
318
}
319
}
320
321
// Token usage trends
322
if (tokenHistory.success && tokenHistory.periods.length > 0) {
323
console.log('\nToken Usage by Month:');
324
const sortedTokens = tokenHistory.periods
325
.sort((a, b) => (a.startDate || '').localeCompare(b.startDate || ''));
326
327
sortedTokens.forEach(period => {
328
const startDate = period.startDate ? new Date(period.startDate).toLocaleDateString() : 'Unknown';
329
const endDate = period.endDate ? new Date(period.endDate).toLocaleDateString() : 'Unknown';
330
console.log(`${startDate} - ${endDate}: ${period.tokensUsed.toLocaleString()} tokens`);
331
});
332
333
// Calculate trends
334
if (sortedTokens.length >= 2) {
335
const recent = sortedTokens[sortedTokens.length - 1];
336
const previous = sortedTokens[sortedTokens.length - 2];
337
const change = recent.tokensUsed - previous.tokensUsed;
338
const changePercent = (change / previous.tokensUsed) * 100;
339
340
console.log(`\nMonth-over-month change: ${change > 0 ? '+' : ''}${change.toLocaleString()} tokens (${changePercent.toFixed(1)}%)`);
341
}
342
}
343
344
return { creditHistory, tokenHistory };
345
}
346
347
// Usage
348
const history = await analyzeHistoricalUsage();
349
```
350
351
### Multi-API Key Usage Tracking
352
353
```typescript
354
async function trackMultiKeyUsage() {
355
const [creditsByKey, tokensByKey] = await Promise.all([
356
app.getCreditUsageHistorical(true), // byApiKey = true
357
app.getTokenUsageHistorical(true) // byApiKey = true
358
]);
359
360
console.log('=== Usage by API Key ===');
361
362
// Group by API key
363
const keyUsage = new Map<string, {
364
credits: number;
365
tokens: number;
366
periods: number;
367
}>();
368
369
// Process credit data
370
if (creditsByKey.success) {
371
creditsByKey.periods.forEach(period => {
372
const key = period.apiKey || 'unknown';
373
if (!keyUsage.has(key)) {
374
keyUsage.set(key, { credits: 0, tokens: 0, periods: 0 });
375
}
376
const usage = keyUsage.get(key)!;
377
usage.credits += period.creditsUsed;
378
usage.periods++;
379
});
380
}
381
382
// Process token data
383
if (tokensByKey.success) {
384
tokensByKey.periods.forEach(period => {
385
const key = period.apiKey || 'unknown';
386
if (!keyUsage.has(key)) {
387
keyUsage.set(key, { credits: 0, tokens: 0, periods: 0 });
388
}
389
const usage = keyUsage.get(key)!;
390
usage.tokens += period.tokensUsed;
391
});
392
}
393
394
// Display results
395
console.log('\nUsage Summary by API Key:');
396
const sortedKeys = Array.from(keyUsage.entries())
397
.sort(([,a], [,b]) => (b.credits + b.tokens) - (a.credits + a.tokens));
398
399
sortedKeys.forEach(([key, usage]) => {
400
const keyDisplay = key.length > 10 ? `${key.substring(0, 10)}...` : key;
401
console.log(`${keyDisplay}:`);
402
console.log(` Credits: ${usage.credits.toLocaleString()}`);
403
console.log(` Tokens: ${usage.tokens.toLocaleString()}`);
404
console.log(` Periods: ${usage.periods}`);
405
});
406
407
return keyUsage;
408
}
409
410
// Usage
411
const keyUsage = await trackMultiKeyUsage();
412
```
413
414
### Cost Estimation and Budgeting
415
416
```typescript
417
class CostEstimator {
418
private creditCost = 0.001; // $0.001 per credit (example)
419
private tokenCost = 0.0001; // $0.0001 per token (example)
420
421
async estimateCurrentCosts() {
422
const [credits, tokens] = await Promise.all([
423
app.getCreditUsage(),
424
app.getTokenUsage()
425
]);
426
427
const creditCost = credits.planCredits
428
? (credits.planCredits - credits.remainingCredits) * this.creditCost
429
: 0;
430
431
const tokenCost = tokens.planTokens
432
? (tokens.planTokens - tokens.remainingTokens) * this.tokenCost
433
: 0;
434
435
return {
436
creditCost,
437
tokenCost,
438
totalCost: creditCost + tokenCost,
439
billing: {
440
periodStart: credits.billingPeriodStart,
441
periodEnd: credits.billingPeriodEnd
442
}
443
};
444
}
445
446
async estimateJobCost(jobType: 'scrape' | 'crawl' | 'batch', params: any) {
447
// Rough cost estimation based on job parameters
448
let estimatedCredits = 0;
449
let estimatedTokens = 0;
450
451
switch (jobType) {
452
case 'scrape':
453
estimatedCredits = 1; // 1 credit per URL
454
if (params.formats?.includes('json')) {
455
estimatedTokens = 100; // Estimated tokens for JSON processing
456
}
457
break;
458
459
case 'crawl':
460
estimatedCredits = params.limit || 100; // 1 credit per page
461
if (params.scrapeOptions?.formats?.includes('json')) {
462
estimatedTokens = (params.limit || 100) * 50; // Estimated tokens per page
463
}
464
break;
465
466
case 'batch':
467
estimatedCredits = params.urls?.length || 0;
468
if (params.options?.formats?.includes('json')) {
469
estimatedTokens = (params.urls?.length || 0) * 100;
470
}
471
break;
472
}
473
474
const estimatedCost = (estimatedCredits * this.creditCost) + (estimatedTokens * this.tokenCost);
475
476
return {
477
estimatedCredits,
478
estimatedTokens,
479
estimatedCost,
480
breakdown: {
481
creditCost: estimatedCredits * this.creditCost,
482
tokenCost: estimatedTokens * this.tokenCost
483
}
484
};
485
}
486
487
async checkBudgetStatus(monthlyBudget: number) {
488
const currentCosts = await this.estimateCurrentCosts();
489
const budgetUsed = (currentCosts.totalCost / monthlyBudget) * 100;
490
491
console.log('=== Budget Status ===');
492
console.log(`Monthly Budget: $${monthlyBudget.toFixed(2)}`);
493
console.log(`Current Spend: $${currentCosts.totalCost.toFixed(2)}`);
494
console.log(`Budget Used: ${budgetUsed.toFixed(1)}%`);
495
console.log(`Remaining: $${(monthlyBudget - currentCosts.totalCost).toFixed(2)}`);
496
497
if (budgetUsed > 90) {
498
console.log('🔴 Budget nearly exhausted!');
499
} else if (budgetUsed > 75) {
500
console.log('⚠️ High budget usage');
501
} else {
502
console.log('✅ Budget usage normal');
503
}
504
505
return {
506
budget: monthlyBudget,
507
spent: currentCosts.totalCost,
508
remaining: monthlyBudget - currentCosts.totalCost,
509
percentUsed: budgetUsed
510
};
511
}
512
}
513
514
// Usage
515
const estimator = new CostEstimator();
516
517
// Check current costs
518
const costs = await estimator.estimateCurrentCosts();
519
console.log('Current costs:', costs);
520
521
// Estimate job cost before running
522
const crawlEstimate = await estimator.estimateJobCost('crawl', {
523
limit: 500,
524
scrapeOptions: { formats: ['markdown', 'json'] }
525
});
526
console.log('Estimated crawl cost:', crawlEstimate);
527
528
// Check budget status
529
const budgetStatus = await estimator.checkBudgetStatus(100); // $100 monthly budget
530
```
531
532
### Queue Management and Optimization
533
534
```typescript
535
async function manageQueue() {
536
const queueStatus = await app.getQueueStatus();
537
538
console.log('=== Queue Management ===');
539
console.log(`Jobs in queue: ${queueStatus.jobsInQueue}`);
540
console.log(`Active jobs: ${queueStatus.activeJobsInQueue}`);
541
console.log(`Waiting jobs: ${queueStatus.waitingJobsInQueue}`);
542
console.log(`Max concurrency: ${queueStatus.maxConcurrency}`);
543
544
// Calculate queue health metrics
545
const queueEfficiency = queueStatus.maxConcurrency > 0
546
? (queueStatus.activeJobsInQueue / queueStatus.maxConcurrency) * 100
547
: 0;
548
549
console.log(`Queue efficiency: ${queueEfficiency.toFixed(1)}%`);
550
551
// Recommendations based on queue status
552
if (queueStatus.waitingJobsInQueue > queueStatus.maxConcurrency * 2) {
553
console.log('⚠️ High queue backlog - consider reducing job submission rate');
554
}
555
556
if (queueStatus.activeJobsInQueue < queueStatus.maxConcurrency * 0.5) {
557
console.log('💡 Low queue utilization - can increase job submission rate');
558
}
559
560
if (queueStatus.mostRecentSuccess) {
561
const lastSuccess = new Date(queueStatus.mostRecentSuccess);
562
const timeSinceSuccess = Date.now() - lastSuccess.getTime();
563
const hoursSinceSuccess = timeSinceSuccess / (1000 * 60 * 60);
564
565
if (hoursSinceSuccess > 1) {
566
console.log(`⚠️ No successful jobs in ${hoursSinceSuccess.toFixed(1)} hours`);
567
}
568
}
569
570
return {
571
queueStatus,
572
efficiency: queueEfficiency,
573
recommendations: generateQueueRecommendations(queueStatus)
574
};
575
}
576
577
function generateQueueRecommendations(status: QueueStatusResponse) {
578
const recommendations = [];
579
580
if (status.waitingJobsInQueue > status.maxConcurrency) {
581
recommendations.push('Reduce job submission rate to prevent queue buildup');
582
}
583
584
if (status.activeJobsInQueue === 0 && status.waitingJobsInQueue > 0) {
585
recommendations.push('Check for system issues - jobs not starting');
586
}
587
588
const utilization = status.activeJobsInQueue / status.maxConcurrency;
589
if (utilization < 0.3) {
590
recommendations.push('Low utilization - consider batch processing for efficiency');
591
}
592
593
return recommendations;
594
}
595
596
// Usage
597
const queueAnalysis = await manageQueue();
598
```