Usage Analytics

/**
 * Get current concurrency usage
 * @returns Promise resolving to concurrency information
 */
getConcurrency(): Promise<ConcurrencyCheck>;

/**
 * Get current credit usage and billing period
 * @returns Promise resolving to credit usage information
 */
getCreditUsage(): Promise<CreditUsage>;

/**
 * Get recent token usage and billing period
 * @returns Promise resolving to token usage information
 */
getTokenUsage(): Promise<TokenUsage>;

/**
 * Get historical credit usage by month
 * @param byApiKey - Break down usage by API key
 * @returns Promise resolving to historical credit data
 */
getCreditUsageHistorical(byApiKey?: boolean): Promise<CreditUsageHistoricalResponse>;

/**
 * Get historical token usage by month
 * @param byApiKey - Break down usage by API key
 * @returns Promise resolving to historical token data
 */
getTokenUsageHistorical(byApiKey?: boolean): Promise<TokenUsageHistoricalResponse>;

/**
 * Get metrics about team's scrape queue
 * @returns Promise resolving to queue status information
 */
getQueueStatus(): Promise<QueueStatusResponse>;

// Current concurrency information
interface ConcurrencyCheck {
  concurrency: number;
  maxConcurrency: number;
}

// Current credit usage
interface CreditUsage {
  remainingCredits: number;
  planCredits?: number;
  billingPeriodStart?: string | null;
  billingPeriodEnd?: string | null;
}

// Current token usage
interface TokenUsage {
  remainingTokens: number;
  planTokens?: number;
  billingPeriodStart?: string | null;
  billingPeriodEnd?: string | null;
}

// Historical credit usage period
interface CreditUsageHistoricalPeriod {
  startDate: string | null;
  endDate: string | null;
  apiKey?: string;
  creditsUsed: number;
}

// Historical credit usage response
interface CreditUsageHistoricalResponse {
  success: boolean;
  periods: CreditUsageHistoricalPeriod[];
}

// Historical token usage period
interface TokenUsageHistoricalPeriod {
  startDate: string | null;
  endDate: string | null;
  apiKey?: string;
  tokensUsed: number;
}

// Historical token usage response
interface TokenUsageHistoricalResponse {
  success: boolean;
  periods: TokenUsageHistoricalPeriod[];
}

// Queue status information
interface QueueStatusResponse {
  success: boolean;
  jobsInQueue: number;
  activeJobsInQueue: number;
  waitingJobsInQueue: number;
  maxConcurrency: number;
  mostRecentSuccess: string | null;
}

// Check current usage across all metrics
async function checkCurrentUsage() {
  const [concurrency, credits, tokens, queue] = await Promise.all([
    app.getConcurrency(),
    app.getCreditUsage(),
    app.getTokenUsage(),
    app.getQueueStatus()
  ]);
  
  console.log('=== Current Usage Status ===');
  
  // Concurrency status
  console.log(`Concurrency: ${concurrency.concurrency}/${concurrency.maxConcurrency}`);
  const concurrencyUsed = (concurrency.concurrency / concurrency.maxConcurrency) * 100;
  console.log(`Concurrency utilization: ${concurrencyUsed.toFixed(1)}%`);
  
  // Credit status
  console.log(`\nCredits remaining: ${credits.remainingCredits.toLocaleString()}`);
  if (credits.planCredits) {
    const creditsUsed = credits.planCredits - credits.remainingCredits;
    const creditUsage = (creditsUsed / credits.planCredits) * 100;
    console.log(`Credits used: ${creditsUsed.toLocaleString()}/${credits.planCredits.toLocaleString()} (${creditUsage.toFixed(1)}%)`);
  }
  
  // Token status
  console.log(`\nTokens remaining: ${tokens.remainingTokens.toLocaleString()}`);
  if (tokens.planTokens) {
    const tokensUsed = tokens.planTokens - tokens.remainingTokens;
    const tokenUsage = (tokensUsed / tokens.planTokens) * 100;
    console.log(`Tokens used: ${tokensUsed.toLocaleString()}/${tokens.planTokens.toLocaleString()} (${tokenUsage.toFixed(1)}%)`);
  }
  
  // Queue status
  console.log(`\nQueue status:`);
  console.log(`- Total jobs in queue: ${queue.jobsInQueue}`);
  console.log(`- Active jobs: ${queue.activeJobsInQueue}`);
  console.log(`- Waiting jobs: ${queue.waitingJobsInQueue}`);
  console.log(`- Max concurrency: ${queue.maxConcurrency}`);
  
  if (queue.mostRecentSuccess) {
    const lastSuccess = new Date(queue.mostRecentSuccess);
    console.log(`- Last successful job: ${lastSuccess.toLocaleString()}`);
  }
  
  return {
    concurrency,
    credits,
    tokens,
    queue
  };
}

// Usage
const usage = await checkCurrentUsage();

class UsageMonitor {
  private warningThresholds = {
    credits: 0.8, // 80%
    tokens: 0.8,  // 80%
    concurrency: 0.9 // 90%
  };
  
  private criticalThresholds = {
    credits: 0.95, // 95%
    tokens: 0.95,  // 95%
    concurrency: 1.0 // 100%
  };
  
  async checkThresholds() {
    const [concurrency, credits, tokens] = await Promise.all([
      app.getConcurrency(),
      app.getCreditUsage(),
      app.getTokenUsage()
    ]);
    
    const alerts = [];
    
    // Check concurrency
    const concurrencyRatio = concurrency.concurrency / concurrency.maxConcurrency;
    if (concurrencyRatio >= this.criticalThresholds.concurrency) {
      alerts.push({
        type: 'critical',
        metric: 'concurrency',
        message: `Concurrency at maximum: ${concurrency.concurrency}/${concurrency.maxConcurrency}`,
        value: concurrencyRatio
      });
    } else if (concurrencyRatio >= this.warningThresholds.concurrency) {
      alerts.push({
        type: 'warning',
        metric: 'concurrency',
        message: `High concurrency usage: ${(concurrencyRatio * 100).toFixed(1)}%`,
        value: concurrencyRatio
      });
    }
    
    // Check credits
    if (credits.planCredits) {
      const creditRatio = (credits.planCredits - credits.remainingCredits) / credits.planCredits;
      if (creditRatio >= this.criticalThresholds.credits) {
        alerts.push({
          type: 'critical',
          metric: 'credits',
          message: `Credits nearly exhausted: ${credits.remainingCredits} remaining`,
          value: creditRatio
        });
      } else if (creditRatio >= this.warningThresholds.credits) {
        alerts.push({
          type: 'warning',
          metric: 'credits',
          message: `Credit usage high: ${(creditRatio * 100).toFixed(1)}%`,
          value: creditRatio
        });
      }
    }
    
    // Check tokens
    if (tokens.planTokens) {
      const tokenRatio = (tokens.planTokens - tokens.remainingTokens) / tokens.planTokens;
      if (tokenRatio >= this.criticalThresholds.tokens) {
        alerts.push({
          type: 'critical',
          metric: 'tokens',
          message: `Tokens nearly exhausted: ${tokens.remainingTokens} remaining`,
          value: tokenRatio
        });
      } else if (tokenRatio >= this.warningThresholds.tokens) {
        alerts.push({
          type: 'warning',
          metric: 'tokens',
          message: `Token usage high: ${(tokenRatio * 100).toFixed(1)}%`,
          value: tokenRatio
        });
      }
    }
    
    return alerts;
  }
  
  async startMonitoring(intervalMinutes: number = 5) {
    const checkInterval = setInterval(async () => {
      try {
        const alerts = await this.checkThresholds();
        
        if (alerts.length > 0) {
          console.log('\n🚨 Usage Alerts:');
          alerts.forEach(alert => {
            const emoji = alert.type === 'critical' ? '🔴' : '⚠️';
            console.log(`${emoji} ${alert.message}`);
          });
          console.log('');
        }
      } catch (error) {
        console.error('Error checking usage thresholds:', error);
      }
    }, intervalMinutes * 60 * 1000);
    
    return () => clearInterval(checkInterval);
  }
}

// Usage
const monitor = new UsageMonitor();
const stopMonitoring = await monitor.startMonitoring(5); // Check every 5 minutes

// Stop monitoring when done
// stopMonitoring();

async function analyzeHistoricalUsage() {
  const [creditHistory, tokenHistory] = await Promise.all([
    app.getCreditUsageHistorical(),
    app.getTokenUsageHistorical()
  ]);
  
  console.log('=== Historical Usage Analysis ===');
  
  // Credit usage trends
  if (creditHistory.success && creditHistory.periods.length > 0) {
    console.log('\nCredit Usage by Month:');
    const sortedCredits = creditHistory.periods
      .sort((a, b) => (a.startDate || '').localeCompare(b.startDate || ''));
    
    sortedCredits.forEach(period => {
      const startDate = period.startDate ? new Date(period.startDate).toLocaleDateString() : 'Unknown';
      const endDate = period.endDate ? new Date(period.endDate).toLocaleDateString() : 'Unknown';
      console.log(`${startDate} - ${endDate}: ${period.creditsUsed.toLocaleString()} credits`);
    });
    
    // Calculate trends
    if (sortedCredits.length >= 2) {
      const recent = sortedCredits[sortedCredits.length - 1];
      const previous = sortedCredits[sortedCredits.length - 2];
      const change = recent.creditsUsed - previous.creditsUsed;
      const changePercent = (change / previous.creditsUsed) * 100;
      
      console.log(`\nMonth-over-month change: ${change > 0 ? '+' : ''}${change.toLocaleString()} credits (${changePercent.toFixed(1)}%)`);
    }
  }
  
  // Token usage trends
  if (tokenHistory.success && tokenHistory.periods.length > 0) {
    console.log('\nToken Usage by Month:');
    const sortedTokens = tokenHistory.periods
      .sort((a, b) => (a.startDate || '').localeCompare(b.startDate || ''));
    
    sortedTokens.forEach(period => {
      const startDate = period.startDate ? new Date(period.startDate).toLocaleDateString() : 'Unknown';
      const endDate = period.endDate ? new Date(period.endDate).toLocaleDateString() : 'Unknown';
      console.log(`${startDate} - ${endDate}: ${period.tokensUsed.toLocaleString()} tokens`);
    });
    
    // Calculate trends
    if (sortedTokens.length >= 2) {
      const recent = sortedTokens[sortedTokens.length - 1];
      const previous = sortedTokens[sortedTokens.length - 2];
      const change = recent.tokensUsed - previous.tokensUsed;
      const changePercent = (change / previous.tokensUsed) * 100;
      
      console.log(`\nMonth-over-month change: ${change > 0 ? '+' : ''}${change.toLocaleString()} tokens (${changePercent.toFixed(1)}%)`);
    }
  }
  
  return { creditHistory, tokenHistory };
}

// Usage
const history = await analyzeHistoricalUsage();

async function trackMultiKeyUsage() {
  const [creditsByKey, tokensByKey] = await Promise.all([
    app.getCreditUsageHistorical(true), // byApiKey = true
    app.getTokenUsageHistorical(true)   // byApiKey = true
  ]);
  
  console.log('=== Usage by API Key ===');
  
  // Group by API key
  const keyUsage = new Map<string, {
    credits: number;
    tokens: number;
    periods: number;
  }>();
  
  // Process credit data
  if (creditsByKey.success) {
    creditsByKey.periods.forEach(period => {
      const key = period.apiKey || 'unknown';
      if (!keyUsage.has(key)) {
        keyUsage.set(key, { credits: 0, tokens: 0, periods: 0 });
      }
      const usage = keyUsage.get(key)!;
      usage.credits += period.creditsUsed;
      usage.periods++;
    });
  }
  
  // Process token data
  if (tokensByKey.success) {
    tokensByKey.periods.forEach(period => {
      const key = period.apiKey || 'unknown';
      if (!keyUsage.has(key)) {
        keyUsage.set(key, { credits: 0, tokens: 0, periods: 0 });
      }
      const usage = keyUsage.get(key)!;
      usage.tokens += period.tokensUsed;
    });
  }
  
  // Display results
  console.log('\nUsage Summary by API Key:');
  const sortedKeys = Array.from(keyUsage.entries())
    .sort(([,a], [,b]) => (b.credits + b.tokens) - (a.credits + a.tokens));
  
  sortedKeys.forEach(([key, usage]) => {
    const keyDisplay = key.length > 10 ? `${key.substring(0, 10)}...` : key;
    console.log(`${keyDisplay}:`);
    console.log(`  Credits: ${usage.credits.toLocaleString()}`);
    console.log(`  Tokens: ${usage.tokens.toLocaleString()}`);
    console.log(`  Periods: ${usage.periods}`);
  });
  
  return keyUsage;
}

// Usage
const keyUsage = await trackMultiKeyUsage();

class CostEstimator {
  private creditCost = 0.001; // $0.001 per credit (example)
  private tokenCost = 0.0001; // $0.0001 per token (example)
  
  async estimateCurrentCosts() {
    const [credits, tokens] = await Promise.all([
      app.getCreditUsage(),
      app.getTokenUsage()
    ]);
    
    const creditCost = credits.planCredits 
      ? (credits.planCredits - credits.remainingCredits) * this.creditCost
      : 0;
    
    const tokenCost = tokens.planTokens
      ? (tokens.planTokens - tokens.remainingTokens) * this.tokenCost
      : 0;
    
    return {
      creditCost,
      tokenCost,
      totalCost: creditCost + tokenCost,
      billing: {
        periodStart: credits.billingPeriodStart,
        periodEnd: credits.billingPeriodEnd
      }
    };
  }
  
  async estimateJobCost(jobType: 'scrape' | 'crawl' | 'batch', params: any) {
    // Rough cost estimation based on job parameters
    let estimatedCredits = 0;
    let estimatedTokens = 0;
    
    switch (jobType) {
      case 'scrape':
        estimatedCredits = 1; // 1 credit per URL
        if (params.formats?.includes('json')) {
          estimatedTokens = 100; // Estimated tokens for JSON processing
        }
        break;
        
      case 'crawl':
        estimatedCredits = params.limit || 100; // 1 credit per page
        if (params.scrapeOptions?.formats?.includes('json')) {
          estimatedTokens = (params.limit || 100) * 50; // Estimated tokens per page
        }
        break;
        
      case 'batch':
        estimatedCredits = params.urls?.length || 0;
        if (params.options?.formats?.includes('json')) {
          estimatedTokens = (params.urls?.length || 0) * 100;
        }
        break;
    }
    
    const estimatedCost = (estimatedCredits * this.creditCost) + (estimatedTokens * this.tokenCost);
    
    return {
      estimatedCredits,
      estimatedTokens,
      estimatedCost,
      breakdown: {
        creditCost: estimatedCredits * this.creditCost,
        tokenCost: estimatedTokens * this.tokenCost
      }
    };
  }
  
  async checkBudgetStatus(monthlyBudget: number) {
    const currentCosts = await this.estimateCurrentCosts();
    const budgetUsed = (currentCosts.totalCost / monthlyBudget) * 100;
    
    console.log('=== Budget Status ===');
    console.log(`Monthly Budget: $${monthlyBudget.toFixed(2)}`);
    console.log(`Current Spend: $${currentCosts.totalCost.toFixed(2)}`);
    console.log(`Budget Used: ${budgetUsed.toFixed(1)}%`);
    console.log(`Remaining: $${(monthlyBudget - currentCosts.totalCost).toFixed(2)}`);
    
    if (budgetUsed > 90) {
      console.log('🔴 Budget nearly exhausted!');
    } else if (budgetUsed > 75) {
      console.log('⚠️ High budget usage');
    } else {
      console.log('✅ Budget usage normal');
    }
    
    return {
      budget: monthlyBudget,
      spent: currentCosts.totalCost,
      remaining: monthlyBudget - currentCosts.totalCost,
      percentUsed: budgetUsed
    };
  }
}

// Usage
const estimator = new CostEstimator();

// Check current costs
const costs = await estimator.estimateCurrentCosts();
console.log('Current costs:', costs);

// Estimate job cost before running
const crawlEstimate = await estimator.estimateJobCost('crawl', {
  limit: 500,
  scrapeOptions: { formats: ['markdown', 'json'] }
});
console.log('Estimated crawl cost:', crawlEstimate);

// Check budget status
const budgetStatus = await estimator.checkBudgetStatus(100); // $100 monthly budget

async function manageQueue() {
  const queueStatus = await app.getQueueStatus();
  
  console.log('=== Queue Management ===');
  console.log(`Jobs in queue: ${queueStatus.jobsInQueue}`);
  console.log(`Active jobs: ${queueStatus.activeJobsInQueue}`);
  console.log(`Waiting jobs: ${queueStatus.waitingJobsInQueue}`);
  console.log(`Max concurrency: ${queueStatus.maxConcurrency}`);
  
  // Calculate queue health metrics
  const queueEfficiency = queueStatus.maxConcurrency > 0 
    ? (queueStatus.activeJobsInQueue / queueStatus.maxConcurrency) * 100
    : 0;
  
  console.log(`Queue efficiency: ${queueEfficiency.toFixed(1)}%`);
  
  // Recommendations based on queue status
  if (queueStatus.waitingJobsInQueue > queueStatus.maxConcurrency * 2) {
    console.log('⚠️ High queue backlog - consider reducing job submission rate');
  }
  
  if (queueStatus.activeJobsInQueue < queueStatus.maxConcurrency * 0.5) {
    console.log('💡 Low queue utilization - can increase job submission rate');
  }
  
  if (queueStatus.mostRecentSuccess) {
    const lastSuccess = new Date(queueStatus.mostRecentSuccess);
    const timeSinceSuccess = Date.now() - lastSuccess.getTime();
    const hoursSinceSuccess = timeSinceSuccess / (1000 * 60 * 60);
    
    if (hoursSinceSuccess > 1) {
      console.log(`⚠️ No successful jobs in ${hoursSinceSuccess.toFixed(1)} hours`);
    }
  }
  
  return {
    queueStatus,
    efficiency: queueEfficiency,
    recommendations: generateQueueRecommendations(queueStatus)
  };
}

function generateQueueRecommendations(status: QueueStatusResponse) {
  const recommendations = [];
  
  if (status.waitingJobsInQueue > status.maxConcurrency) {
    recommendations.push('Reduce job submission rate to prevent queue buildup');
  }
  
  if (status.activeJobsInQueue === 0 && status.waitingJobsInQueue > 0) {
    recommendations.push('Check for system issues - jobs not starting');
  }
  
  const utilization = status.activeJobsInQueue / status.maxConcurrency;
  if (utilization < 0.3) {
    recommendations.push('Low utilization - consider batch processing for efficiency');
  }
  
  return recommendations;
}

// Usage
const queueAnalysis = await manageQueue();