tessl install github:jeremylongshore/claude-code-plugins-plus-skills --skill firecrawl-reliability-patternsImplement FireCrawl reliability patterns including circuit breakers, idempotency, and graceful degradation. Use when building fault-tolerant FireCrawl integrations, implementing retry strategies, or adding resilience to production FireCrawl services. Trigger with phrases like "firecrawl reliability", "firecrawl circuit breaker", "firecrawl idempotent", "firecrawl resilience", "firecrawl fallback", "firecrawl bulkhead".
Review Score
84%
Validation Score
12/16
Implementation Score
73%
Activation Score
100%
Production-grade reliability patterns for FireCrawl integrations.
import CircuitBreaker from 'opossum';
const firecrawlBreaker = new CircuitBreaker(
async (operation: () => Promise<any>) => operation(),
{
timeout: 30000,
errorThresholdPercentage: 50,
resetTimeout: 30000,
volumeThreshold: 10,
}
);
// Events
firecrawlBreaker.on('open', () => {
console.warn('FireCrawl circuit OPEN - requests failing fast');
alertOps('FireCrawl circuit breaker opened');
});
firecrawlBreaker.on('halfOpen', () => {
console.info('FireCrawl circuit HALF-OPEN - testing recovery');
});
firecrawlBreaker.on('close', () => {
console.info('FireCrawl circuit CLOSED - normal operation');
});
// Usage
async function safeFireCrawlCall<T>(fn: () => Promise<T>): Promise<T> {
return firecrawlBreaker.fire(fn);
}import { v4 as uuidv4 } from 'uuid';
import crypto from 'crypto';
// Generate deterministic idempotency key from input
function generateIdempotencyKey(
operation: string,
params: Record<string, any>
): string {
const data = JSON.stringify({ operation, params });
return crypto.createHash('sha256').update(data).digest('hex');
}
// Or use random key with storage
class IdempotencyManager {
private store: Map<string, { key: string; expiresAt: Date }> = new Map();
getOrCreate(operationId: string): string {
const existing = this.store.get(operationId);
if (existing && existing.expiresAt > new Date()) {
return existing.key;
}
const key = uuidv4();
this.store.set(operationId, {
key,
expiresAt: new Date(Date.now() + 24 * 60 * 60 * 1000),
});
return key;
}
}import PQueue from 'p-queue';
// Separate queues for different operations
const firecrawlQueues = {
critical: new PQueue({ concurrency: 10 }),
normal: new PQueue({ concurrency: 5 }),
bulk: new PQueue({ concurrency: 2 }),
};
async function prioritizedFireCrawlCall<T>(
priority: 'critical' | 'normal' | 'bulk',
fn: () => Promise<T>
): Promise<T> {
return firecrawlQueues[priority].add(fn);
}
// Usage
await prioritizedFireCrawlCall('critical', () =>
firecrawlClient.processPayment(order)
);
await prioritizedFireCrawlCall('bulk', () =>
firecrawlClient.syncCatalog(products)
);const TIMEOUT_CONFIG = {
connect: 5000, // Initial connection
request: 30000, // Standard requests
upload: 120000, // File uploads
longPoll: 300000, // Webhook long-polling
};
async function timedoutFireCrawlCall<T>(
operation: 'connect' | 'request' | 'upload' | 'longPoll',
fn: () => Promise<T>
): Promise<T> {
const timeout = TIMEOUT_CONFIG[operation];
return Promise.race([
fn(),
new Promise<never>((_, reject) =>
setTimeout(() => reject(new Error(`FireCrawl ${operation} timeout`)), timeout)
),
]);
}interface FireCrawlFallback {
enabled: boolean;
data: any;
staleness: 'fresh' | 'stale' | 'very_stale';
}
async function withFireCrawlFallback<T>(
fn: () => Promise<T>,
fallbackFn: () => Promise<T>
): Promise<{ data: T; fallback: boolean }> {
try {
const data = await fn();
// Update cache for future fallback
await updateFallbackCache(data);
return { data, fallback: false };
} catch (error) {
console.warn('FireCrawl failed, using fallback:', error.message);
const data = await fallbackFn();
return { data, fallback: true };
}
}interface DeadLetterEntry {
id: string;
operation: string;
payload: any;
error: string;
attempts: number;
lastAttempt: Date;
}
class FireCrawlDeadLetterQueue {
private queue: DeadLetterEntry[] = [];
add(entry: Omit<DeadLetterEntry, 'id' | 'lastAttempt'>): void {
this.queue.push({
...entry,
id: uuidv4(),
lastAttempt: new Date(),
});
}
async processOne(): Promise<boolean> {
const entry = this.queue.shift();
if (!entry) return false;
try {
await firecrawlClient[entry.operation](entry.payload);
console.log(`DLQ: Successfully reprocessed ${entry.id}`);
return true;
} catch (error) {
entry.attempts++;
entry.lastAttempt = new Date();
if (entry.attempts < 5) {
this.queue.push(entry);
} else {
console.error(`DLQ: Giving up on ${entry.id} after 5 attempts`);
await alertOnPermanentFailure(entry);
}
return false;
}
}
}type HealthStatus = 'healthy' | 'degraded' | 'unhealthy';
async function firecrawlHealthCheck(): Promise<{
status: HealthStatus;
details: Record<string, any>;
}> {
const checks = {
api: await checkApiConnectivity(),
circuitBreaker: firecrawlBreaker.stats(),
dlqSize: deadLetterQueue.size(),
};
const status: HealthStatus =
!checks.api.connected ? 'unhealthy' :
checks.circuitBreaker.state === 'open' ? 'degraded' :
checks.dlqSize > 100 ? 'degraded' :
'healthy';
return { status, details: checks };
}Wrap FireCrawl calls with circuit breaker.
Generate deterministic keys for operations.
Separate queues for different priorities.
Handle permanent failures gracefully.
| Issue | Cause | Solution |
|---|---|---|
| Circuit stays open | Threshold too low | Adjust error percentage |
| Duplicate operations | Missing idempotency | Add idempotency key |
| Queue full | Rate too high | Increase concurrency |
| DLQ growing | Persistent failures | Investigate root cause |
const state = firecrawlBreaker.stats().state;
console.log('FireCrawl circuit:', state);For policy enforcement, see firecrawl-policy-guardrails.