Internal multi-priority queue system for managing request execution order with configurable priority levels and efficient task scheduling. These queue classes are not directly exported from the main crawler module.
Priority-based queue implementation for managing tasks with different priority levels.
/**
* Multi-priority queue for task scheduling
* @template T - Type of items stored in the queue
*/
class multiPriorityQueue<T> {
constructor(priorities: number);
/** Get current queue size */
size(): number;
/** Add item to queue with priority */
enqueue(value: T, priority: number): void;
/** Remove and return highest priority item */
dequeue(): T;
}Creates a new multi-priority queue with specified number of priority levels.
/**
* Creates a multi-priority queue
* @param priorities - Number of priority levels (minimum 1)
*/
constructor(priorities: number);Usage Example:
// Create queue with 5 priority levels (0-4, where 0 is highest)
const taskQueue = new multiPriorityQueue<string>(5);
// Add tasks with different priorities
taskQueue.enqueue("urgent-task", 0); // Highest priority
taskQueue.enqueue("normal-task", 2); // Normal priority
taskQueue.enqueue("low-task", 4); // Lowest priority
taskQueue.enqueue("high-task", 1); // High priority
console.log(taskQueue.size()); // 4Adds an item to the queue with specified priority level.
/**
* Add item to queue with priority
* @param value - Item to add to queue
* @param priority - Priority level (0 = highest priority)
* @throws RangeError if priority is out of range
*/
enqueue(value: T, priority: number): void;Usage Example:
const requestQueue = new multiPriorityQueue<RequestTask>(10);
// Add high priority request
requestQueue.enqueue({
url: "https://critical-api.com",
callback: criticalCallback
}, 0);
// Add normal priority request
requestQueue.enqueue({
url: "https://normal-site.com",
callback: normalCallback
}, 5);
// Invalid priority throws error
try {
requestQueue.enqueue(task, 15); // Priority 15 > max 9
} catch (error) {
console.error("Invalid priority:", error.message);
}Removes and returns the highest priority item from the queue.
/**
* Remove and return highest priority item
* @returns Highest priority item
* @throws ReferenceError if queue is empty
*/
dequeue(): T;Usage Example:
const queue = new multiPriorityQueue<string>(3);
queue.enqueue("low", 2);
queue.enqueue("high", 0);
queue.enqueue("medium", 1);
// Dequeue returns items in priority order
console.log(queue.dequeue()); // "high" (priority 0)
console.log(queue.dequeue()); // "medium" (priority 1)
console.log(queue.dequeue()); // "low" (priority 2)
// Empty queue throws error
try {
queue.dequeue();
} catch (error) {
console.error("Queue is empty:", error.message);
}Returns the current number of items in the queue.
/**
* Get current queue size
* @returns Number of items in queue across all priority levels
*/
size(): number;Simple FIFO (First In, First Out) queue implementation.
/**
* Basic FIFO queue implementation
* @template T - Type of items stored in the queue
*/
class Queue<T> {
constructor();
/** Add item to back of queue */
enqueue(value: T): number;
/** Remove and return item from front of queue */
dequeue(): T | undefined;
/** Check if queue is empty */
isEmpty(): boolean;
/** Get item at front without removing */
front(): T | undefined;
/** Get item at back without removing */
back(): T | undefined;
/** Number of items in queue (readonly) */
readonly length: number;
/** Number of items in queue (readonly, same as length) */
readonly size: number;
}Creates a new empty FIFO queue.
/**
* Creates a new empty queue
*/
constructor();Core queue operations for adding, removing, and inspecting items.
/**
* Add item to back of queue
* @param value - Item to add
* @returns New length of queue
*/
enqueue(value: T): number;
/**
* Remove and return item from front of queue
* @returns Item from front or undefined if empty
*/
dequeue(): T | undefined;
/**
* Check if queue is empty
* @returns True if queue has no items
*/
isEmpty(): boolean;
/**
* Get item at front without removing it
* @returns Item at front or undefined if empty
*/
front(): T | undefined;
/**
* Get item at back without removing it
* @returns Item at back or undefined if empty
*/
back(): T | undefined;Usage Example:
const basicQueue = new Queue<string>();
// Add items
console.log(basicQueue.enqueue("first")); // 1
console.log(basicQueue.enqueue("second")); // 2
console.log(basicQueue.enqueue("third")); // 3
// Inspect without removing
console.log(basicQueue.front()); // "first"
console.log(basicQueue.back()); // "third"
console.log(basicQueue.length); // 3
// Remove items in FIFO order
console.log(basicQueue.dequeue()); // "first"
console.log(basicQueue.dequeue()); // "second"
console.log(basicQueue.length); // 1
// Check if empty
console.log(basicQueue.isEmpty()); // false
basicQueue.dequeue(); // "third"
console.log(basicQueue.isEmpty()); // trueInternal node structures used by the queue implementation.
/**
* Base node interface for queue implementation
*/
interface AbstractNode {
next?: AbstractNode | null;
prev?: AbstractNode | null;
}
/**
* Queue node containing a value
* @template T - Type of the stored value
*/
class Node<T> implements AbstractNode {
constructor(value: T);
/** Stored value */
value: T;
/** Reference to next node */
next: AbstractNode | null;
/** Reference to previous node */
prev: AbstractNode | null;
}
/**
* Dummy head node for queue boundaries
*/
class DummyHeadNode implements AbstractNode {
constructor();
next: AbstractNode | null;
}
/**
* Dummy tail node for queue boundaries
*/
class DummyTailNode implements AbstractNode {
constructor();
prev: AbstractNode | null;
}// multiPriorityQueue is used internally by the crawler and not directly accessible
interface CrawlRequest {
url: string;
callback: Function;
retries: number;
}
// Create priority queue for crawl requests
const requestQueue = new multiPriorityQueue<CrawlRequest>(5);
// Add requests with different priorities
requestQueue.enqueue({
url: "https://critical-data.com/api",
callback: urgentHandler,
retries: 5
}, 0); // Highest priority
requestQueue.enqueue({
url: "https://news-site.com/articles",
callback: newsHandler,
retries: 2
}, 2); // Medium priority
requestQueue.enqueue({
url: "https://background-data.com/bulk",
callback: bulkHandler,
retries: 1
}, 4); // Lowest priority
// Process requests in priority order
while (requestQueue.size() > 0) {
const request = requestQueue.dequeue();
console.log(`Processing: ${request.url}`);
// Process the request
processRequest(request);
}// Different queues for different types of work
const highPriorityQueue = new multiPriorityQueue<Task>(3);
const normalQueue = new Queue<Task>();
const backgroundQueue = new Queue<Task>();
interface Task {
id: string;
execute: () => Promise<void>;
priority?: number;
}
class TaskScheduler {
private processing = false;
addTask(task: Task, queueType: 'high' | 'normal' | 'background' = 'normal') {
switch (queueType) {
case 'high':
highPriorityQueue.enqueue(task, task.priority || 1);
break;
case 'normal':
normalQueue.enqueue(task);
break;
case 'background':
backgroundQueue.enqueue(task);
break;
}
this.processNext();
}
private async processNext() {
if (this.processing) return;
this.processing = true;
let task: Task | undefined;
// Check queues in priority order
if (highPriorityQueue.size() > 0) {
task = highPriorityQueue.dequeue();
} else if (!normalQueue.isEmpty()) {
task = normalQueue.dequeue();
} else if (!backgroundQueue.isEmpty()) {
task = backgroundQueue.dequeue();
}
if (task) {
try {
await task.execute();
console.log(`Completed task: ${task.id}`);
} catch (error) {
console.error(`Task ${task.id} failed:`, error);
}
// Process next task
this.processing = false;
this.processNext();
} else {
this.processing = false;
}
}
}
// Usage
const scheduler = new TaskScheduler();
scheduler.addTask({
id: "urgent-backup",
execute: async () => { /* backup critical data */ },
priority: 0
}, 'high');
scheduler.addTask({
id: "daily-report",
execute: async () => { /* generate report */ }
}, 'normal');
scheduler.addTask({
id: "cleanup-logs",
execute: async () => { /* cleanup old logs */ }
}, 'background');class QueueMonitor {
private queues: Map<string, multiPriorityQueue<any> | Queue<any>>;
constructor() {
this.queues = new Map();
}
registerQueue(name: string, queue: multiPriorityQueue<any> | Queue<any>) {
this.queues.set(name, queue);
}
getStats() {
const stats = new Map();
for (const [name, queue] of this.queues) {
if (queue instanceof multiPriorityQueue) {
stats.set(name, {
type: 'multi-priority',
size: queue.size(),
isEmpty: queue.size() === 0
});
} else {
stats.set(name, {
type: 'basic',
size: queue.length,
isEmpty: queue.isEmpty(),
front: queue.isEmpty() ? null : queue.front(),
back: queue.isEmpty() ? null : queue.back()
});
}
}
return stats;
}
printStats() {
const stats = this.getStats();
console.log("Queue Statistics:");
for (const [name, stat] of stats) {
console.log(` ${name}: ${stat.type}, size: ${stat.size}, empty: ${stat.isEmpty}`);
}
}
}
// Usage
const monitor = new QueueMonitor();
const crawlerQueue = new multiPriorityQueue<string>(5);
const downloadQueue = new Queue<string>();
monitor.registerQueue('crawler', crawlerQueue);
monitor.registerQueue('downloads', downloadQueue);
// Add some items
crawlerQueue.enqueue("url1", 0);
crawlerQueue.enqueue("url2", 2);
downloadQueue.enqueue("file1");
downloadQueue.enqueue("file2");
// Monitor queues
setInterval(() => {
monitor.printStats();
}, 5000);The crawler uses these queue implementations internally:
// Internal usage in rate limiter
class RateLimiter {
private _waitingTasks: multiPriorityQueue<Task>;
constructor(options: RateLimiterOptions) {
this._waitingTasks = new multiPriorityQueue<Task>(options.priorityLevels);
}
submit(priority: number, task: Task): void {
this._waitingTasks.enqueue(task, priority);
this._schedule();
}
private dequeue(): Task | undefined {
return this._waitingTasks.dequeue();
}
}
// Usage with crawler
const crawler = new Crawler({
priorityLevels: 10, // Creates internal multi-priority queue with 10 levels
callback: (error, res, done) => {
console.log(`Processed: ${res.options.url}`);
done();
}
});
// Add requests with different priorities
crawler.add({ url: "https://urgent.com", priority: 0 }); // Highest
crawler.add({ url: "https://normal.com", priority: 5 }); // Medium
crawler.add({ url: "https://background.com", priority: 9 }); // Lowest