or run

npx @tessl/cli init

Version

Tile

Overview

Evals

Files

docs

http-client.md index.md logger.md mappers.md repository-management.md state-management.md types-extraction.md types-loading.md worker-management.md

tile.json

HTTP Client

The Airdrop SDK provides pre-configured Axios HTTP clients with automatic retry logic and error handling for resilient network operations.

Capabilities

Axios Client

Pre-configured Axios instance with automatic retry capabilities.

/**
 * Pre-configured Axios client with retry logic
 * - 5 retries with exponential backoff
 * - Retries on network errors, idempotent requests, and 5xx errors
 * - Excludes 429 (rate limit) from automatic retries
 * - Removes sensitive headers from error logs
 */
const axiosClient: AxiosInstance;

Usage Examples:

import { axiosClient } from '@devrev/ts-adaas';

// GET request with automatic retries
const response = await axiosClient.get('https://api.example.com/data', {
  params: { page: 1, limit: 100 },
  headers: {
    Authorization: `Bearer ${apiKey}`,
  },
});

// POST request with automatic retries
const createResponse = await axiosClient.post(
  'https://api.example.com/items',
  {
    title: 'New Item',
    description: 'Item description',
  },
  {
    headers: {
      Authorization: `Bearer ${apiKey}`,
      'Content-Type': 'application/json',
    },
  }
);

// PUT request (idempotent, will retry)
await axiosClient.put('https://api.example.com/items/123', {
  title: 'Updated Title',
});

// DELETE request
await axiosClient.delete('https://api.example.com/items/123', {
  headers: {
    Authorization: `Bearer ${apiKey}`,
  },
});

Original Axios Instance

The original axios instance for custom configurations.

/**
 * Original axios instance without retry configuration
 * Use for custom configurations or when retries are not desired
 */
const axios: AxiosInstance;

Usage Examples:

import { axios } from '@devrev/ts-adaas';

// Use original axios for custom configuration
const customClient = axios.create({
  baseURL: 'https://api.example.com',
  timeout: 5000,
  headers: {
    'Custom-Header': 'value',
  },
});

// Make request without retry logic
const response = await axios.get('https://api.example.com/data');

Retry Configuration

The axiosClient is configured with the following retry behavior:

Retry Conditions

Network errors: Requests where no response is received
Idempotent requests: GET, HEAD, OPTIONS, PUT (by default)
Server errors: All 5xx status codes (500-599)
Exclusions: 429 (Too Many Requests) is NOT automatically retried

Retry Strategy

Maximum retries: 5 attempts
Backoff: Exponential backoff with formula: 1 * 2^retryCount * 1000ms
- 1st retry: ~2 seconds
- 2nd retry: ~4 seconds
- 3rd retry: ~8 seconds
- 4th retry: ~16 seconds
- 5th retry: ~32 seconds

Error Handling

Sensitive headers (authorization) are removed from error logs
Warning logs are generated for each retry attempt
On max retries exceeded, a final warning is logged

Usage Patterns

Basic GET Request

import { axiosClient } from '@devrev/ts-adaas';
import { processTask, ExtractorEventType } from '@devrev/ts-adaas';

processTask({
  task: async ({ adapter }) => {
    try {
      const response = await axiosClient.get('https://api.example.com/tasks', {
        headers: {
          Authorization: `Bearer ${adapter.event.payload.connection_data.key}`,
        },
        params: {
          page: 1,
          per_page: 100,
        },
      });

      const tasks = response.data;
      await adapter.getRepo('tasks')?.push(tasks);

      await adapter.emit(ExtractorEventType.ExtractionDataDone);
    } catch (error) {
      console.error('Failed to fetch tasks:', error);
      await adapter.emit(ExtractorEventType.ExtractionDataError, {
        error: { message: `Failed to fetch tasks: ${error.message}` },
      });
    }
  },
  onTimeout: async ({ adapter }) => {
    await adapter.emit(ExtractorEventType.ExtractionDataError, {
      error: { message: 'Timeout' },
    });
  },
});

Paginated Requests

processTask({
  task: async ({ adapter }) => {
    let page = 1;
    let hasMore = true;

    while (hasMore) {
      const response = await axiosClient.get('https://api.example.com/items', {
        params: { page, per_page: 100 },
      });

      const items = response.data.items;
      await adapter.getRepo('items')?.push(items);

      hasMore = response.data.has_more;
      page++;
    }

    await adapter.emit(ExtractorEventType.ExtractionDataDone);
  },
  onTimeout: async ({ adapter }) => {
    await adapter.emit(ExtractorEventType.ExtractionDataError, {
      error: { message: 'Timeout' },
    });
  },
});

Handling Rate Limits

processTask({
  task: async ({ adapter }) => {
    try {
      const response = await axiosClient.get('https://api.example.com/data');
      await processData(response.data);
      await adapter.emit(ExtractorEventType.ExtractionDataDone);
    } catch (error) {
      // Check for rate limiting (429 is not auto-retried)
      if (error.response?.status === 429) {
        const retryAfter = parseInt(error.response.headers['retry-after'] || '60');

        // Emit delay event
        await adapter.emit(ExtractorEventType.ExtractionDataDelay, {
          delay: retryAfter,
        });
      } else {
        await adapter.emit(ExtractorEventType.ExtractionDataError, {
          error: { message: error.message },
        });
      }
    }
  },
  onTimeout: async ({ adapter }) => {
    await adapter.emit(ExtractorEventType.ExtractionDataError, {
      error: { message: 'Timeout' },
    });
  },
});

POST Request for Loading

import { axiosClient } from '@devrev/ts-adaas';
import { processTask, LoaderEventType } from '@devrev/ts-adaas';

processTask({
  task: async ({ adapter }) => {
    const items = await getItemsToLoad();

    for (const item of items) {
      try {
        const response = await axiosClient.post(
          'https://api.example.com/items',
          {
            title: item.data.title,
            description: item.data.description,
          },
          {
            headers: {
              Authorization: `Bearer ${adapter.event.payload.connection_data.key}`,
              'Content-Type': 'application/json',
            },
          }
        );

        const externalId = response.data.id;

        // Create mapping
        await adapter.mappers.create({
          sync_unit: adapter.event.payload.event_context.sync_unit,
          external_ids: [externalId],
          targets: [item.id.devrev],
          status: 'operational',
        });
      } catch (error) {
        console.error(`Failed to create item ${item.id.devrev}:`, error);
      }
    }

    await adapter.emit(LoaderEventType.DataLoadingDone);
  },
  onTimeout: async ({ adapter }) => {
    await adapter.emit(LoaderEventType.DataLoadingError, {
      error: { message: 'Timeout' },
    });
  },
});

Custom Client Configuration

import { axios } from '@devrev/ts-adaas';
import axiosRetry from 'axios-retry';

// Create custom client with different retry configuration
const customClient = axios.create({
  baseURL: 'https://api.example.com',
  timeout: 10000,
});

// Configure custom retry logic
axiosRetry(customClient, {
  retries: 3,
  retryDelay: (retryCount) => retryCount * 2000, // Linear backoff
  retryCondition: (error) => {
    return error.response?.status === 503; // Only retry on 503
  },
});

// Use custom client
const response = await customClient.get('/data');

Streaming Responses

processTask({
  task: async ({ adapter }) => {
    const attachments = await getAttachmentsToExtract();

    for (const attachment of attachments) {
      try {
        // Stream attachment download
        const response = await axiosClient.get(attachment.url, {
          responseType: 'stream',
        });

        // Process stream
        await processAttachmentStream(response.data);
      } catch (error) {
        console.error(`Failed to download attachment ${attachment.id}:`, error);
      }
    }

    await adapter.emit(ExtractorEventType.ExtractionAttachmentsDone);
  },
  onTimeout: async ({ adapter }) => {
    await adapter.emit(ExtractorEventType.ExtractionAttachmentsError, {
      error: { message: 'Timeout' },
    });
  },
});

Best Practices

Always Use Authorization Headers

const apiKey = adapter.event.payload.connection_data.key;

const response = await axiosClient.get('https://api.example.com/data', {
  headers: {
    Authorization: `Bearer ${apiKey}`,
  },
});

Handle Errors Gracefully

try {
  const response = await axiosClient.get('https://api.example.com/data');
  return response.data;
} catch (error) {
  if (error.response) {
    // Server responded with error status
    console.error('Server error:', error.response.status, error.response.data);
  } else if (error.request) {
    // Request made but no response received
    console.error('Network error:', error.message);
  } else {
    // Error setting up request
    console.error('Request error:', error.message);
  }
  throw error;
}

Use Appropriate Methods

// GET for retrieval (idempotent, will retry)
await axiosClient.get('/items/123');

// POST for creation (not idempotent, won't retry by default)
await axiosClient.post('/items', data);

// PUT for full updates (idempotent, will retry)
await axiosClient.put('/items/123', data);

// PATCH for partial updates (not idempotent, won't retry by default)
await axiosClient.patch('/items/123', partialData);

// DELETE for removal (idempotent, will retry)
await axiosClient.delete('/items/123');

Timeout Configuration

// Set per-request timeout
await axiosClient.get('https://api.example.com/data', {
  timeout: 30000, // 30 seconds
});

// Create client with default timeout
const client = axios.create({
  baseURL: 'https://api.example.com',
  timeout: 10000, // 10 seconds default
});