or run

npx @tessl/cli init
Log in

Version

Tile

Overview

Evals

Files

docs

http-client.mdindex.mdlogger.mdmappers.mdrepository-management.mdstate-management.mdtypes-extraction.mdtypes-loading.mdworker-management.md
tile.json

types-loading.mddocs/

Loading Types

Type definitions for loading workflows in the Airdrop SDK. These types define the structure of events, data, and responses used during data loading from DevRev to external systems.

Capabilities

Loader Event Types

Events used during loading operations to communicate progress and status.

/**
 * LoaderEventType defines events sent from loaders to Airdrop platform
 */
enum LoaderEventType {
  // Data Loading
  DataLoadingProgress = 'DATA_LOADING_PROGRESS',
  DataLoadingDelay = 'DATA_LOADING_DELAYED',
  DataLoadingDone = 'DATA_LOADING_DONE',
  DataLoadingError = 'DATA_LOADING_ERROR',

  // Attachment Loading
  AttachmentLoadingProgress = 'ATTACHMENT_LOADING_PROGRESS',
  AttachmentLoadingDelayed = 'ATTACHMENT_LOADING_DELAYED',
  AttachmentLoadingDone = 'ATTACHMENT_LOADING_DONE',
  AttachmentLoadingError = 'ATTACHMENT_LOADING_ERROR',

  // State Deletion
  LoaderStateDeletionDone = 'LOADER_STATE_DELETION_DONE',
  LoaderStateDeletionError = 'LOADER_STATE_DELETION_ERROR',
  LoaderAttachmentStateDeletionDone = 'LOADER_ATTACHMENT_STATE_DELETION_DONE',
  LoaderAttachmentStateDeletionError = 'LOADER_ATTACHMENT_STATE_DELETION_ERROR',

  // Unknown
  UnknownEventType = 'UNKNOWN_EVENT_TYPE',
}

/**
 * ActionType represents the outcome of a loading operation
 */
enum ActionType {
  CREATED = 'created',
  UPDATED = 'updated',
  SKIPPED = 'skipped',
  DELETED = 'deleted',
  FAILED = 'failed',
}

Usage Examples:

import { processTask, LoaderEventType, ActionType } from '@devrev/ts-adaas';

processTask({
  task: async ({ adapter }) => {
    // Report progress during loading
    await adapter.emit(LoaderEventType.DataLoadingProgress);

    // Complete loading with report
    await adapter.emit(LoaderEventType.DataLoadingDone, {
      reports: [
        {
          item_type: 'tasks',
          [ActionType.CREATED]: 10,
          [ActionType.UPDATED]: 5,
          [ActionType.FAILED]: 2,
        },
      ],
    });
  },
  onTimeout: async ({ adapter }) => {
    await adapter.emit(LoaderEventType.DataLoadingError, {
      error: { message: 'Timeout' },
    });
  },
});

External System Item

Represents an item to be loaded into an external system.

/**
 * ExternalSystemItem represents data to be loaded to external system
 */
interface ExternalSystemItem {
  /** Item identifiers */
  id: {
    /** DevRev object ID */
    devrev: DonV2;
    /** Optional external system ID */
    external?: string;
  };
  /** RFC3339 timestamp when item was created */
  created_date: string;
  /** RFC3339 timestamp when item was last modified */
  modified_date: string;
  /** Item data payload */
  data: any;
}

/**
 * DonV2 is a DevRev object identifier string
 */
type DonV2 = string;

Usage Examples:

processTask({
  task: async ({ adapter }) => {
    // Read items to load from state
    const filesToLoad = adapter.state.fromDevRev?.filesToLoad || [];

    for (const file of filesToLoad) {
      const items: ExternalSystemItem[] = await readItemsFromFile(file);

      for (const item of items) {
        // Access item information
        const devrevId = item.id.devrev;
        const existingExternalId = item.id.external;
        const itemData = item.data;

        // Load to external system
        await loadItemToExternalSystem(item);
      }
    }
  },
  onTimeout: async ({ adapter }) => {
    await adapter.emit(LoaderEventType.DataLoadingError, {
      error: { message: 'Timeout' },
    });
  },
});

External System Attachment

Represents an attachment to be loaded into an external system.

/**
 * ExternalSystemAttachment represents an attachment for loading
 */
interface ExternalSystemAttachment {
  /** Reference ID of the attachment */
  reference_id: DonV2;
  /** Type of parent object */
  parent_type: string;
  /** Reference ID of parent object */
  parent_reference_id: DonV2;
  /** File name */
  file_name: string;
  /** MIME type of file */
  file_type: string;
  /** File size in bytes */
  file_size: number;
  /** URL to download attachment */
  url: string;
  /** ISO 8601 timestamp when URL expires */
  valid_until: string;
  /** ID of user who created attachment */
  created_by_id: string;
  /** RFC3339 timestamp when attachment was created */
  created_date: string;
  /** ID of user who last modified attachment */
  modified_by_id: string;
  /** RFC3339 timestamp when attachment was last modified */
  modified_date: string;
  /** Optional parent ID in external system */
  parent_id?: string;
  /** Optional grandparent ID */
  grand_parent_id?: string;
}

Usage Examples:

processTask({
  task: async ({ adapter }) => {
    const attachments: ExternalSystemAttachment[] = await getAttachmentsToLoad();

    for (const attachment of attachments) {
      // Download attachment from DevRev
      const file = await downloadFile(attachment.url);

      // Upload to external system
      const externalId = await uploadToExternalSystem({
        file,
        fileName: attachment.file_name,
        fileType: attachment.file_type,
        parentId: attachment.parent_id,
      });

      // Record mapping
      await adapter.mappers.create({
        sync_unit: adapter.event.payload.event_context.sync_unit,
        external_ids: [externalId],
        targets: [attachment.reference_id],
        status: 'operational',
      });
    }

    await adapter.emit(LoaderEventType.AttachmentLoadingDone);
  },
  onTimeout: async ({ adapter }) => {
    await adapter.emit(LoaderEventType.AttachmentLoadingError, {
      error: { message: 'Timeout' },
    });
  },
});

Loading Function Types

Function signatures for loading items to external systems.

/**
 * Parameters passed to loading functions
 */
interface ExternalSystemItemLoadingParams<Type> {
  /** Item to load */
  item: Type;
  /** Mappers instance for sync mapper operations */
  mappers: Mappers;
  /** Airdrop event context */
  event: AirdropEvent;
}

/**
 * Response from loading functions
 */
interface ExternalSystemItemLoadingResponse {
  /** External system ID if item was created/updated */
  id?: string;
  /** Error message if loading failed */
  error?: string;
  /** Modified date of item in external system */
  modifiedDate?: string;
  /** Delay in seconds if rate limited */
  delay?: number;
}

/**
 * Result of loading a single item
 */
interface ExternalSystemItemLoadedItem {
  /** External system ID if successful */
  id?: string;
  /** Error message if failed */
  error?: string;
  /** Modified date in external system */
  modifiedDate?: string;
}

/**
 * Function type for loading items to external system
 */
type ExternalSystemLoadingFunction<Item> = (
  params: ExternalSystemItemLoadingParams<Item>
) => Promise<ExternalSystemItemLoadingResponse>;

Usage Examples:

// Define create function
const createItem: ExternalSystemLoadingFunction<ExternalSystemItem> = async ({
  item,
  mappers,
  event,
}) => {
  try {
    // Create in external system
    const externalId = await externalApi.createItem(item.data);

    // Create mapping
    await mappers.create({
      sync_unit: event.payload.event_context.sync_unit,
      external_ids: [externalId],
      targets: [item.id.devrev],
      status: 'operational',
    });

    return {
      id: externalId,
      modifiedDate: new Date().toISOString(),
    };
  } catch (error) {
    if (error.response?.status === 429) {
      return { delay: 60 }; // Rate limited
    }
    return { error: error.message };
  }
};

// Define update function
const updateItem: ExternalSystemLoadingFunction<ExternalSystemItem> = async ({
  item,
  mappers,
  event,
}) => {
  try {
    // Get existing mapping
    const mapping = await mappers.getByTargetId({
      sync_unit: event.payload.event_context.sync_unit,
      target: item.id.devrev,
    });

    const externalId = mapping.data.sync_mapper_record.external_ids[0];

    // Update in external system
    await externalApi.updateItem(externalId, item.data);

    return {
      id: externalId,
      modifiedDate: new Date().toISOString(),
    };
  } catch (error) {
    return { error: error.message };
  }
};

Item Type Configuration

Configuration for loading different item types.

/**
 * Configuration for loading a specific item type
 */
interface ItemTypeToLoad {
  /** Type identifier for the item */
  itemType: string;
  /** Function to create items in external system */
  create: ExternalSystemLoadingFunction<ExternalSystemItem>;
  /** Function to update items in external system */
  update: ExternalSystemLoadingFunction<ExternalSystemItem>;
}

/**
 * Parameters for configuring multiple item types
 */
interface ItemTypesToLoadParams {
  /** Array of item type configurations */
  itemTypesToLoad: ItemTypeToLoad[];
}

Usage Examples:

const itemTypesToLoad: ItemTypeToLoad[] = [
  {
    itemType: 'tasks',
    create: createTask,
    update: updateTask,
  },
  {
    itemType: 'comments',
    create: createComment,
    update: updateComment,
  },
];

// Use in loading workflow
processTask({
  task: async ({ adapter }) => {
    for (const config of itemTypesToLoad) {
      const items = await getItemsOfType(config.itemType);

      for (const item of items) {
        // Determine if create or update
        const hasExternalId = !!item.id.external;

        const result = hasExternalId
          ? await config.update({ item, mappers: adapter.mappers, event: adapter.event })
          : await config.create({ item, mappers: adapter.mappers, event: adapter.event });

        if (result.delay) {
          await adapter.emit(LoaderEventType.DataLoadingDelay, { delay: result.delay });
          return;
        }
      }
    }

    await adapter.emit(LoaderEventType.DataLoadingDone);
  },
  onTimeout: async ({ adapter }) => {
    await adapter.emit(LoaderEventType.DataLoadingError, {
      error: { message: 'Timeout' },
    });
  },
});

Loader Reports

Reports summarizing loading operation results.

/**
 * LoaderReport summarizes loading results for an item type
 */
interface LoaderReport {
  /** Item type identifier */
  item_type: string;
  /** Number of items created (optional) */
  [ActionType.CREATED]?: number;
  /** Number of items updated (optional) */
  [ActionType.UPDATED]?: number;
  /** Number of items skipped (optional) */
  [ActionType.SKIPPED]?: number;
  /** Number of items deleted (optional) */
  [ActionType.DELETED]?: number;
  /** Number of items that failed (optional) */
  [ActionType.FAILED]?: number;
}

/**
 * Response containing loading results
 */
interface LoadItemResponse {
  /** Error if loading failed */
  error?: ErrorRecord;
  /** Report of loading results */
  report?: LoaderReport;
  /** Rate limit information */
  rateLimit?: RateLimited;
}

/**
 * Response containing results for multiple item types
 */
interface LoadItemTypesResponse {
  /** Reports for each item type */
  reports: LoaderReport[];
  /** List of processed file names */
  processed_files: string[];
}

Usage Examples:

processTask({
  task: async ({ adapter }) => {
    const reports: LoaderReport[] = [];

    // Load tasks
    const taskResults = await loadItems('tasks');
    reports.push({
      item_type: 'tasks',
      [ActionType.CREATED]: taskResults.created,
      [ActionType.UPDATED]: taskResults.updated,
      [ActionType.FAILED]: taskResults.failed,
    });

    // Load comments
    const commentResults = await loadItems('comments');
    reports.push({
      item_type: 'comments',
      [ActionType.CREATED]: commentResults.created,
      [ActionType.UPDATED]: commentResults.updated,
      [ActionType.FAILED]: commentResults.failed,
    });

    // Emit completion with reports
    await adapter.emit(LoaderEventType.DataLoadingDone, {
      reports,
      processed_files: ['tasks.jsonl', 'comments.jsonl'],
    });
  },
  onTimeout: async ({ adapter }) => {
    await adapter.emit(LoaderEventType.DataLoadingError, {
      error: { message: 'Timeout' },
    });
  },
});

Rate Limiting

Rate limit information for handling API throttling.

/**
 * RateLimited contains rate limit delay information
 */
interface RateLimited {
  /** Delay in seconds to wait before retrying */
  delay: number;
}

Usage Examples:

processTask({
  task: async ({ adapter }) => {
    const result = await loadItemsToExternalSystem();

    if (result.rateLimit) {
      // Emit delay event
      await adapter.emit(LoaderEventType.DataLoadingDelay, {
        delay: result.rateLimit.delay,
      });
      return;
    }

    await adapter.emit(LoaderEventType.DataLoadingDone);
  },
  onTimeout: async ({ adapter }) => {
    await adapter.emit(LoaderEventType.DataLoadingError, {
      error: { message: 'Timeout' },
    });
  },
});

File To Load

Information about files containing items to load.

/**
 * FileToLoad represents a file containing items to be loaded
 */
interface FileToLoad {
  /** File identifier */
  id: string;
  /** File name */
  file_name: string;
  /** Type of items in file */
  itemType: string;
  /** Total number of items in file */
  count: number;
  /** Line number of next item to process */
  lineToProcess: number;
  /** Whether file processing is completed */
  completed: boolean;
}

Usage Examples:

processTask({
  task: async ({ adapter }) => {
    // Get files to load from state
    const filesToLoad = adapter.state.fromDevRev?.filesToLoad || [];

    for (const file of filesToLoad) {
      if (file.completed) {
        continue; // Skip completed files
      }

      // Process items starting from lineToProcess
      const items = await readItemsFromFile(file.file_name, file.lineToProcess);

      for (const item of items) {
        await loadItem(item);
        file.lineToProcess++;
      }

      // Mark file as completed
      file.completed = file.lineToProcess >= file.count;

      // Update state
      adapter.state = adapter.state;
    }

    await adapter.emit(LoaderEventType.DataLoadingDone);
  },
  onTimeout: async ({ adapter }) => {
    // State is automatically saved with current lineToProcess
    await adapter.emit(LoaderEventType.DataLoadingError, {
      error: { message: 'Timeout' },
    });
  },
});

Stats File

File statistics for tracking processing progress.

/**
 * StatsFileObject contains statistics about a processed file
 */
interface StatsFileObject {
  /** File identifier */
  id: string;
  /** Item type in file */
  item_type: string;
  /** File name */
  file_name: string;
  /** Count of items processed */
  count: string;
}

Sync Mapper Record (Loading Context)

Simplified sync mapper record structure used in loading context.

/**
 * SyncMapperRecord links external IDs to DevRev IDs
 */
type SyncMapperRecord = {
  /** External system identifiers */
  external_ids: string[];
  /** Secondary identifiers for context-specific lookups */
  secondary_ids: string[];
  /** DevRev entity IDs */
  devrev_ids: string[];
  /** Record status values */
  status: string[];
  /** Optional input file reference */
  input_file?: string;
};

Common Patterns

Loading Workflow Pattern

processTask({
  task: async ({ adapter }) => {
    // 1. Read items to load
    const filesToLoad = adapter.state.fromDevRev?.filesToLoad || [];

    const reports: LoaderReport[] = [];

    // 2. Process each file
    for (const file of filesToLoad) {
      const items = await readItemsFromFile(file);
      const stats = { created: 0, updated: 0, failed: 0 };

      // 3. Load each item
      for (const item of items) {
        const result = await loadItem(item, adapter);

        if (result.error) {
          stats.failed++;
        } else if (item.id.external) {
          stats.updated++;
        } else {
          stats.created++;
        }
      }

      // 4. Create report
      reports.push({
        item_type: file.itemType,
        [ActionType.CREATED]: stats.created,
        [ActionType.UPDATED]: stats.updated,
        [ActionType.FAILED]: stats.failed,
      });
    }

    // 5. Emit completion
    await adapter.emit(LoaderEventType.DataLoadingDone, { reports });
  },
  onTimeout: async ({ adapter }) => {
    await adapter.emit(LoaderEventType.DataLoadingError, {
      error: { message: 'Timeout' },
    });
  },
});

Error Recovery Pattern

processTask({
  task: async ({ adapter }) => {
    const results = { created: 0, updated: 0, failed: 0 };

    for (const item of items) {
      try {
        const result = await loadItem(item);

        if (result.id) {
          item.id.external ? results.updated++ : results.created++;
        }
      } catch (error) {
        console.error(`Failed to load item ${item.id.devrev}:`, error);
        results.failed++;
        // Continue processing remaining items
      }
    }

    await adapter.emit(LoaderEventType.DataLoadingDone, {
      reports: [
        {
          item_type: 'items',
          [ActionType.CREATED]: results.created,
          [ActionType.UPDATED]: results.updated,
          [ActionType.FAILED]: results.failed,
        },
      ],
    });
  },
  onTimeout: async ({ adapter }) => {
    await adapter.emit(LoaderEventType.DataLoadingError, {
      error: { message: 'Timeout' },
    });
  },
});