CtrlK
BlogDocsLog inGet started
Tessl Logo

tessl/pypi-flyteidl

IDL for Flyte Platform containing protobuf specifications, gRPC API definitions, and generated clients for multiple languages

—

Pending
Overview
Eval results
Files

multi-language.mddocs/

Multi-language Support

Identical APIs across Python, Go, JavaScript/TypeScript, and Rust with language-specific client utilities, authentication support, and idiomatic patterns. The multi-language support ensures consistent developer experience while providing language-specific optimizations and idioms.

Capabilities

Python Client Support

Comprehensive Python client with protobuf message classes and gRPC service stubs.

# Core imports for Python
from flyteidl.core import literals_pb2, tasks_pb2, workflow_pb2, types_pb2, identifier_pb2
from flyteidl.admin import task_pb2, workflow_pb2, execution_pb2, project_pb2
from flyteidl.service import admin_pb2_grpc, dataproxy_pb2_grpc, auth_pb2_grpc

# Service client creation
import grpc
from flyteidl.service.admin_pb2_grpc import AdminServiceStub

def create_admin_client(endpoint: str) -> AdminServiceStub:
    """Create gRPC admin client."""
    channel = grpc.insecure_channel(endpoint)
    return AdminServiceStub(channel)

# Plugin imports
from flyteidl.plugins import spark_pb2, pytorch_pb2, array_job_pb2, dask_pb2
from flyteidl.plugins.kubeflow import tensorflow_pb2, mpi_pb2

# Data management imports  
from flyteidl.datacatalog import datacatalog_pb2
from flyteidl.cacheservice import cacheservice_pb2

Go Client Support

Rich Go client libraries with authentication, configuration management, and idiomatic patterns.

// Core imports for Go
import (
    "github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/core"
    "github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/admin"
    "github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/service"
    "github.com/flyteorg/flyte/flyteidl/clients/go/admin"
)

// AdminClient interface with rich functionality
type AdminClient interface {
    CreateTask(ctx context.Context, request *admin.TaskCreateRequest) (*admin.TaskCreateResponse, error)
    GetTask(ctx context.Context, request *admin.ObjectGetRequest) (*admin.Task, error)
    ListTasks(ctx context.Context, request *admin.ResourceListRequest) (*admin.TaskList, error)
    CreateWorkflow(ctx context.Context, request *admin.WorkflowCreateRequest) (*admin.WorkflowCreateResponse, error)
    CreateExecution(ctx context.Context, request *admin.ExecutionCreateRequest) (*admin.ExecutionCreateResponse, error)
    GetExecution(ctx context.Context, request *admin.WorkflowExecutionGetRequest) (*admin.Execution, error)
    ListExecutions(ctx context.Context, request *admin.ResourceListRequest) (*admin.ExecutionList, error)
    TerminateExecution(ctx context.Context, request *admin.ExecutionTerminateRequest) (*admin.ExecutionTerminateResponse, error)
}

// Client factory function
func NewAdminClient(ctx context.Context, cfg *Config) (AdminClient, error)

// Configuration management
type Config struct {
    Endpoint        string
    UseInsecure     bool
    MaxRetries      int
    PerRetryTimeout time.Duration
    Credentials     Credentials
}

// Authentication support
type Credentials struct {
    ClientID         string
    ClientSecret     string
    AuthorizeURL     string
    TokenURL         string
    RedirectURL      string
    Scopes           []string
    UseAudienceFromAdmin bool
}

Go Authentication Utilities

Advanced OAuth2 authentication with token management and device flow support.

// Token orchestrator for OAuth2 flows
type TokenOrchestrator interface {
    FetchTokenFromCacheOrRefreshIt(ctx context.Context) (*oauth2.Token, error)
    FetchTokenFromRefreshFlow(ctx context.Context) (*oauth2.Token, error)
    FetchTokenFromClientCredentialsFlow(ctx context.Context) (*oauth2.Token, error)
    FetchTokenFromDeviceFlow(ctx context.Context) (*oauth2.Token, error)
}

// Device flow support
type DeviceFlowTokenSource struct {
    cfg         *oauth2.Config
    tokenURL    string
    audience    string
    httpClient  *http.Client
}

// PKCE (Proof Key for Code Exchange) support  
type PKCETokenSource struct {
    cfg          *oauth2.Config
    verifier     string
    authURL      string
    httpClient   *http.Client
}

// Certificate handling for mTLS
type CertificateManager interface {
    GetClientCertificate() (tls.Certificate, error)
    GetRootCAs() (*x509.CertPool, error)
}

Go Client Utilities

Helper functions and utilities for working with Flyte types and data structures.

// Literal utilities for type conversion
func MakeLiteral(v interface{}) (*core.Literal, error)
func ExtractFromLiteral(lit *core.Literal) (interface{}, error)
func MakeLiteralForType(t *core.LiteralType, v interface{}) (*core.Literal, error)

// Identifier utilities
func MakeIdentifier(resourceType core.ResourceType, project, domain, name, version string) *core.Identifier
func MakeWorkflowExecutionIdentifier(project, domain, name string) *core.WorkflowExecutionIdentifier

// Type utilities
func MakePrimitive(v interface{}) (*core.Primitive, error)
func MakeScalar(v interface{}) (*core.Scalar, error)
func MakeLiteralType(t reflect.Type) (*core.LiteralType, error)

// Interface utilities
func MakeTypedInterface(inputs, outputs map[string]*core.LiteralType) *core.TypedInterface
func MakeVariable(t *core.LiteralType, description string) *core.Variable

JavaScript/TypeScript Support

Modern TypeScript support with Connect-style clients and full type definitions.

// Core imports for TypeScript
import { AdminServiceClient } from '@flyteorg/flyteidl/service/admin_connect';
import { Task, Workflow, Execution } from '@flyteorg/flyteidl/admin/admin_pb';
import { Literal, LiteralType, LiteralMap } from '@flyteorg/flyteidl/core/literals_pb';
import { Identifier, ResourceType } from '@flyteorg/flyteidl/core/identifier_pb';
import { TaskTemplate, WorkflowTemplate } from '@flyteorg/flyteidl/core/tasks_pb';

// Client creation with transport
import { createGrpcWebTransport } from "@connectrpc/connect-web";

function createAdminClient(baseUrl: string): AdminServiceClient {
    const transport = createGrpcWebTransport({
        baseUrl,
        credentials: "include",
    });
    return new AdminServiceClient(transport);
}

// Plugin type imports
import { SparkJob } from '@flyteorg/flyteidl/plugins/spark_pb';
import { PyTorchJob } from '@flyteorg/flyteidl/plugins/kubeflow/pytorch_pb';
import { ArrayJob } from '@flyteorg/flyteidl/plugins/array_job_pb';

// Data management imports
import { Dataset, Artifact } from '@flyteorg/flyteidl/datacatalog/datacatalog_pb';

TypeScript Type Definitions

Complete TypeScript definitions with proper type safety and IntelliSense support.

// Literal type utilities
interface LiteralValue {
    scalar?: Scalar;
    collection?: LiteralCollection;
    map?: LiteralMap;
}

interface LiteralTypeDefinition {
    simple?: SimpleType;
    blob?: BlobType;
    collection_type?: LiteralType;
    map_value_type?: LiteralType;
    schema?: SchemaType;
    structured_dataset_type?: StructuredDatasetType;
    union_type?: UnionType;
}

// Client configuration
interface ClientConfig {
    endpoint: string;
    credentials?: RequestCredentials;
    timeout?: number;
    retries?: number;
}

// Authentication configuration
interface AuthConfig {
    clientId: string;
    clientSecret?: string;
    authUrl: string;
    tokenUrl: string;
    scopes: string[];
}

Rust Support

Rust crate with Tonic gRPC support and idiomatic Rust patterns.

// Core imports for Rust
use flyteidl::flyteidl::core::{Literal, LiteralType, Identifier, TaskTemplate, WorkflowTemplate};
use flyteidl::flyteidl::admin::{Task, Workflow, Execution, AdminServiceClient};
use flyteidl::flyteidl::service::admin::admin_service_client::AdminServiceClient;

// Client creation with tonic
use tonic::transport::Channel;
use tonic::Request;

pub async fn create_admin_client(endpoint: String) -> Result<AdminServiceClient<Channel>, Box<dyn std::error::Error>> {
    let channel = Channel::from_shared(endpoint)?
        .connect()
        .await?;
    Ok(AdminServiceClient::new(channel))
}

// Plugin imports
use flyteidl::flyteidl::plugins::{SparkJob, ArrayJob};
use flyteidl::flyteidl::plugins::kubeflow::{PyTorchJob, TensorFlowJob};

// Data management
use flyteidl::flyteidl::datacatalog::{Dataset, Artifact, DataCatalogServiceClient};
use flyteidl::flyteidl::cacheservice::{CacheServiceClient, GetCacheRequest};

Rust Type Conversions

Rust-specific utilities for working with protobuf types and conversions.

// Literal creation utilities
impl Literal {
    pub fn from_primitive<T>(value: T) -> Self where T: Into<Primitive> { ... }
    pub fn from_collection(values: Vec<Literal>) -> Self { ... }
    pub fn from_map(values: std::collections::HashMap<String, Literal>) -> Self { ... }
    pub fn to_value<T>(&self) -> Result<T, ConversionError> where T: TryFrom<&Literal> { ... }
}

// Identifier creation
impl Identifier {
    pub fn new(resource_type: ResourceType, project: String, domain: String, name: String, version: String) -> Self { ... }
    pub fn task(project: String, domain: String, name: String, version: String) -> Self { ... }
    pub fn workflow(project: String, domain: String, name: String, version: String) -> Self { ... }
}

// Error handling
#[derive(Debug, thiserror::Error)]
pub enum FlyteidlError {
    #[error("gRPC transport error: {0}")]
    Transport(#[from] tonic::transport::Error),
    #[error("gRPC status error: {0}")]
    Status(#[from] tonic::Status),
    #[error("Conversion error: {0}")]
    Conversion(String),
}

Language-Specific Usage Examples

Python Usage

import grpc
from flyteidl.service.admin_pb2_grpc import AdminServiceStub
from flyteidl.admin.task_pb2 import TaskCreateRequest
from flyteidl.core.identifier_pb2 import Identifier, ResourceType

# Create client
channel = grpc.insecure_channel('localhost:8089')
admin_client = AdminServiceStub(channel)

# Create task identifier
task_id = Identifier(
    resource_type=ResourceType.TASK,
    project="my-project",
    domain="development",
    name="my-task",
    version="v1.0.0"
)

# Create and register task
request = TaskCreateRequest(id=task_id, spec=task_template)
response = admin_client.CreateTask(request)

Go Usage

package main

import (
    "context"
    "github.com/flyteorg/flyte/flyteidl/clients/go/admin"
    "github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/core"
    adminpb "github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/admin"
)

func main() {
    // Create client with authentication
    cfg := &admin.Config{
        Endpoint: "flyte.example.com",
        UseInsecure: false,
        Credentials: admin.Credentials{
            ClientID: "flytectl",
            ClientSecret: "secret",
            TokenURL: "https://flyte.example.com/oauth2/token",
        },
    }
    
    client, err := admin.NewAdminClient(context.Background(), cfg)
    if err != nil {
        panic(err)
    }

    // Create task identifier
    taskID := &core.Identifier{
        ResourceType: core.ResourceType_TASK,
        Project:      "my-project",
        Domain:       "development",
        Name:         "my-task",
        Version:      "v1.0.0",
    }

    // Get task
    task, err := client.GetTask(context.Background(), &adminpb.ObjectGetRequest{
        Id: taskID,
    })
    if err != nil {
        panic(err)
    }
}

TypeScript Usage

import { AdminServiceClient } from '@flyteorg/flyteidl/service/admin_connect';
import { ObjectGetRequest } from '@flyteorg/flyteidl/admin/admin_pb';
import { Identifier, ResourceType } from '@flyteorg/flyteidl/core/identifier_pb';
import { createGrpcWebTransport } from "@connectrpc/connect-web";

// Create client
const transport = createGrpcWebTransport({
    baseUrl: "https://flyte.example.com",
    credentials: "include",
});
const adminClient = new AdminServiceClient(transport);

// Create task identifier
const taskId = new Identifier({
    resourceType: ResourceType.TASK,
    project: "my-project",
    domain: "development",
    name: "my-task",
    version: "v1.0.0"
});

// Get task
const request = new ObjectGetRequest({ id: taskId });
const task = await adminClient.getTask(request);

Rust Usage

use flyteidl::flyteidl::service::admin::admin_service_client::AdminServiceClient;
use flyteidl::flyteidl::admin::ObjectGetRequest;
use flyteidl::flyteidl::core::{Identifier, ResourceType};
use tonic::transport::Channel;

#[tokio::main]
async fn main() -> Result<(), Box<dyn std::error::Error>> {
    // Create client
    let channel = Channel::from_static("http://localhost:8089")
        .connect()
        .await?;
    let mut client = AdminServiceClient::new(channel);

    // Create task identifier
    let task_id = Identifier {
        resource_type: ResourceType::Task as i32,
        project: "my-project".to_string(),
        domain: "development".to_string(),
        name: "my-task".to_string(),
        version: "v1.0.0".to_string(),
    };

    // Get task
    let request = tonic::Request::new(ObjectGetRequest {
        id: Some(task_id),
    });
    
    let response = client.get_task(request).await?;
    let task = response.into_inner();

    Ok(())
}

Installation and Setup

Python Installation

pip install flyteidl

# With specific dependencies for gRPC
pip install flyteidl[grpc]

# Development installation  
pip install flyteidl[dev]

Go Installation

go get github.com/flyteorg/flyte/flyteidl

# Import in go.mod
require github.com/flyteorg/flyte/flyteidl v1.16.0

JavaScript/TypeScript Installation

npm install @flyteorg/flyteidl

# With Connect dependencies
npm install @flyteorg/flyteidl @connectrpc/connect @connectrpc/connect-web

Rust Installation

Add to Cargo.toml:

[dependencies]
flyteidl = "1.16.0"
tonic = "0.8"
prost = "0.11"
tokio = { version = "1.0", features = ["full"] }

Cross-Language Compatibility

All language implementations provide identical API surfaces with:

  • Protocol Buffer Compatibility: Identical message structures across languages
  • gRPC Service Compatibility: Same service definitions and method signatures
  • Type Safety: Strong typing with language-appropriate type systems
  • Error Handling: Consistent error codes and handling patterns
  • Authentication: OAuth2 and certificate-based authentication support
  • Configuration: Similar configuration patterns adapted to language conventions

Install with Tessl CLI

npx tessl i tessl/pypi-flyteidl

docs

admin-api.md

core-types.md

data-management.md

index.md

multi-language.md

plugins.md

tasks-workflows.md

tile.json