0
# Flyteidl
1

2
The Interface Definition Language (IDL) for the Flyte workflow orchestration platform, providing comprehensive protobuf specifications, gRPC service definitions, and generated client libraries for multiple programming languages. This package serves as the core communication layer for Flyte's distributed architecture, enabling strongly-typed interfaces for workflow definitions, execution metadata, task specifications, and administrative operations.
3

4
## Package Information
5

6
- **Package Name**: flyteidl
7
- **Package Type**: pypi (also available as Go module, npm package, Rust crate)
8
- **Language**: Protocol Buffers with generated clients in Python, Go, JavaScript/TypeScript, Rust
9
- **Installation**: `pip install flyteidl`
10

11
## Core Imports
12

13
### Python
14

15
```python
16
# Core protobuf messages
17
from flyteidl.core import literals_pb2, tasks_pb2, workflow_pb2, types_pb2
18
from flyteidl.admin import task_pb2, workflow_pb2, execution_pb2
19

20
# gRPC service clients
21
from flyteidl.service import admin_pb2_grpc
22
from flyteidl.service.admin_pb2_grpc import AdminServiceStub
23

24
# Plugin types
25
from flyteidl.plugins import spark_pb2, pytorch_pb2, array_job_pb2
26
```
27

28
### Go
29

30
```go
31
import (
32
    "github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/core"
33
    "github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/admin"
34
    "github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/service"
35
    "github.com/flyteorg/flyte/flyteidl/clients/go/admin"
36
)
37
```
38

39
### JavaScript/TypeScript
40

41
```typescript
42
import { AdminServiceClient } from '@flyteorg/flyteidl/service/admin_connect';
43
import { Task, Workflow, Execution } from '@flyteorg/flyteidl/admin/admin_pb';
44
import { Literal, LiteralType } from '@flyteorg/flyteidl/core/literals_pb';
45
```
46

47
### Rust
48

49
```rust
50
use flyteidl::flyteidl::core::{Literal, LiteralType, TaskTemplate};
51
use flyteidl::flyteidl::admin::{Task, Workflow, Execution};
52
use flyteidl::flyteidl::service::admin::AdminServiceClient;
53
```
54

55
## Basic Usage
56

57
### Python Client Example
58

59
```python
60
import grpc
61
from flyteidl.service.admin_pb2_grpc import AdminServiceStub
62
from flyteidl.admin.task_pb2 import TaskCreateRequest
63
from flyteidl.core.identifier_pb2 import Identifier
64
from flyteidl.core.tasks_pb2 import TaskTemplate
65

66
# Create gRPC channel and client
67
channel = grpc.insecure_channel('localhost:8089')
68
client = AdminServiceStub(channel)
69

70
# Create a task identifier
71
task_id = Identifier(
72
    resource_type=0,  # TASK
73
    project="my-project",
74
    domain="development", 
75
    name="my-task",
76
    version="v1.0.0"
77
)
78

79
# Create task template
80
task_template = TaskTemplate(
81
    id=task_id,
82
    type="python-task",
83
    metadata=None,
84
    interface=None,
85
    custom={}
86
)
87

88
# Create task via Admin API
89
request = TaskCreateRequest(id=task_id, spec=task_template)
90
response = client.CreateTask(request)
91
```
92

93
### Go Client Example
94

95
```go
96
package main
97

98
import (
99
    "context"
100
    
101
    "github.com/flyteorg/flyte/flyteidl/clients/go/admin"
102
    "github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/core"
103
    adminpb "github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/admin"
104
)
105

106
func main() {
107
    // Create admin client with configuration
108
    cfg := admin.GetConfig()
109
    adminClient, err := admin.NewAdminClient(context.Background(), cfg)
110
    if err != nil {
111
        panic(err)
112
    }
113

114
    // Create task identifier
115
    taskID := &core.Identifier{
116
        ResourceType: core.ResourceType_TASK,
117
        Project:      "my-project",
118
        Domain:       "development",
119
        Name:         "my-task",
120
        Version:      "v1.0.0",
121
    }
122

123
    // Get task from Admin API
124
    task, err := adminClient.GetTask(context.Background(), &adminpb.ObjectGetRequest{
125
        Id: taskID,
126
    })
127
    if err != nil {
128
        panic(err)
129
    }
130
}
131
```
132

133
## Architecture
134

135
Flyteidl follows a layered architecture designed for maximum interoperability:
136

137
- **Service Layer**: gRPC service definitions (AdminService, DataProxyService, etc.) providing the API contract
138
- **Core Types**: Fundamental data structures (Literals, Identifiers, Tasks, Workflows) used across all components  
139
- **Administrative Types**: Management entities (Projects, Executions, Resource Attributes) for operational control
140
- **Plugin System**: Extensible task types (Spark, ML frameworks, array jobs) for diverse compute requirements
141
- **Multi-language Support**: Identical APIs across Python, Go, JavaScript/TypeScript, and Rust via protocol buffers
142

143
This design enables seamless integration across different technology stacks while maintaining strong type safety and API consistency.
144

145
## Capabilities
146

147
### Administrative API
148

149
The primary interface for managing Flyte resources including tasks, workflows, launch plans, executions, and projects. Provides comprehensive CRUD operations, state management, and resource attributes configuration.
150

151
```python { .api }
152
class AdminServiceStub:
153
    def CreateTask(self, request: TaskCreateRequest) -> TaskCreateResponse: ...
154
    def GetTask(self, request: ObjectGetRequest) -> Task: ...
155
    def ListTasks(self, request: ResourceListRequest) -> TaskList: ...
156
    def CreateWorkflow(self, request: WorkflowCreateRequest) -> WorkflowCreateResponse: ...
157
    def CreateExecution(self, request: ExecutionCreateRequest) -> ExecutionCreateResponse: ...
158
    def GetExecution(self, request: WorkflowExecutionGetRequest) -> Execution: ...
159
    def ListExecutions(self, request: ResourceListRequest) -> ExecutionList: ...
160
    def TerminateExecution(self, request: ExecutionTerminateRequest) -> ExecutionTerminateResponse: ...
161
```
162

163
[Administrative API](./admin-api.md)
164

165
### Core Types and Literals
166

167
Foundational type system providing strongly-typed data handling, identifier management, and interface definitions. Includes the Literal system for runtime type safety and data serialization across language boundaries.
168

169
```python { .api }
170
class Literal:
171
    scalar: Scalar
172
    collection: LiteralCollection
173
    map: LiteralMap
174

175
class LiteralType:
176
    simple: SimpleType
177
    blob: BlobType
178
    collection_type: LiteralType
179
    map_value_type: LiteralType
180
    schema: SchemaType
181
    structured_dataset_type: StructuredDatasetType
182
    union_type: UnionType
183

184
class Identifier:
185
    resource_type: ResourceType
186
    project: str
187
    domain: str
188
    name: str
189
    version: str
190
```
191

192
[Core Types and Literals](./core-types.md)
193

194
### Task and Workflow Management
195

196
Comprehensive definitions for tasks and workflows including templates, specifications, bindings, and execution models. Supports complex workflow composition with conditional branching, parallel execution, and data flow management.
197

198
```python { .api }
199
class TaskTemplate:
200
    id: Identifier
201
    type: str
202
    metadata: TaskMetadata
203
    interface: TypedInterface
204
    custom: dict
205
    security_context: SecurityContext
206
    
207
class WorkflowTemplate:
208
    id: Identifier
209
    metadata: WorkflowMetadata
210
    interface: TypedInterface
211
    nodes: List[Node]
212
    outputs: List[Binding]
213

214
class Node:
215
    id: str
216
    metadata: NodeMetadata
217
    inputs: List[Binding]
218
    upstream_node_ids: List[str]
219
    output_aliases: List[Alias]
220
    task_node: TaskNode
221
    workflow_node: WorkflowNode
222
    branch_node: BranchNode
223
    gate_node: GateNode
224
```
225

226
[Task and Workflow Management](./tasks-workflows.md)
227

228
### Plugin System
229

230
Extensible plugin framework supporting diverse compute requirements including Apache Spark, machine learning frameworks (TensorFlow, PyTorch), distributed computing (Dask, Ray), and specialized execution patterns (array jobs, MPI).
231

232
```python { .api }
233
class SparkJob:
234
    spark_conf: dict
235
    application_file: str
236
    executor_path: str
237
    main_class: str
238
    spark_type: SparkType
239

240
class PyTorchJob:
241
    workers: int
242
    master_replicas: DistributedPyTorchTrainingReplicaSpec
243
    worker_replicas: DistributedPyTorchTrainingReplicaSpec
244

245
class ArrayJob:
246
    parallelism: int
247
    size: int
248
    min_successes: int
249
    min_success_ratio: float
250
```
251

252
[Plugin System](./plugins.md)
253

254
### Data Management
255

256
Data catalog and caching services providing versioned artifact storage, metadata management, tagging systems, and concurrent access coordination. Enables efficient data sharing and lineage tracking across workflow executions.
257

258
```python { .api }
259
class Dataset:
260
    id: DatasetID
261
    metadata: Metadata
262
    partition_keys: List[str]
263

264
class Artifact:
265
    id: ArtifactID
266
    dataset: DatasetID
267
    data: List[ArtifactData]
268
    metadata: Metadata
269
    partitions: List[Partition]
270
    tags: List[Tag]
271

272
# Cache service operations
273
def Get(request: GetCacheRequest) -> GetCacheResponse: ...
274
def Put(request: PutCacheRequest) -> PutCacheResponse: ...
275
def Delete(request: DeleteCacheRequest) -> DeleteCacheResponse: ...
276
```
277

278
[Data Management](./data-management.md)
279

280
### Multi-language Support
281

282
Identical APIs across Python, Go, JavaScript/TypeScript, and Rust with language-specific client utilities, authentication support, and idiomatic patterns. Includes rich client libraries for advanced features like OAuth2 token management and type conversion helpers.
283

284
```python { .api }
285
# Python
286
from flyteidl.service.admin_pb2_grpc import AdminServiceStub
287

288
// Go  
289
import "github.com/flyteorg/flyte/flyteidl/clients/go/admin"
290
type AdminClient interface {
291
    CreateTask(ctx context.Context, request *admin.TaskCreateRequest) (*admin.TaskCreateResponse, error)
292
    GetTask(ctx context.Context, request *admin.ObjectGetRequest) (*admin.Task, error)
293
}
294

295
// TypeScript
296
import { AdminServiceClient } from '@flyteorg/flyteidl/service/admin_connect';
297

298
// Rust
299
use flyteidl::flyteidl::service::admin::AdminServiceClient;
300
```
301

302
[Multi-language Support](./multi-language.md)
303

304
## Types
305

306
### Core Identifiers
307

308
```python { .api }
309
class ResourceType:
310
    UNSPECIFIED = 0
311
    TASK = 1
312
    WORKFLOW = 2 
313
    LAUNCH_PLAN = 3
314
    DATASET = 4
315

316
class Identifier:
317
    resource_type: ResourceType
318
    project: str
319
    domain: str
320
    name: str
321
    version: str
322
    org: str
323
    
324
class WorkflowExecutionIdentifier:
325
    project: str
326
    domain: str
327
    name: str
328
    org: str
329
    
330
class NodeExecutionIdentifier:
331
    node_id: str
332
    execution_id: WorkflowExecutionIdentifier
333
    
334
class TaskExecutionIdentifier:
335
    task_id: Identifier
336
    node_execution_id: NodeExecutionIdentifier
337
    retry_attempt: int
338
```
339

340
### Data Types
341

342
```python { .api }
343
class SimpleType:
344
    NONE = 0
345
    INTEGER = 1
346
    FLOAT = 2
347
    STRING = 3
348
    BOOLEAN = 4
349
    DATETIME = 5
350
    DURATION = 6
351
    BINARY = 7
352
    ERROR = 8
353
    STRUCT = 9
354

355
class Scalar:
356
    primitive: Primitive
357
    blob: Blob
358
    binary: Binary
359
    schema: Schema
360
    none_type: Void
361
    error: Error
362
    generic: Struct
363
    structured_dataset: StructuredDataset
364
    union: Union
365

366
class Primitive:
367
    integer: int
368
    float_value: float
369
    string_value: str
370
    boolean: bool
371
    datetime: datetime
372
    duration: timedelta
373
```