0
# Data Management
1

2
Data loading and management interfaces for importing external data from various sources into Pyomo models. Supports databases, spreadsheets, structured files, and programmatic data sources with validation and transformation capabilities.
3

4
## Capabilities
5

6
### Data Portal Interface
7

8
Main interface for loading external data into Pyomo models with support for multiple data sources and formats.
9

10
```python { .api }
11
class DataPortal:
12
    """
13
    Main data loading interface for Pyomo models.
14
    
15
    Provides unified interface for loading data from various sources
16
    including files, databases, and programmatic sources.
17
    """
18
    def __init__(self, **kwargs): ...
19
    
20
    def load(self, filename=None, data=None, model=None, **kwargs):
21
        """
22
        Load data from external source.
23
        
24
        Args:
25
            filename (str, optional): Data file path
26
            data (dict, optional): Direct data dictionary
27
            model (Model, optional): Target Pyomo model
28
            **kwargs: Additional loading options
29
        """
30
    
31
    def store(self, name, data=None, **kwargs):
32
        """
33
        Store data in the portal.
34
        
35
        Args:
36
            name (str): Data identifier
37
            data: Data to store
38
            **kwargs: Storage options
39
        """
40
    
41
    def connect(self, *args, **kwargs):
42
        """
43
        Connect to external data source.
44
        
45
        Args:
46
            *args, **kwargs: Connection parameters
47
        """
48
    
49
    def disconnect(self):
50
        """Disconnect from external data source."""
51
    
52
    def data(self, name=None):
53
        """
54
        Access stored data.
55
        
56
        Args:
57
            name (str, optional): Data identifier
58
            
59
        Returns:
60
            Data object or dictionary of all data
61
        """
62
    
63
    def keys(self):
64
        """
65
        Get names of all stored data.
66
        
67
        Returns:
68
            list: Data identifiers
69
        """
70
```
71

72
### Table Data Container
73

74
Container for tabular data with indexing and transformation capabilities.
75

76
```python { .api }
77
class TableData:
78
    """
79
    Tabular data container for structured data management.
80
    
81
    Handles tabular data with support for indexing, filtering,
82
    and transformation operations.
83
    """
84
    def __init__(self, **kwargs): ...
85
    
86
    def read(self, filename=None, **kwargs):
87
        """
88
        Read data from file.
89
        
90
        Args:
91
            filename (str): File path
92
            **kwargs: Reading options
93
        """
94
    
95
    def write(self, filename, **kwargs):
96
        """
97
        Write data to file.
98
        
99
        Args:
100
            filename (str): Output file path
101
            **kwargs: Writing options
102
        """
103
    
104
    def select(self, *args, **kwargs):
105
        """
106
        Select subset of data.
107
        
108
        Args:
109
            *args, **kwargs: Selection criteria
110
            
111
        Returns:
112
            TableData: Filtered data
113
        """
114
    
115
    def add_column(self, name, values):
116
        """
117
        Add column to table.
118
        
119
        Args:
120
            name (str): Column name
121
            values: Column values
122
        """
123
    
124
    def get_column(self, name):
125
        """
126
        Get column data.
127
        
128
        Args:
129
            name (str): Column name
130
            
131
        Returns:
132
            list: Column values
133
        """
134
```
135

136
### Data Manager Factory
137

138
Factory for creating data managers for different data source types and formats.
139

140
```python { .api }
141
class DataManagerFactory:
142
    """Factory for data managers."""
143
    
144
    @staticmethod
145
    def register(name, cls):
146
        """
147
        Register data manager class.
148
        
149
        Args:
150
            name (str): Manager name
151
            cls: Manager class
152
        """
153
    
154
    def __call__(self, manager_type, **kwargs):
155
        """
156
        Create data manager instance.
157
        
158
        Args:
159
            manager_type (str): Type of data manager
160
            **kwargs: Manager options
161
            
162
        Returns:
163
            Data manager instance
164
        """
165

166
class UnknownDataManager(Exception):
167
    """Exception raised for unknown data manager types."""
168
```
169

170
### Data Command Parsing
171

172
Utilities for parsing data commands and configuration files.
173

174
```python { .api }
175
def parse_datacmds(data_commands):
176
    """
177
    Parse data loading commands.
178
    
179
    Args:
180
        data_commands (str or list): Data commands to parse
181
        
182
    Returns:
183
        list: Parsed command structures
184
    """
185
```
186

187
## Usage Examples
188

189
### Loading Data from Files
190

191
```python
192
from pyomo.environ import *
193
from pyomo.dataportal import DataPortal
194

195
# Create model with abstract components
196
model = AbstractModel()
197
model.I = Set()
198
model.J = Set()
199
model.cost = Param(model.I, model.J)
200
model.demand = Param(model.I)
201
model.supply = Param(model.J)
202

203
# Create data portal
204
data = DataPortal()
205

206
# Load data from different file formats
207
data.load(filename='sets.dat')      # Load sets from .dat file
208
data.load(filename='params.csv')    # Load parameters from CSV
209
data.load(filename='data.xlsx')     # Load from Excel file
210
data.load(filename='data.json')     # Load from JSON file
211

212
# Create concrete model instance
213
instance = model.create_instance(data)
214
```
215

216
### Direct Data Loading
217

218
```python
219
from pyomo.environ import *
220
from pyomo.dataportal import DataPortal
221

222
model = AbstractModel()
223
model.CITIES = Set()
224
model.distance = Param(model.CITIES, model.CITIES)
225
model.demand = Param(model.CITIES)
226

227
# Load data directly
228
data = DataPortal()
229

230
# Load sets
231
data.load(data={
232
    'CITIES': ['NYC', 'LA', 'Chicago', 'Houston']
233
})
234

235
# Load parameter data
236
distance_data = {
237
    ('NYC', 'LA'): 2445,
238
    ('NYC', 'Chicago'): 713,
239
    ('NYC', 'Houston'): 1416,
240
    ('LA', 'Chicago'): 1745,
241
    ('LA', 'Houston'): 1374,
242
    ('Chicago', 'Houston'): 925
243
}
244

245
data.load(data={'distance': distance_data})
246

247
# Load demand data
248
data.load(data={
249
    'demand': {
250
        'NYC': 100,
251
        'LA': 80,
252
        'Chicago': 120,
253
        'Houston': 90
254
    }
255
})
256

257
instance = model.create_instance(data)
258
```
259

260
### Database Integration
261

262
```python
263
from pyomo.environ import *
264
from pyomo.dataportal import DataPortal
265

266
model = AbstractModel()
267
model.PRODUCTS = Set()
268
model.CUSTOMERS = Set()
269
model.price = Param(model.PRODUCTS)
270
model.orders = Param(model.CUSTOMERS, model.PRODUCTS)
271

272
# Connect to database
273
data = DataPortal()
274
data.connect('postgresql://user:pass@localhost:5432/mydb')
275

276
# Load sets from database tables
277
data.load(
278
    name='PRODUCTS',
279
    table='products',
280
    select='product_id',
281
    using='db'
282
)
283

284
data.load(
285
    name='CUSTOMERS', 
286
    table='customers',
287
    select='customer_id',
288
    using='db'
289
)
290

291
# Load parameters with SQL queries
292
data.load(
293
    name='price',
294
    table='products',
295
    select=['product_id', 'unit_price'],
296
    using='db'
297
)
298

299
data.load(
300
    name='orders',
301
    query='''
302
        SELECT customer_id, product_id, quantity 
303
        FROM orders 
304
        WHERE order_date >= '2023-01-01'
305
    ''',
306
    using='db'
307
)
308

309
data.disconnect()
310
instance = model.create_instance(data)
311
```
312

313
### Working with Spreadsheet Data
314

315
```python
316
from pyomo.environ import *
317
from pyomo.dataportal import DataPortal, TableData
318

319
# Load data from Excel file
320
table = TableData()
321
table.read('production_data.xlsx', sheet='costs')
322

323
# Process and filter data
324
cost_data = table.select('Product', 'Region', 'UnitCost')
325
filtered_data = cost_data.select(
326
    where=lambda row: row['Region'] in ['North', 'South']
327
)
328

329
# Create model and load processed data
330
model = AbstractModel()
331
model.PRODUCTS = Set()
332
model.REGIONS = Set()
333
model.unit_cost = Param(model.PRODUCTS, model.REGIONS)
334

335
data = DataPortal()
336

337
# Extract unique products and regions
338
products = set(row['Product'] for row in filtered_data)
339
regions = set(row['Region'] for row in filtered_data)
340

341
data.load(data={'PRODUCTS': products})
342
data.load(data={'REGIONS': regions})
343

344
# Load cost parameters
345
cost_dict = {}
346
for row in filtered_data:
347
    key = (row['Product'], row['Region'])
348
    cost_dict[key] = row['UnitCost']
349

350
data.load(data={'unit_cost': cost_dict})
351

352
instance = model.create_instance(data)
353
```
354

355
### Time Series Data Loading
356

357
```python
358
from pyomo.environ import *
359
from pyomo.dataportal import DataPortal
360
import pandas as pd
361

362
# Load time series data
363
df = pd.read_csv('timeseries.csv', parse_dates=['timestamp'])
364
df.set_index('timestamp', inplace=True)
365

366
# Create model with time-indexed parameters
367
model = AbstractModel()
368
model.TIME_PERIODS = Set()
369
model.demand = Param(model.TIME_PERIODS)
370
model.price = Param(model.TIME_PERIODS)
371

372
data = DataPortal()
373

374
# Convert time index to strings for Pyomo
375
time_periods = [str(t) for t in df.index]
376
data.load(data={'TIME_PERIODS': time_periods})
377

378
# Load time-indexed parameters
379
demand_data = {str(t): v for t, v in df['demand'].items()}
380
price_data = {str(t): v for t, v in df['price'].items()}
381

382
data.load(data={'demand': demand_data})
383
data.load(data={'price': price_data})
384

385
instance = model.create_instance(data)
386
```
387

388
### Data Validation and Transformation
389

390
```python
391
from pyomo.environ import *
392
from pyomo.dataportal import DataPortal
393

394
model = AbstractModel()
395
model.FACILITIES = Set()
396
model.capacity = Param(model.FACILITIES)
397
model.fixed_cost = Param(model.FACILITIES)
398

399
# Load data with validation
400
data = DataPortal()
401

402
# Load facility data
403
facilities_data = ['Plant1', 'Plant2', 'Plant3', 'Warehouse1', 'Warehouse2']
404
data.load(data={'FACILITIES': facilities_data})
405

406
# Load capacity data with validation
407
raw_capacity = {
408
    'Plant1': 1000,
409
    'Plant2': 1500, 
410
    'Plant3': 800,
411
    'Warehouse1': 2000,
412
    'Warehouse2': 1200
413
}
414

415
# Validate and transform data
416
validated_capacity = {}
417
for facility, cap in raw_capacity.items():
418
    if cap <= 0:
419
        raise ValueError(f"Invalid capacity for {facility}: {cap}")
420
    if facility.startswith('Plant') and cap < 500:
421
        print(f"Warning: Low capacity for {facility}: {cap}")
422
    validated_capacity[facility] = cap
423

424
data.load(data={'capacity': validated_capacity})
425

426
# Load fixed costs with transformation
427
base_costs = {
428
    'Plant1': 10000,
429
    'Plant2': 15000,
430
    'Plant3': 8000,
431
    'Warehouse1': 5000,
432
    'Warehouse2': 6000
433
}
434

435
# Apply inflation adjustment
436
inflation_factor = 1.03
437
adjusted_costs = {
438
    facility: cost * inflation_factor 
439
    for facility, cost in base_costs.items()
440
}
441

442
data.load(data={'fixed_cost': adjusted_costs})
443

444
instance = model.create_instance(data)
445
```
446

447
### Multi-Source Data Integration
448

449
```python
450
from pyomo.environ import *
451
from pyomo.dataportal import DataPortal
452

453
model = AbstractModel()
454
model.SUPPLIERS = Set()
455
model.PRODUCTS = Set()
456
model.CUSTOMERS = Set()
457
model.supply_cost = Param(model.SUPPLIERS, model.PRODUCTS)
458
model.transport_cost = Param(model.SUPPLIERS, model.CUSTOMERS)
459
model.demand = Param(model.CUSTOMERS, model.PRODUCTS)
460

461
data = DataPortal()
462

463
# Load sets from different sources
464
data.load(filename='suppliers.dat')  # Load SUPPLIERS set
465
data.load(filename='products.csv')   # Load PRODUCTS set  
466
data.load(filename='customers.json') # Load CUSTOMERS set
467

468
# Load supply costs from database
469
data.connect('sqlite:///supply_chain.db')
470
data.load(
471
    name='supply_cost',
472
    table='supplier_costs',
473
    select=['supplier_id', 'product_id', 'unit_cost'],
474
    using='db'
475
)
476

477
# Load transport costs from Excel
478
data.load(
479
    filename='transport_costs.xlsx',
480
    sheet='costs',
481
    param='transport_cost'
482
)
483

484
# Load demand from API or programmatic source
485
import requests
486
demand_response = requests.get('http://api.example.com/demand')
487
demand_data = demand_response.json()
488

489
processed_demand = {}
490
for entry in demand_data:
491
    key = (entry['customer'], entry['product'])
492
    processed_demand[key] = entry['quantity']
493

494
data.load(data={'demand': processed_demand})
495

496
data.disconnect()
497
instance = model.create_instance(data)
498
```