tessl/pypi-pyomo
The Pyomo optimization modeling framework for formulating, analyzing, and solving mathematical optimization problems
—
Pending
data-management.mddocs/
Data Management
Data loading and management interfaces for importing external data from various sources into Pyomo models. Supports databases, spreadsheets, structured files, and programmatic data sources with validation and transformation capabilities.
Capabilities
Data Portal Interface
Main interface for loading external data into Pyomo models with support for multiple data sources and formats.
class DataPortal:
"""
Main data loading interface for Pyomo models.
Provides unified interface for loading data from various sources
including files, databases, and programmatic sources.
"""
def __init__(self, **kwargs): ...
def load(self, filename=None, data=None, model=None, **kwargs):
"""
Load data from external source.
Args:
filename (str, optional): Data file path
data (dict, optional): Direct data dictionary
model (Model, optional): Target Pyomo model
**kwargs: Additional loading options
"""
def store(self, name, data=None, **kwargs):
"""
Store data in the portal.
Args:
name (str): Data identifier
data: Data to store
**kwargs: Storage options
"""
def connect(self, *args, **kwargs):
"""
Connect to external data source.
Args:
*args, **kwargs: Connection parameters
"""
def disconnect(self):
"""Disconnect from external data source."""
def data(self, name=None):
"""
Access stored data.
Args:
name (str, optional): Data identifier
Returns:
Data object or dictionary of all data
"""
def keys(self):
"""
Get names of all stored data.
Returns:
list: Data identifiers
"""Table Data Container
Container for tabular data with indexing and transformation capabilities.
class TableData:
"""
Tabular data container for structured data management.
Handles tabular data with support for indexing, filtering,
and transformation operations.
"""
def __init__(self, **kwargs): ...
def read(self, filename=None, **kwargs):
"""
Read data from file.
Args:
filename (str): File path
**kwargs: Reading options
"""
def write(self, filename, **kwargs):
"""
Write data to file.
Args:
filename (str): Output file path
**kwargs: Writing options
"""
def select(self, *args, **kwargs):
"""
Select subset of data.
Args:
*args, **kwargs: Selection criteria
Returns:
TableData: Filtered data
"""
def add_column(self, name, values):
"""
Add column to table.
Args:
name (str): Column name
values: Column values
"""
def get_column(self, name):
"""
Get column data.
Args:
name (str): Column name
Returns:
list: Column values
"""Data Manager Factory
Factory for creating data managers for different data source types and formats.
class DataManagerFactory:
"""Factory for data managers."""
@staticmethod
def register(name, cls):
"""
Register data manager class.
Args:
name (str): Manager name
cls: Manager class
"""
def __call__(self, manager_type, **kwargs):
"""
Create data manager instance.
Args:
manager_type (str): Type of data manager
**kwargs: Manager options
Returns:
Data manager instance
"""
class UnknownDataManager(Exception):
"""Exception raised for unknown data manager types."""Data Command Parsing
Utilities for parsing data commands and configuration files.
def parse_datacmds(data_commands):
"""
Parse data loading commands.
Args:
data_commands (str or list): Data commands to parse
Returns:
list: Parsed command structures
"""Usage Examples
Loading Data from Files
from pyomo.environ import *
from pyomo.dataportal import DataPortal
# Create model with abstract components
model = AbstractModel()
model.I = Set()
model.J = Set()
model.cost = Param(model.I, model.J)
model.demand = Param(model.I)
model.supply = Param(model.J)
# Create data portal
data = DataPortal()
# Load data from different file formats
data.load(filename='sets.dat') # Load sets from .dat file
data.load(filename='params.csv') # Load parameters from CSV
data.load(filename='data.xlsx') # Load from Excel file
data.load(filename='data.json') # Load from JSON file
# Create concrete model instance
instance = model.create_instance(data)Direct Data Loading
from pyomo.environ import *
from pyomo.dataportal import DataPortal
model = AbstractModel()
model.CITIES = Set()
model.distance = Param(model.CITIES, model.CITIES)
model.demand = Param(model.CITIES)
# Load data directly
data = DataPortal()
# Load sets
data.load(data={
'CITIES': ['NYC', 'LA', 'Chicago', 'Houston']
})
# Load parameter data
distance_data = {
('NYC', 'LA'): 2445,
('NYC', 'Chicago'): 713,
('NYC', 'Houston'): 1416,
('LA', 'Chicago'): 1745,
('LA', 'Houston'): 1374,
('Chicago', 'Houston'): 925
}
data.load(data={'distance': distance_data})
# Load demand data
data.load(data={
'demand': {
'NYC': 100,
'LA': 80,
'Chicago': 120,
'Houston': 90
}
})
instance = model.create_instance(data)Database Integration
from pyomo.environ import *
from pyomo.dataportal import DataPortal
model = AbstractModel()
model.PRODUCTS = Set()
model.CUSTOMERS = Set()
model.price = Param(model.PRODUCTS)
model.orders = Param(model.CUSTOMERS, model.PRODUCTS)
# Connect to database
data = DataPortal()
data.connect('postgresql://user:pass@localhost:5432/mydb')
# Load sets from database tables
data.load(
name='PRODUCTS',
table='products',
select='product_id',
using='db'
)
data.load(
name='CUSTOMERS',
table='customers',
select='customer_id',
using='db'
)
# Load parameters with SQL queries
data.load(
name='price',
table='products',
select=['product_id', 'unit_price'],
using='db'
)
data.load(
name='orders',
query='''
SELECT customer_id, product_id, quantity
FROM orders
WHERE order_date >= '2023-01-01'
''',
using='db'
)
data.disconnect()
instance = model.create_instance(data)Working with Spreadsheet Data
from pyomo.environ import *
from pyomo.dataportal import DataPortal, TableData
# Load data from Excel file
table = TableData()
table.read('production_data.xlsx', sheet='costs')
# Process and filter data
cost_data = table.select('Product', 'Region', 'UnitCost')
filtered_data = cost_data.select(
where=lambda row: row['Region'] in ['North', 'South']
)
# Create model and load processed data
model = AbstractModel()
model.PRODUCTS = Set()
model.REGIONS = Set()
model.unit_cost = Param(model.PRODUCTS, model.REGIONS)
data = DataPortal()
# Extract unique products and regions
products = set(row['Product'] for row in filtered_data)
regions = set(row['Region'] for row in filtered_data)
data.load(data={'PRODUCTS': products})
data.load(data={'REGIONS': regions})
# Load cost parameters
cost_dict = {}
for row in filtered_data:
key = (row['Product'], row['Region'])
cost_dict[key] = row['UnitCost']
data.load(data={'unit_cost': cost_dict})
instance = model.create_instance(data)Time Series Data Loading
from pyomo.environ import *
from pyomo.dataportal import DataPortal
import pandas as pd
# Load time series data
df = pd.read_csv('timeseries.csv', parse_dates=['timestamp'])
df.set_index('timestamp', inplace=True)
# Create model with time-indexed parameters
model = AbstractModel()
model.TIME_PERIODS = Set()
model.demand = Param(model.TIME_PERIODS)
model.price = Param(model.TIME_PERIODS)
data = DataPortal()
# Convert time index to strings for Pyomo
time_periods = [str(t) for t in df.index]
data.load(data={'TIME_PERIODS': time_periods})
# Load time-indexed parameters
demand_data = {str(t): v for t, v in df['demand'].items()}
price_data = {str(t): v for t, v in df['price'].items()}
data.load(data={'demand': demand_data})
data.load(data={'price': price_data})
instance = model.create_instance(data)Data Validation and Transformation
from pyomo.environ import *
from pyomo.dataportal import DataPortal
model = AbstractModel()
model.FACILITIES = Set()
model.capacity = Param(model.FACILITIES)
model.fixed_cost = Param(model.FACILITIES)
# Load data with validation
data = DataPortal()
# Load facility data
facilities_data = ['Plant1', 'Plant2', 'Plant3', 'Warehouse1', 'Warehouse2']
data.load(data={'FACILITIES': facilities_data})
# Load capacity data with validation
raw_capacity = {
'Plant1': 1000,
'Plant2': 1500,
'Plant3': 800,
'Warehouse1': 2000,
'Warehouse2': 1200
}
# Validate and transform data
validated_capacity = {}
for facility, cap in raw_capacity.items():
if cap <= 0:
raise ValueError(f"Invalid capacity for {facility}: {cap}")
if facility.startswith('Plant') and cap < 500:
print(f"Warning: Low capacity for {facility}: {cap}")
validated_capacity[facility] = cap
data.load(data={'capacity': validated_capacity})
# Load fixed costs with transformation
base_costs = {
'Plant1': 10000,
'Plant2': 15000,
'Plant3': 8000,
'Warehouse1': 5000,
'Warehouse2': 6000
}
# Apply inflation adjustment
inflation_factor = 1.03
adjusted_costs = {
facility: cost * inflation_factor
for facility, cost in base_costs.items()
}
data.load(data={'fixed_cost': adjusted_costs})
instance = model.create_instance(data)Multi-Source Data Integration
from pyomo.environ import *
from pyomo.dataportal import DataPortal
model = AbstractModel()
model.SUPPLIERS = Set()
model.PRODUCTS = Set()
model.CUSTOMERS = Set()
model.supply_cost = Param(model.SUPPLIERS, model.PRODUCTS)
model.transport_cost = Param(model.SUPPLIERS, model.CUSTOMERS)
model.demand = Param(model.CUSTOMERS, model.PRODUCTS)
data = DataPortal()
# Load sets from different sources
data.load(filename='suppliers.dat') # Load SUPPLIERS set
data.load(filename='products.csv') # Load PRODUCTS set
data.load(filename='customers.json') # Load CUSTOMERS set
# Load supply costs from database
data.connect('sqlite:///supply_chain.db')
data.load(
name='supply_cost',
table='supplier_costs',
select=['supplier_id', 'product_id', 'unit_cost'],
using='db'
)
# Load transport costs from Excel
data.load(
filename='transport_costs.xlsx',
sheet='costs',
param='transport_cost'
)
# Load demand from API or programmatic source
import requests
demand_response = requests.get('http://api.example.com/demand')
demand_data = demand_response.json()
processed_demand = {}
for entry in demand_data:
key = (entry['customer'], entry['product'])
processed_demand[key] = entry['quantity']
data.load(data={'demand': processed_demand})
data.disconnect()
instance = model.create_instance(data)Install with Tessl CLI
npx tessl i tessl/pypi-pyomo