tessl/pypi-orange3
Orange, a component-based data mining framework.
—
Pending
Quality
Pending
Does it follow best practices?
Impact
Pending
No eval scenarios have been run
evaluation.mddocs/
Model Evaluation and Validation
Orange3 provides a comprehensive evaluation framework for assessing machine learning model performance through various validation techniques and metrics.
Capabilities
Cross-Validation Methods
Different strategies for splitting data and validating model performance.
class CrossValidation:
"""
k-fold cross-validation.
Args:
data: Dataset for evaluation
learners: List of learners to evaluate
k: Number of folds
stratified: Use stratified sampling
random_state: Random seed for reproducibility
"""
def __init__(self, data, learners, k=10, stratified=True, random_state=None): ...
class LeaveOneOut:
"""
Leave-one-out cross-validation.
Args:
data: Dataset for evaluation
learners: List of learners to evaluate
"""
def __init__(self, data, learners): ...
class TestOnTrainingData:
"""
Evaluate on the same data used for training.
Args:
data: Training dataset
learners: List of learners to evaluate
"""
def __init__(self, data, learners): ...
class TestOnTestData:
"""
Train on one dataset, test on another.
Args:
train_data: Training dataset
test_data: Test dataset
learners: List of learners to evaluate
"""
def __init__(self, train_data, test_data, learners): ...Classification Metrics
Performance measures for classification tasks.
def CA(results):
"""
Classification Accuracy.
Args:
results: Evaluation results object
Returns:
ndarray: Accuracy scores for each learner
"""
def AUC(results):
"""
Area Under the ROC Curve.
Args:
results: Evaluation results object
Returns:
ndarray: AUC scores for each learner
"""
def Precision(results, pos_label=1, average='binary'):
"""
Precision score.
Args:
results: Evaluation results object
pos_label: Positive class label
average: Averaging strategy
Returns:
ndarray: Precision scores
"""
def Recall(results, pos_label=1, average='binary'):
"""
Recall score.
Args:
results: Evaluation results object
pos_label: Positive class label
average: Averaging strategy
Returns:
ndarray: Recall scores
"""
def F1(results, pos_label=1, average='binary'):
"""
F1 score.
Args:
results: Evaluation results object
pos_label: Positive class label
average: Averaging strategy
Returns:
ndarray: F1 scores
"""
def PrecisionRecallFSupport(results, pos_label=1, average='binary'):
"""
Combined precision, recall, and F1 scores.
Args:
results: Evaluation results object
pos_label: Positive class label
average: Averaging strategy
Returns:
tuple: (precision, recall, f1, support) arrays
"""Regression Metrics
Performance measures for regression tasks.
def MSE(results):
"""
Mean Squared Error.
Args:
results: Evaluation results object
Returns:
ndarray: MSE scores for each learner
"""
def RMSE(results):
"""
Root Mean Squared Error.
Args:
results: Evaluation results object
Returns:
ndarray: RMSE scores for each learner
"""
def MAE(results):
"""
Mean Absolute Error.
Args:
results: Evaluation results object
Returns:
ndarray: MAE scores for each learner
"""
def R2(results):
"""
R-squared coefficient of determination.
Args:
results: Evaluation results object
Returns:
ndarray: R² scores for each learner
"""Advanced Classification Metrics
Specialized metrics for detailed model analysis.
def LogLoss(results):
"""
Logarithmic loss for probabilistic predictions.
Args:
results: Evaluation results object
Returns:
ndarray: Log loss scores
"""
def Specificity(results, pos_label=1):
"""
Specificity (True Negative Rate).
Args:
results: Evaluation results object
pos_label: Positive class label
Returns:
ndarray: Specificity scores
"""
def Sensitivity(results, pos_label=1):
"""
Sensitivity (True Positive Rate, same as Recall).
Args:
results: Evaluation results object
pos_label: Positive class label
Returns:
ndarray: Sensitivity scores
"""Results Container
Object containing evaluation results and predictions.
class Results:
"""
Container for evaluation results.
Attributes:
data: Original dataset
predicted: Predicted values
probabilities: Prediction probabilities (if available)
actual: True target values
learners: List of learners used
folds: Cross-validation fold information
"""
def __init__(self, data=None, learners=None): ...
@property
def predicted(self):
"""Predicted class labels."""
@property
def probabilities(self):
"""Prediction probabilities."""
@property
def actual(self):
"""True class labels."""Clustering Evaluation
Metrics for unsupervised learning evaluation.
class ClusteringEvaluation:
"""Evaluation methods for clustering algorithms."""
@staticmethod
def adjusted_rand_score(labels_true, labels_pred):
"""Adjusted Rand Index."""
@staticmethod
def silhouette_score(X, labels):
"""Silhouette coefficient."""
@staticmethod
def calinski_harabasz_score(X, labels):
"""Calinski-Harabasz index."""Statistical Testing
Statistical significance testing for model comparison.
def compute_CD(avranks, n, alpha='0.05', test='nemenyi'):
"""
Compute critical difference for statistical significance testing.
Args:
avranks: Average ranks of methods
n: Number of datasets
alpha: Significance level
test: Statistical test ('nemenyi', 'bonferroni-dunn')
Returns:
float: Critical difference value
"""Usage Examples
# Basic evaluation workflow
from Orange.data import Table
from Orange.classification import TreeLearner, LogisticRegressionLearner
from Orange.evaluation import CrossValidation, CA, AUC, Precision, Recall, F1
# Load data
data = Table("iris")
# Create learners
learners = [
TreeLearner(max_depth=5),
LogisticRegressionLearner(C=1.0)
]
# Cross-validation
results = CrossValidation(data, learners, k=10, stratified=True)
# Calculate metrics
accuracies = CA(results)
auc_scores = AUC(results)
precisions = Precision(results)
recalls = Recall(results)
f1_scores = F1(results)
print("Classification Results:")
for i, learner in enumerate(learners):
print(f"{learner.__class__.__name__}:")
print(f" Accuracy: {accuracies[i]:.3f}")
print(f" AUC: {auc_scores[i]:.3f}")
print(f" Precision: {precisions[i]:.3f}")
print(f" Recall: {recalls[i]:.3f}")
print(f" F1: {f1_scores[i]:.3f}")
# Train-test split evaluation
from Orange.evaluation import TestOnTestData
train_data = data[:100]
test_data = data[100:]
test_results = TestOnTestData(train_data, test_data, learners)
test_accuracies = CA(test_results)
# Regression evaluation example
from Orange.data import Table
from Orange.regression import LinearRegressionLearner, TreeLearner
from Orange.evaluation import MSE, RMSE, MAE, R2
housing_data = Table("housing")
reg_learners = [
LinearRegressionLearner(),
TreeLearner()
]
reg_results = CrossValidation(housing_data, reg_learners, k=5)
mse_scores = MSE(reg_results)
rmse_scores = RMSE(reg_results)
mae_scores = MAE(reg_results)
r2_scores = R2(reg_results)
print("Regression Results:")
for i, learner in enumerate(reg_learners):
print(f"{learner.__class__.__name__}:")
print(f" MSE: {mse_scores[i]:.3f}")
print(f" RMSE: {rmse_scores[i]:.3f}")
print(f" MAE: {mae_scores[i]:.3f}")
print(f" R²: {r2_scores[i]:.3f}")Install with Tessl CLI
npx tessl i tessl/pypi-orange3