Data Utilities and Helpers

# Real-world datasets
def adult(display=False, n_points=None) -> tuple[pd.DataFrame, np.ndarray]:
    """
    Census income prediction dataset (>50K income classification).
    
    Parameters:
    - display: Return human-readable labels instead of encoded values (bool)
    - n_points: Sample n data points (int, optional)
    
    Returns:
    (features, targets) - DataFrame with 14 features, binary target array
    """

def california(n_points=None) -> tuple[pd.DataFrame, np.ndarray]:
    """
    California housing regression dataset.
    
    Median house values for California districts with geographic and
    demographic features.
    
    Returns:
    (features, targets) - DataFrame with 8 features, continuous target array
    """

def imagenet50(resolution=224, n_points=None) -> tuple[np.ndarray, np.ndarray]:
    """
    50 representative ImageNet images for background distributions.
    
    Parameters:
    - resolution: Image resolution (currently only 224 supported)
    - n_points: Sample n images (optional)
    
    Returns:
    (images, labels) - Image array (N, H, W, C), label array
    """

def imdb(n_points=None) -> tuple[list[str], np.ndarray]:
    """
    Movie review sentiment classification dataset.
    
    Returns:
    (reviews, sentiments) - List of review text strings, binary sentiment array
    """

def diabetes(n_points=None) -> tuple[pd.DataFrame, np.ndarray]:
    """
    Diabetes progression prediction dataset.
    
    Physiological measurements predicting diabetes progression after one year.
    
    Returns:
    (features, targets) - DataFrame with 10 features, continuous target array
    """

def iris(display=False, n_points=None) -> tuple[pd.DataFrame, np.ndarray]:
    """
    Classic iris flower classification dataset.
    
    Parameters:
    - display: Return species names instead of encoded labels (bool)
    
    Returns:
    (features, targets) - DataFrame with 4 features, class labels
    """

def linnerud(n_points=None) -> tuple[pd.DataFrame, pd.DataFrame]:
    """
    Multi-target physiological/exercise dataset.
    
    Exercise measurements predicting physiological parameters.
    
    Returns:
    (exercise_features, physiological_targets) - Both as DataFrames
    """

def nhanesi(display=False, n_points=None) -> tuple[pd.DataFrame, np.ndarray]:
    """
    NHANES I survival analysis dataset.
    
    National Health and Nutrition Examination Survey data with survival times
    as labels, used for survival analysis and mortality prediction tasks.
    
    Parameters:
    - display: Return features with modified display format (bool)
    - n_points: Number of data points to sample (int, optional)
    
    Returns:
    (features, survival_times) - DataFrame with health measurements, survival time array
    """

def communitiesandcrime(n_points=None) -> tuple[pd.DataFrame, np.ndarray]:
    """
    Communities and Crime regression dataset from UCI ML Repository.
    
    Community demographic and social features for predicting total number
    of violent crimes per 100K population.
    
    Parameters:
    - n_points: Number of data points to sample (int, optional)
    
    Returns:
    (features, crime_rates) - DataFrame with community features, crime rate targets
    """

# Sparse and ranking datasets  
def a1a(n_points=None) -> tuple[scipy.sparse.csr_matrix, np.ndarray]:
    """
    Sparse binary classification dataset in SVM light format.
    
    High-dimensional sparse feature matrix for binary classification,
    commonly used for testing sparse algorithms.
    
    Parameters:
    - n_points: Number of data points to sample (int, optional)
    
    Returns:
    (sparse_features, binary_targets) - CSR sparse matrix and binary labels
    """

def rank() -> tuple[scipy.sparse.csr_matrix, np.ndarray, scipy.sparse.csr_matrix, 
                    np.ndarray, np.ndarray, np.ndarray]:
    """
    Learning-to-rank datasets from LightGBM repository.
    
    Ranking datasets with query-document pairs and relevance judgments,
    used for learning-to-rank model evaluation.
    
    Returns:
    (train_X, train_y, test_X, test_y, train_queries, test_queries) - 
    Training/test sparse matrices, relevance labels, and query group IDs
    """

# Synthetic datasets
def corrgroups60(n_points=1000) -> tuple[pd.DataFrame, np.ndarray]:
    """
    Synthetic dataset with 60 features organized in correlated groups.
    
    Generated dataset with known correlation structure between distinct
    feature groups, useful for testing correlation-aware algorithms.
    
    Parameters:
    - n_points: Number of data points to generate (int, default: 1000)
    
    Returns:
    (features, targets) - DataFrame with correlated features, linear targets
    """

def independentlinear60(n_points=1000) -> tuple[pd.DataFrame, np.ndarray]:
    """
    Synthetic dataset with 60 independent linear features.
    
    Generated dataset with independent Gaussian features and linear
    target relationships, used for benchmarking linear methods.
    
    Parameters:
    - n_points: Number of data points to generate (int, default: 1000)
    
    Returns:
    (features, targets) - DataFrame with independent features, linear targets
    """

import shap

# Load real-world dataset
X, y = shap.datasets.adult()
print(f"Adult dataset: {X.shape[0]} samples, {X.shape[1]} features")

# Load image dataset for computer vision
images, labels = shap.datasets.imagenet50(n_points=10)
print(f"ImageNet sample: {images.shape}")

# Load text dataset for NLP
reviews, sentiments = shap.datasets.imdb(n_points=100)
print(f"IMDB sample: {len(reviews)} reviews")

class Masker:
    """Abstract base class for all maskers."""
    def __call__(self, mask, *args):
        """Apply masking with binary mask array."""
    
    @property
    def shape(self):
        """Expected input dimensions."""
    
    @property  
    def supports_delta_masking(self):
        """Whether masker supports efficient delta masking."""

# Tabular data maskers
class Independent:
    """
    Independent feature masking with background data integration.
    
    Replaces masked features with values sampled independently
    from background distribution.
    """
    def __init__(self, data, max_samples=100):
        """
        Parameters:
        - data: Background dataset for sampling replacement values
        - max_samples: Maximum background samples to use
        """

class Partition:
    """
    Hierarchical feature masking respecting feature correlations.
    
    Groups correlated features and masks them together to maintain
    realistic feature relationships.
    """
    def __init__(self, data, max_samples=100, clustering="correlation"):
        """
        Parameters:
        - data: Background dataset for correlation analysis
        - clustering: Clustering method ("correlation", "tree", custom)
        """

class Impute:
    """
    Missing value imputation for masking.
    
    Uses feature correlations to impute realistic values for
    masked features instead of random sampling.
    """
    def __init__(self, data, method="linear"):
        """
        Parameters:
        - data: Training data for imputation model
        - method: Imputation method ("linear", "tree", "knn")
        """

# Specialized maskers  
class Text:
    """
    Text tokenization and masking for NLP models.
    
    Handles tokenization, special tokens, and text-specific
    masking strategies for language models.
    """
    def __init__(self, tokenizer=None, mask_token=None, 
                 collapse_mask_token="auto", output_type="string"):
        """
        Parameters:
        - tokenizer: Custom tokenizer (optional, uses default splitting)
        - mask_token: Token to use for masking (e.g., "[MASK]")
        - collapse_mask_token: How to handle consecutive masked tokens
        - output_type: Output format ("string", "token_ids", "tokens")
        """

class Image:
    """
    Image region masking with realistic perturbations.
    
    Supports various masking strategies including blur, inpainting,
    and noise for computer vision models.
    """
    def __init__(self, mask_value, shape=None):
        """
        Parameters:
        - mask_value: Value/strategy for masked regions (scalar, "blur", "inpaint", "noise")
        - shape: Expected image shape (optional, inferred from data)
        """

class Fixed:
    """
    Fixed background values for masking.
    
    Simple masking strategy using predetermined values
    for all masked features.
    """
    def __init__(self, mask_value):
        """
        Parameters:
        - mask_value: Fixed value(s) to use for masking
        """

# Composite maskers
class Composite:
    """
    Combine multiple maskers for different feature groups.
    
    Allows different masking strategies for different parts
    of the input (e.g., tabular + text + image).
    """
    def __init__(self, **maskers):
        """
        Parameters:
        - **maskers: Named maskers for different feature groups
        """

class FixedComposite:
    """Fixed composite masking with predetermined feature groups."""
    def __init__(self, **maskers):
        """Initialize with fixed feature-to-masker mapping."""

class OutputComposite:
    """Output-specific masking for multi-output models."""
    def __init__(self, **maskers):
        """Initialize with output-specific masking strategies."""

class Model:
    """
    Universal model wrapper with automatic tensor conversion.
    
    Standardizes model interfaces and handles tensor conversions
    between NumPy arrays and framework-specific tensors.
    """
    def __init__(self, model=None):
        """
        Parameters:
        - model: Model object to wrap (optional, can be set later)
        """
    
    def __call__(self, *args):
        """
        Call wrapped model with automatic tensor conversion.
        
        Converts NumPy inputs to appropriate framework tensors,
        calls model, and converts outputs back to NumPy arrays.
        """
    
    def save(self, out_file):
        """Serialize model to file."""
    
    @staticmethod
    def load(in_file, instantiate=True):
        """Load model from file."""

class TeacherForcing:
    """
    Model wrapper for teacher forcing in sequence models.
    
    Handles sequence generation with known target sequences
    during training/explanation phases.
    """
    def __init__(self, model, similarity_model=None, masker=None):
        """Initialize teacher forcing wrapper for sequence models."""

class TextGeneration:
    """
    Wrapper for text generation models.
    
    Standardizes interface for autoregressive text models
    with generation parameters and stopping criteria.
    """
    def __init__(self, model, masker=None, similarity_model=None):
        """Initialize text generation model wrapper."""

class TopKLM:
    """
    Top-K language model wrapper.
    
    Restricts language model outputs to top-K most likely tokens
    for more stable explanations.
    """
    def __init__(self, model, similarity_model=None, masker=None):
        """Initialize top-K language model wrapper."""

class TransformersPipeline:
    """
    HuggingFace transformers pipeline wrapper.
    
    Integrates with HuggingFace pipelines for standardized
    transformer model interfaces.
    """
    def __init__(self, pipeline):
        """
        Parameters:
        - pipeline: HuggingFace pipeline object
        """

# Sampling and data manipulation
def sample(X, nsamples=100, random_state=0):
    """
    Sample data points without replacement.
    
    Parameters:
    - X: Input data (array, DataFrame, sparse matrix)
    - nsamples: Number of samples to draw
    - random_state: Random seed for reproducibility
    
    Returns:
    Sampled data in same format as input
    """

def approximate_interactions(index, shap_values, X, feature_names=None) -> np.ndarray:
    """
    Find features with high interactions with target feature.
    
    Parameters:
    - index: Target feature index or name
    - shap_values: SHAP values array or Explanation object
    - X: Input feature data
    - feature_names: List of feature names (optional)
    
    Returns:
    Array of interaction strength scores for each feature
    """

# Clustering functions
def hclust(data, metric="sqeuclidean"):
    """
    Hierarchical clustering of features.
    
    Parameters:
    - data: Feature data for clustering
    - metric: Distance metric for clustering
    
    Returns:
    Clustering linkage matrix
    """

def hclust_ordering(X, metric="sqeuclidean"):
    """
    Optimal leaf ordering for hierarchical clustering dendrograms.
    
    Minimizes distances between adjacent leaves in dendrogram.
    """

def delta_minimization_order():
    """Compute ordering that minimizes partition tree delta."""

def partition_tree():  
    """Create hierarchical partition tree for feature grouping."""

def partition_tree_shuffle():
    """Shuffle partition tree leaves while preserving structure."""

# Mathematical utilities
def shapley_coefficients(n) -> np.ndarray:
    """
    Compute Shapley coefficients for n players.
    
    Parameters:
    - n: Number of features/players
    
    Returns:
    Array of Shapley coefficients
    """

# Utility classes
class OpChain:
    """
    Chainable operations for delayed execution.
    
    Enables method chaining on Explanation objects with
    lazy evaluation for performance optimization.
    """
    def __init__(self, op, *args, **kwargs):
        """Initialize operation chain."""
    
    def __call__(self, obj):
        """Apply operation chain to object."""

class MaskedModel:
    """
    Wrapper for masked model evaluation.
    
    Handles feature masking during model evaluation with
    efficient batching and caching.
    """
    def __init__(self, model, masker, *args, **kwargs):
        """
        Parameters:
        - model: Model function to wrap
        - masker: Masker object for feature perturbation
        """
    
    def __call__(self, masks, *args, **kwargs):
        """Evaluate model with masked inputs."""

def make_masks():
    """Generate binary masks for features."""

# Display and progress utilities
def show_progress():
    """Display progress bars for long computations."""

# Import and error handling
def assert_import(package_name):
    """Assert that required package is available."""

def record_import_error(package_name, msg, e):
    """Record import errors for debugging."""

def safe_isinstance(obj, class_path_str) -> bool:
    """Safe type checking without importing classes."""

# String formatting utilities  
def format_value(s, format_str):
    """Format values for display in plots and outputs."""

def ordinal_str(n):
    """Convert numbers to ordinal strings (1st, 2nd, 3rd, etc.)."""

def convert_name():
    """Convert feature names between different formats."""

def potential_interactions(shap_values_column, shap_values_matrix):
    """
    Order features by interaction strength with target feature.
    
    Bins SHAP values for a feature along that feature's value to identify
    potential interactions. For exact Shapley interaction values, use
    interaction_contribs in XGBoost.
    
    Parameters:
    - shap_values_column: SHAP values for target feature
    - shap_values_matrix: SHAP values matrix for all features
    
    Returns:
    Feature ordering by interaction strength
    """

def make_masks(cluster_matrix):
    """
    Build sparse CSR mask matrix from hierarchical clustering.
    
    Optimized function for creating binary masks from clustering results,
    particularly useful for large image datasets and tree structures.
    
    Parameters:
    - cluster_matrix: Hierarchical clustering matrix
    
    Returns:
    scipy.sparse.csr_matrix: Binary mask matrix for feature groups
    """

def suppress_stderr():
    """Context manager to suppress stderr output during operations."""

class Action:
    """
    Abstract action class with cost parameter.
    
    Base class for defining actions in optimization problems
    with associated costs and execution logic.
    """
    def __init__(self, cost):
        """
        Parameters:
        - cost: Cost of executing this action (numeric)
        """
    
    def __call__(self, *args):
        """Execute the action - must be implemented by subclasses."""
    
    def __lt__(self, other_action):
        """Compare actions by cost for priority queue ordering."""

class ActionOptimizer:
    """
    Optimize action sequences to satisfy model constraints.
    
    Uses priority queue search to find minimum-cost action sequences
    that satisfy specified model constraints.
    
    Warning:
        ActionOptimizer is in alpha state and subject to API changes.
    """
    def __init__(self, model, actions):
        """
        Parameters:
        - model: Function returning True when constraints are satisfied
        - actions: List of Action objects or lists of mutually exclusive actions
        """
    
    def __call__(self, *args, max_evals=10000):
        """
        Find optimal action sequence.
        
        Parameters:
        - max_evals: Maximum evaluations before raising ConvergenceError
        
        Returns:
        List of actions that satisfy constraints with minimum cost
        """

def identity(x):
    """
    Identity link function (no transformation).
    
    Returns input unchanged. Used for regression models
    and when no output transformation is needed.
    
    Parameters:
    - x: Input values
    
    Returns:
    Unchanged input values
    """

identity.inverse = lambda x: x  # Inverse transformation

def logit(x):
    """
    Logit link function for probability to log-odds conversion.
    
    Transforms probabilities [0,1] to log-odds (-∞,∞).
    Useful for binary classification models.
    
    Parameters:
    - x: Probability values in [0,1]
    
    Returns:
    Log-odds values log(x/(1-x))
    """

logit.inverse = lambda x: 1 / (1 + np.exp(-x))  # Sigmoid inverse

import shap

# Load dataset with optional sampling
X, y = shap.datasets.adult(n_points=1000)

# Use for model training
from sklearn.ensemble import RandomForestClassifier
model = RandomForestClassifier()
model.fit(X, y)

# Background data for explanations
X_background = shap.utils.sample(X, 100)

# Tabular data with correlations
masker = shap.maskers.Partition(X_background, clustering="correlation")

# Text data
masker = shap.maskers.Text(mask_token="[MASK]", output_type="string")

# Image data
masker = shap.maskers.Image(mask_value="blur")

# Composite data (tabular + text)
masker = shap.maskers.Composite(
    tabular=shap.maskers.Independent(X_tabular),
    text=shap.maskers.Text()
)

# Wrap PyTorch model for consistent interface
wrapped_model = shap.models.Model(pytorch_model)

# Use with any explainer
explainer = shap.KernelExplainer(wrapped_model, X_background)
shap_values = explainer(X_test)