tessl/pypi-ngt
High-performance library for approximate nearest neighbor search in high-dimensional vector spaces
—
Pending
optimization.mddocs/
Index Optimization
The Optimizer class provides tools for improving index performance through graph structure optimization and search parameter tuning. It reconstructs indexes with optimized graph connectivity and automatically adjusts search coefficients for better accuracy-speed trade-offs.
Capabilities
Optimizer Configuration
Create and configure optimization parameters for index reconstruction and performance tuning.
class Optimizer:
def __init__(self, num_of_outgoings=-1, num_of_incomings=-1, num_of_queries=-1,
num_of_objects=-1, low_accuracy_from=-1.0, low_accuracy_to=-1.0,
high_accuracy_from=-1.0, high_accuracy_to=-1.0, gt_epsilon=-1.0, margin=-1.0, log_disabled=False):
"""
Create optimizer with specified parameters.
Args:
num_of_outgoings (int): Number of outgoing edges per node (-1 for auto)
num_of_incomings (int): Number of incoming edges per node (-1 for auto)
num_of_queries (int): Number of queries for optimization (-1 for auto)
num_of_objects (int): Number of objects for optimization (-1 for auto)
low_accuracy_from (float): Starting low accuracy range (-1.0 for auto)
low_accuracy_to (float): Ending low accuracy range (-1.0 for auto)
high_accuracy_from (float): Starting high accuracy range (-1.0 for auto)
high_accuracy_to (float): Ending high accuracy range (-1.0 for auto)
gt_epsilon (float): Ground truth epsilon parameter (-1.0 for auto)
margin (float): Optimization margin parameter (-1.0 for auto)
log_disabled (bool): Disable progress logging (default: False)
"""Index Reconstruction
Reconstruct indexes with optimized graph structure for improved search performance.
class Optimizer:
def execute(self, in_path, out_path):
"""
Reconstruct index with optimization and adjust search coefficients.
Args:
in_path (str): Input index path
out_path (str): Output optimized index path
Returns:
None
"""Search Parameter Optimization
Optimize search coefficients and parameters for existing indexes.
class Optimizer:
def adjust_search_coefficients(self, path):
"""
Optimize search coefficients for existing index.
Args:
path (str): Index path to optimize
Returns:
None
"""
def optimize_search_parameters(self, path):
"""
Optimize search parameters for better performance.
Args:
path (str): Index path to optimize
Returns:
None
"""
def optimize_number_of_edges_for_anng(self, path, num_of_queries=-1, num_of_results=-1, num_of_threads=-1, target_accuracy=-1, target_num_of_objects=-1, num_of_sample_objects=-1, max_num_of_edges=-1):
"""
Optimize edge numbers for ANNG (Approximate Nearest Neighbor Graph).
Args:
path (str): Index path to optimize
num_of_queries (int): Number of queries for optimization (-1 for auto)
num_of_results (int): Number of results per query (-1 for auto)
num_of_threads (int): Number of threads for optimization (-1 for auto)
target_accuracy (float): Target accuracy level (-1 for auto)
target_num_of_objects (int): Target number of objects (-1 for auto)
num_of_sample_objects (int): Number of sample objects (-1 for auto)
max_num_of_edges (int): Maximum number of edges (-1 for auto)
Returns:
int: Optimized number of edges
"""Parameter Configuration
Configure optimization parameters and processing modes for fine-tuned control.
class Optimizer:
def set(self, num_of_outgoings=-1, num_of_incomings=-1, num_of_queries=-1,
num_of_objects=-1, low_accuracy_from=-1.0, low_accuracy_to=-1.0,
high_accuracy_from=-1.0, high_accuracy_to=-1.0, gt_epsilon=-1.0, margin=-1.0):
"""
Set optimization parameters.
Args:
num_of_outgoings (int): Number of outgoing edges per node (-1 for auto)
num_of_incomings (int): Number of incoming edges per node (-1 for auto)
num_of_queries (int): Number of queries for optimization (-1 for auto)
num_of_objects (int): Number of objects for optimization (-1 for auto)
low_accuracy_from (float): Starting low accuracy range (-1.0 for auto)
low_accuracy_to (float): Ending low accuracy range (-1.0 for auto)
high_accuracy_from (float): Starting high accuracy range (-1.0 for auto)
high_accuracy_to (float): Ending high accuracy range (-1.0 for auto)
gt_epsilon (float): Ground truth epsilon parameter (-1.0 for auto)
margin (float): Optimization margin parameter (-1.0 for auto)
Returns:
None
"""
def set_processing_modes(self, shortcut_reduction=True, search_parameter_optimization=True,
prefetch_parameter_optimization=True, accuracy_table_generation=True):
"""
Configure optimization processing modes.
Args:
shortcut_reduction (bool): Enable shortcut reduction (default: True)
search_parameter_optimization (bool): Enable search parameter optimization (default: True)
prefetch_parameter_optimization (bool): Enable prefetch optimization (default: True)
accuracy_table_generation (bool): Enable accuracy table generation (default: True)
Returns:
None
"""Usage Examples
Basic Index Optimization
import ngtpy
# Create optimizer with default parameters
optimizer = ngtpy.Optimizer(log_disabled=False)
# Reconstruct index with optimization
optimizer.execute("original_index", "optimized_index")
print("Index optimization completed")Custom Optimization Parameters
import ngtpy
# Create optimizer with custom parameters for high accuracy
optimizer = ngtpy.Optimizer(
num_of_outgoings=20, # More outgoing edges
num_of_incomings=150, # More incoming edges
num_of_queries=1000, # More queries for optimization
base_accuracy_from=0.8, # Start from 80% accuracy
base_accuracy_to=0.95, # Target 95% accuracy
log_disabled=False
)
# Set processing modes
optimizer.set_processing_modes(
shortcut_reduction=True,
search_parameter_optimization=True,
prefetch_parameter_optimization=True,
accuracy_table_generation=True
)
# Execute optimization
optimizer.execute("large_index", "high_accuracy_index")
print("High-accuracy optimization completed")Search Parameter Tuning
import ngtpy
# Create optimizer for search parameter tuning only
optimizer = ngtpy.Optimizer(
num_of_queries=500,
num_of_results=10,
log_disabled=False
)
# Optimize search coefficients for existing index
optimizer.adjust_search_coefficients("existing_index")
# Optimize search parameters
optimizer.optimize_search_parameters("existing_index")
print("Search parameters optimized")ANNG Edge Optimization
import ngtpy
# Create optimizer for ANNG edge optimization
optimizer = ngtpy.Optimizer(log_disabled=False)
# Optimize number of edges with accuracy table
optimizer.optimize_number_of_edges_for_anng(
"anng_index",
num_of_queries=2000,
accuracy_table_path="accuracy_results.tsv"
)
print("ANNG edge optimization completed")Progressive Optimization
import ngtpy
# Stage 1: Basic optimization
basic_optimizer = ngtpy.Optimizer(
num_of_outgoings=15,
num_of_incomings=100,
log_disabled=False
)
basic_optimizer.execute("raw_index", "basic_optimized")
# Stage 2: Search parameter optimization
search_optimizer = ngtpy.Optimizer(
num_of_queries=1500,
num_of_results=20,
base_accuracy_from=0.85,
base_accuracy_to=0.95
)
search_optimizer.adjust_search_coefficients("basic_optimized")
# Stage 3: ANNG fine-tuning
search_optimizer.optimize_number_of_edges_for_anng(
"basic_optimized",
num_of_queries=1000
)
print("Progressive optimization completed")Memory-Optimized Configuration
import ngtpy
# Configure optimizer for memory efficiency
memory_optimizer = ngtpy.Optimizer(
num_of_outgoings=8, # Fewer outgoing edges
num_of_incomings=60, # Fewer incoming edges
ratio=0.02, # Lower ratio for compression
margin=0.2, # Tighter margin
log_disabled=False
)
# Disable some processing modes to save memory
memory_optimizer.set_processing_modes(
shortcut_reduction=True,
search_parameter_optimization=False, # Skip to save memory
prefetch_parameter_optimization=False, # Skip to save memory
accuracy_table_generation=True
)
memory_optimizer.execute("large_index", "memory_optimized_index")
print("Memory-optimized index created")Install with Tessl CLI
npx tessl i tessl/pypi-ngt