tessl/maven-spark-common-utils
Core utility classes and functions for Apache Spark including exception handling, logging, storage configuration, and Java API integration
—
Pending
exception-handling.mddocs/
Exception Handling and Error Management
Comprehensive exception handling system providing structured error reporting, error classes, and detailed context information for debugging and error recovery in Spark applications.
Capabilities
SparkException
Main exception class for Spark operations with support for error classes, message parameters, and query context.
/**
* Main Spark exception with structured error information
* @param message - Human-readable error message
* @param cause - Optional underlying cause
* @param errorClass - Optional error classification
* @param messageParameters - Key-value parameters for error message formatting
* @param context - Array of query context information
*/
class SparkException(
message: String,
cause: Throwable = null,
errorClass: Option[String] = None,
messageParameters: Map[String, String] = Map.empty,
context: Array[QueryContext] = Array.empty
) extends Exception(message, cause) with SparkThrowable {
/** Returns the error class identifier */
def getErrorClass(): String
/** Returns error message parameters as Java Map */
def getMessageParameters(): java.util.Map[String, String]
/** Returns query context information */
def getQueryContext(): Array[QueryContext]
}
object SparkException {
/** Creates an internal error exception */
def internalError(msg: String): SparkException
/** Creates a categorized internal error exception */
def internalError(msg: String, category: String): SparkException
/** Creates an internal error exception with cause */
def internalError(msg: String, cause: Throwable): SparkException
}Usage Examples:
import org.apache.spark.SparkException
// Basic exception creation
val ex = new SparkException("Operation failed")
// Exception with error class and parameters
val structuredEx = new SparkException(
"INVALID_PARAMETER_VALUE",
Map("parameter" -> "timeout", "value" -> "-1"),
null
)
// Creating internal errors
val internalEx = SparkException.internalError("Unexpected state in executor")
// Exception handling
try {
// Some Spark operation
} catch {
case ex: SparkException =>
println(s"Error class: ${ex.getErrorClass}")
println(s"Parameters: ${ex.getMessageParameters}")
ex.getQueryContext.foreach { ctx =>
println(s"Error in ${ctx.objectType}: ${ctx.objectName}")
}
}SparkThrowable
Interface for standardized error handling in Spark exceptions, providing consistent error classification and context.
/**
* Interface for Spark throwables with structured error information
*/
public interface SparkThrowable {
/** Returns the error class identifier */
String getErrorClass();
/** Returns SQL state for the error class */
default String getSqlState() {
return SparkThrowableHelper.getSqlState(this.getErrorClass());
}
/** Checks if this error is an internal error */
default boolean isInternalError() {
return SparkThrowableHelper.isInternalError(this.getErrorClass());
}
/** Returns error parameters for message formatting */
default Map<String, String> getMessageParameters() {
return new HashMap<>();
}
/** Returns query context information */
default QueryContext[] getQueryContext() {
return new QueryContext[0];
}
}Usage Examples:
import org.apache.spark.SparkThrowable;
import org.apache.spark.SparkException;
// Handling SparkThrowable in Java
try {
// Spark operation
} catch (SparkThrowable ex) {
System.out.println("Error class: " + ex.getErrorClass());
System.out.println("Is internal: " + ex.isInternalError());
Map<String, String> params = ex.getMessageParameters();
params.forEach((key, value) ->
System.out.println(key + ": " + value));
}QueryContext
Provides context information for SparkThrowable to help locate error sources in queries and operations.
/**
* Query context information for error reporting
*/
public interface QueryContext {
/** Returns the type of object where error occurred */
String objectType();
/** Returns the name of object where error occurred */
String objectName();
/** Returns start index in the query fragment */
int startIndex();
/** Returns stop index in the query fragment */
int stopIndex();
/** Returns the relevant query fragment */
String fragment();
}Usage Examples:
import org.apache.spark.QueryContext;
// Processing query context from exception
public void handleSparkException(SparkException ex) {
QueryContext[] contexts = ex.getQueryContext();
for (QueryContext ctx : contexts) {
System.out.println("Object type: " + ctx.objectType());
System.out.println("Object name: " + ctx.objectName());
System.out.println("Fragment: " + ctx.fragment());
System.out.println("Position: " + ctx.startIndex() + "-" + ctx.stopIndex());
}
}ErrorClassesJsonReader
Reader for loading error information from JSON configuration files, enabling structured error message formatting.
/**
* Reader for error class definitions from JSON files
* @param jsonFileURLs - Sequence of URLs pointing to error definition JSON files
*/
class ErrorClassesJsonReader(jsonFileURLs: Seq[URL]) {
/**
* Gets formatted error message for the given error class
* @param errorClass - Error class identifier
* @param messageParameters - Parameters for message formatting
* @return Formatted error message
*/
def getErrorMessage(errorClass: String, messageParameters: Map[String, String]): String
/**
* Gets raw message template for the given error class
* @param errorClass - Error class identifier
* @return Message template with parameter placeholders
*/
def getMessageTemplate(errorClass: String): String
/**
* Gets SQL state for the given error class
* @param errorClass - Error class identifier
* @return SQL state code or null if not defined
*/
def getSqlState(errorClass: String): String
}Usage Examples:
import org.apache.spark.ErrorClassesJsonReader
import java.net.URL
// Create reader with error definition files
val errorReader = new ErrorClassesJsonReader(Seq(
new URL("file:///path/to/error-classes.json")
))
// Get formatted error message
val message = errorReader.getErrorMessage(
"INVALID_PARAMETER_VALUE",
Map("parameter" -> "timeout", "value" -> "-1")
)
// Get message template
val template = errorReader.getMessageTemplate("INVALID_PARAMETER_VALUE")
// Returns: "Invalid value '<value>' for parameter '<parameter>'"
// Get SQL state
val sqlState = errorReader.getSqlState("INVALID_PARAMETER_VALUE")SparkArithmeticException
Arithmetic exception with Spark error class support for mathematical operation errors.
/**
* Arithmetic exception thrown from Spark with structured error information
* @param errorClass - Error class identifier
* @param messageParameters - Parameters for error message formatting
* @param context - Query context information
* @param summary - Error summary
*/
class SparkArithmeticException(
errorClass: String,
messageParameters: Map[String, String],
context: Array[QueryContext],
summary: String
) extends ArithmeticException with SparkThrowable {
def this(message: String) = // Constructor for simple messages
override def getErrorClass: String
override def getMessageParameters: java.util.Map[String, String]
override def getQueryContext: Array[QueryContext]
}SparkRuntimeException
Runtime exception with Spark error class support for general runtime errors.
/**
* Runtime exception thrown from Spark with structured error information
* @param errorClass - Error class identifier
* @param messageParameters - Parameters for error message formatting
* @param cause - Optional underlying cause
* @param context - Query context information
* @param summary - Error summary
*/
class SparkRuntimeException(
errorClass: String,
messageParameters: Map[String, String],
cause: Throwable = null,
context: Array[QueryContext] = Array.empty,
summary: String = ""
) extends RuntimeException with SparkThrowable {
def this(message: String, cause: Option[Throwable]) = // Constructor for simple messages
override def getErrorClass: String
override def getMessageParameters: java.util.Map[String, String]
override def getQueryContext: Array[QueryContext]
}SparkIllegalArgumentException
Illegal argument exception with Spark error class support for parameter validation errors.
/**
* Illegal argument exception thrown from Spark with structured error information
* @param errorClass - Error class identifier
* @param messageParameters - Parameters for error message formatting
* @param context - Query context information
* @param summary - Error summary
* @param cause - Optional underlying cause
*/
class SparkIllegalArgumentException(
errorClass: String,
messageParameters: Map[String, String],
context: Array[QueryContext] = Array.empty,
summary: String = "",
cause: Throwable = null
) extends IllegalArgumentException with SparkThrowable {
def this(message: String, cause: Option[Throwable]) = // Constructor for simple messages
override def getErrorClass: String
override def getMessageParameters: java.util.Map[String, String]
override def getQueryContext: Array[QueryContext]
}SparkDateTimeException
DateTime exception with Spark error class support for date/time operation errors.
/**
* DateTime exception thrown from Spark with structured error information
* @param errorClass - Error class identifier
* @param messageParameters - Parameters for error message formatting
* @param context - Query context information
* @param summary - Error summary
*/
class SparkDateTimeException(
errorClass: String,
messageParameters: Map[String, String],
context: Array[QueryContext],
summary: String
) extends DateTimeException with SparkThrowable {
def this(message: String) = // Constructor for simple messages
override def getErrorClass: String
override def getMessageParameters: java.util.Map[String, String]
override def getQueryContext: Array[QueryContext]
}SparkNumberFormatException
Number format exception with Spark error class support for numeric parsing errors.
/**
* Number format exception thrown from Spark with structured error information
* @param errorClass - Error class identifier
* @param messageParameters - Parameters for error message formatting
* @param context - Query context information
* @param summary - Error summary
*/
class SparkNumberFormatException(
errorClass: String,
messageParameters: Map[String, String],
context: Array[QueryContext],
summary: String
) extends NumberFormatException with SparkThrowable {
def this(message: String) = // Constructor for simple messages
override def getErrorClass: String
override def getMessageParameters: java.util.Map[String, String]
override def getQueryContext: Array[QueryContext]
}SparkUnsupportedOperationException
Unsupported operation exception with Spark error class support.
/**
* Unsupported operation exception thrown from Spark with structured error information
* @param errorClass - Error class identifier
* @param messageParameters - Parameters for error message formatting
*/
class SparkUnsupportedOperationException(
errorClass: String,
messageParameters: Map[String, String]
) extends UnsupportedOperationException with SparkThrowable {
def this(message: String) = // Constructor for simple messages
override def getErrorClass: String
override def getMessageParameters: java.util.Map[String, String]
}Error Handling Patterns
Basic Exception Handling
import org.apache.spark.SparkException
try {
// Spark operation that might fail
someSparkOperation()
} catch {
case ex: SparkException if ex.getErrorClass == "RESOURCE_NOT_FOUND" =>
// Handle specific error class
logWarning(s"Resource not found: ${ex.getMessageParameters}")
case ex: SparkException if ex.isInternalError =>
// Handle internal errors differently
logError("Internal Spark error occurred", ex)
throw ex
case ex: SparkException =>
// Handle other Spark exceptions
logError(s"Spark operation failed: ${ex.getMessage}")
}Exception Creation Patterns
import org.apache.spark.SparkException
// Create exception with error class
def validateParameter(name: String, value: Any): Unit = {
if (value == null) {
throw new SparkException(
"NULL_PARAMETER",
Map("parameter" -> name),
null
)
}
}
// Create internal error
def handleUnexpectedState(): Nothing = {
throw SparkException.internalError(
"Reached unexpected code path in partition processing"
)
}Type Definitions
// Exception with structured error information
class SparkException(
message: String,
cause: Throwable,
errorClass: Option[String],
messageParameters: Map[String, String],
context: Array[QueryContext]
) extends Exception with SparkThrowable
// Error classification interface
trait SparkThrowable {
def getErrorClass(): String
def getSqlState(): String
def isInternalError(): Boolean
def getMessageParameters(): java.util.Map[String, String]
def getQueryContext(): Array[QueryContext]
}
// Query context for error location
trait QueryContext {
def objectType(): String
def objectName(): String
def startIndex(): Int
def stopIndex(): Int
def fragment(): String
}Install with Tessl CLI
npx tessl i tessl/maven-spark-common-utils