or run

npx @tessl/cli init

Version

Tile

Overview

Evals

Files

docs

build-info.md exception-handling.md index.md java-functions.md logging.md storage-management.md utilities.md

tile.json

Build Information

Build metadata and version information system providing runtime introspection capabilities for Apache Spark deployments, including version details, build timestamps, and repository information.

Capabilities

SparkBuildInfo Object

Central repository for Spark build and version metadata loaded from embedded properties files.

/**
 * Provides Spark build and version information
 * Loads metadata from spark-version-info.properties at runtime
 */
private[spark] object SparkBuildInfo {
  
  /** Spark version string (e.g., "3.5.6") */
  val spark_version: String
  
  /** Git branch used for the build (e.g., "branch-3.5") */
  val spark_branch: String
  
  /** Git revision/commit hash (e.g., "abc123def456") */
  val spark_revision: String
  
  /** User who performed the build */
  val spark_build_user: String
  
  /** Repository URL where the source code is hosted */
  val spark_repo_url: String
  
  /** Build timestamp (e.g., "2024-02-15T10:30:00Z") */
  val spark_build_date: String
  
  /** Documentation root URL */
  val spark_doc_root: String
}

Usage Examples:

import org.apache.spark.SparkBuildInfo

// Access version information
val version = SparkBuildInfo.spark_version
println(s"Running Spark version: $version")

// Check build details
val buildInfo = s"""
  |Spark Build Information:
  |  Version: ${SparkBuildInfo.spark_version}
  |  Branch: ${SparkBuildInfo.spark_branch}  
  |  Revision: ${SparkBuildInfo.spark_revision}
  |  Build Date: ${SparkBuildInfo.spark_build_date}
  |  Build User: ${SparkBuildInfo.spark_build_user}
  |  Repository: ${SparkBuildInfo.spark_repo_url}
  |""".stripMargin

println(buildInfo)

// Version compatibility checking
def checkSparkVersion(): Unit = {
  val currentVersion = SparkBuildInfo.spark_version
  val requiredMinVersion = "3.5.0"
  
  if (compareVersions(currentVersion, requiredMinVersion) >= 0) {
    println(s"Spark version $currentVersion meets requirements")
  } else {
    throw new IllegalStateException(
      s"Spark version $currentVersion is below required minimum $requiredMinVersion"
    )
  }
}

// Development vs production build detection
def isDevBuild(): Boolean = {
  SparkBuildInfo.spark_build_user match {
    case "jenkins" | "buildbot" => false  // Production builds
    case _ => true  // Developer builds
  }
}

Build Information Access Patterns

Common patterns for accessing and using build metadata in Spark applications.

// Runtime version verification
object SparkVersionChecker {
  def verifyCompatibility(minVersion: String): Unit = {
    val current = SparkBuildInfo.spark_version
    // Version comparison logic
  }
  
  def getBuildSummary(): Map[String, String] = {
    Map(
      "version" -> SparkBuildInfo.spark_version,
      "branch" -> SparkBuildInfo.spark_branch,
      "revision" -> SparkBuildInfo.spark_revision,
      "buildDate" -> SparkBuildInfo.spark_build_date
    )
  }
}

// Application metadata integration
class SparkApplicationInfo {
  def getEnvironmentInfo(): String = {
    s"""
    |Application Environment:
    |  Spark Version: ${SparkBuildInfo.spark_version}
    |  Git Branch: ${SparkBuildInfo.spark_branch}
    |  Build Date: ${SparkBuildInfo.spark_build_date}
    |  JVM Version: ${System.getProperty("java.version")}
    |  Scala Version: ${scala.util.Properties.versionString}
    |""".stripMargin
  }
}

Practical Usage Examples:

import org.apache.spark.SparkBuildInfo

// Logging build information at application startup
class SparkApplication extends Logging {
  def logBuildInfo(): Unit = {
    logInfo(s"Starting Spark application")
    logInfo(s"Spark version: ${SparkBuildInfo.spark_version}")
    logInfo(s"Build from branch: ${SparkBuildInfo.spark_branch}")
    logInfo(s"Build date: ${SparkBuildInfo.spark_build_date}")
    
    if (SparkBuildInfo.spark_build_user != "jenkins") {
      logWarning("Running development build - not recommended for production")
    }
  }
}

// Feature availability checking based on version
object FeatureCompatibility {
  def supportsStructuredStreaming(): Boolean = {
    // Structured Streaming introduced in Spark 2.0
    val version = SparkBuildInfo.spark_version
    val majorVersion = version.split("\\.").head.toInt
    majorVersion >= 2
  }
  
  def supportsAdaptiveQueryExecution(): Boolean = {
    // AQE significantly improved in Spark 3.0
    val version = SparkBuildInfo.spark_version
    val majorVersion = version.split("\\.").head.toInt
    majorVersion >= 3
  }
}

// Build reproducibility verification
class BuildVerifier {
  def verifyBuildIntegrity(): Unit = {
    val expectedRepo = "https://github.com/apache/spark"
    
    if (!SparkBuildInfo.spark_repo_url.startsWith(expectedRepo)) {
      logWarning(s"Unexpected repository URL: ${SparkBuildInfo.spark_repo_url}")
    }
    
    // Verify revision format (Git SHA)
    val revisionPattern = """^[a-f0-9]{7,40}$""".r
    if (!revisionPattern.matches(SparkBuildInfo.spark_revision)) {
      logWarning(s"Invalid revision format: ${SparkBuildInfo.spark_revision}")
    }
  }
}

// Documentation URL generation
object DocumentationHelper {
  def getVersionSpecificDocsUrl(topic: String): String = {
    val baseUrl = SparkBuildInfo.spark_doc_root
    val version = SparkBuildInfo.spark_version
    s"$baseUrl/$version/$topic"
  }
  
  def getApiDocsUrl(): String = {
    getVersionSpecificDocsUrl("api/scala/index.html")
  }
}

// Release information extraction
case class ReleaseInfo(
  version: String,
  branch: String,
  revision: String,
  buildDate: String,
  isSnapshot: Boolean,
  isRelease: Boolean
)

object ReleaseInfoExtractor {
  def extractReleaseInfo(): ReleaseInfo = {
    val version = SparkBuildInfo.spark_version
    val isSnapshot = version.contains("SNAPSHOT")
    val isRelease = !isSnapshot && version.matches("""\d+\.\d+\.\d+""")
    
    ReleaseInfo(
      version = version,
      branch = SparkBuildInfo.spark_branch,
      revision = SparkBuildInfo.spark_revision,
      buildDate = SparkBuildInfo.spark_build_date,
      isSnapshot = isSnapshot,
      isRelease = isRelease
    )
  }
}

Build Information Properties

The build information is loaded from a properties file embedded in the JAR:

Properties File Structure

The spark-version-info.properties file contains:

version=3.5.6
branch=branch-3.5  
revision=abc123def456789
user=jenkins
url=https://github.com/apache/spark
date=2024-02-15T10:30:00Z
docroot=https://spark.apache.org/docs

Loading Mechanism

// Internal loading logic (simplified)
private def loadBuildInfo(): (String, String, String, String, String, String, String) = {
  val resourceStream = Thread.currentThread().getContextClassLoader
    .getResourceAsStream("spark-version-info.properties")
    
  if (resourceStream == null) {
    throw new SparkException("Could not find spark-version-info.properties")
  }
  
  try {
    val unknownProp = "<unknown>"
    val props = new Properties()
    props.load(resourceStream)
    (
      props.getProperty("version", unknownProp),
      props.getProperty("branch", unknownProp), 
      props.getProperty("revision", unknownProp),
      props.getProperty("user", unknownProp),
      props.getProperty("url", unknownProp),
      props.getProperty("date", unknownProp),
      props.getProperty("docroot", unknownProp)
    )
  } catch {
    case e: Exception =>
      throw new SparkException("Error loading properties from spark-version-info.properties", e)
  } finally {
    if (resourceStream != null) {
      resourceStream.close()
    }
  }
}

Integration Patterns

Application Startup Logging

class SparkAppStarter extends Logging {
  def start(): Unit = {
    logBuildInformation()
    // Application initialization
  }
  
  private def logBuildInformation(): Unit = {
    logInfo("="*50)
    logInfo("Spark Application Starting")
    logInfo("="*50)
    logInfo(s"Spark Version: ${SparkBuildInfo.spark_version}")
    logInfo(s"Git Branch: ${SparkBuildInfo.spark_branch}")
    logInfo(s"Git Revision: ${SparkBuildInfo.spark_revision}")
    logInfo(s"Build Date: ${SparkBuildInfo.spark_build_date}")
    logInfo(s"Build User: ${SparkBuildInfo.spark_build_user}")
    logInfo("="*50)
  }
}

Compatibility Matrices

object SparkCompatibility {
  val SUPPORTED_VERSIONS = Seq("3.5.0", "3.5.1", "3.5.2", "3.5.3", "3.5.4", "3.5.5", "3.5.6")
  
  def checkCompatibility(): Unit = {
    val currentVersion = SparkBuildInfo.spark_version
    
    if (!SUPPORTED_VERSIONS.contains(currentVersion)) {
      logWarning(s"Untested Spark version: $currentVersion")
      logWarning(s"Supported versions: ${SUPPORTED_VERSIONS.mkString(", ")}")
    }
  }
}

Diagnostic Information

class DiagnosticCollector {
  def collectEnvironmentInfo(): Map[String, Any] = {
    Map(
      "spark" -> Map(
        "version" -> SparkBuildInfo.spark_version,
        "branch" -> SparkBuildInfo.spark_branch,
        "revision" -> SparkBuildInfo.spark_revision,
        "buildDate" -> SparkBuildInfo.spark_build_date,
        "buildUser" -> SparkBuildInfo.spark_build_user,
        "repoUrl" -> SparkBuildInfo.spark_repo_url
      ),
      "jvm" -> Map(
        "version" -> System.getProperty("java.version"),
        "vendor" -> System.getProperty("java.vendor")
      ),
      "scala" -> Map(
        "version" -> scala.util.Properties.versionString
      )
    )
  }
}