or run

npx @tessl/cli init
Log in

Version

Tile

Overview

Evals

Files

docs

admin.mdadvanced.mdclient-server.mdcredentials-security.mderrors-status.mdhealth.mdindex.mdinterceptors.mdload-balancing.mdmetadata-context.mdname-resolution.mdobservability.mdreflection.mdstreaming.mdtesting.mdxds.md
tile.json

health.mddocs/

Health Checking

This document covers health checking in gRPC-Go, including the health service protocol, server-side implementation, and client-side health checking.

Overview

The health package provides a service that exposes server health status. It implements the standard gRPC health checking protocol defined in gRPC Health Checking Protocol.

import "google.golang.org/grpc/health"
import healthpb "google.golang.org/grpc/health/grpc_health_v1"

Health Service Protocol

Serving Status

type HealthCheckResponse_ServingStatus int32

const (
    // UNKNOWN indicates health status is unknown
    HealthCheckResponse_UNKNOWN HealthCheckResponse_ServingStatus = 0

    // SERVING indicates service is healthy and serving
    HealthCheckResponse_SERVING HealthCheckResponse_ServingStatus = 1

    // NOT_SERVING indicates service is not serving (unhealthy)
    HealthCheckResponse_NOT_SERVING HealthCheckResponse_ServingStatus = 2

    // SERVICE_UNKNOWN indicates requested service is unknown (used only by Watch)
    HealthCheckResponse_SERVICE_UNKNOWN HealthCheckResponse_ServingStatus = 3
)

Health Check Request/Response

type HealthCheckRequest struct {
    // Service name to check
    // Empty string checks overall server health
    Service string
}

type HealthCheckResponse struct {
    Status HealthCheckResponse_ServingStatus
}

Server-Side Health Checking

Health Server

type Server struct {
    // Has unexported fields
}

// NewServer returns a new health server
func NewServer() *Server

// SetServingStatus sets the serving status of a service
func (s *Server) SetServingStatus(service string, servingStatus healthpb.HealthCheckResponse_ServingStatus)

// Resume sets all serving status to SERVING and accepts future status changes
func (s *Server) Resume()

// Shutdown sets all serving status to NOT_SERVING and ignores future status changes
func (s *Server) Shutdown()

// Check implements Health.Check RPC
func (s *Server) Check(ctx context.Context, in *healthpb.HealthCheckRequest) (*healthpb.HealthCheckResponse, error)

// Watch implements Health.Watch RPC (server streaming)
func (s *Server) Watch(in *healthpb.HealthCheckRequest, stream healthgrpc.Health_WatchServer) error

// List implements Health.List RPC
func (s *Server) List(ctx context.Context, in *healthpb.HealthListRequest) (*healthpb.HealthListResponse, error)

Basic Server Setup

import (
    "google.golang.org/grpc"
    "google.golang.org/grpc/health"
    healthpb "google.golang.org/grpc/health/grpc_health_v1"
)

// Create server with health checking
server := grpc.NewServer()

// Create and register health server
healthServer := health.NewServer()
healthpb.RegisterHealthServer(server, healthServer)

// Register your services
pb.RegisterMyServiceServer(server, &myServiceImpl{})

// Set service health status
healthServer.SetServingStatus("mypackage.MyService", healthpb.HealthCheckResponse_SERVING)

// Set overall server health
healthServer.SetServingStatus("", healthpb.HealthCheckResponse_SERVING)

// Start serving
lis, _ := net.Listen("tcp", ":50051")
server.Serve(lis)

Setting Service Status

import (
    "google.golang.org/grpc/health"
    healthpb "google.golang.org/grpc/health/grpc_health_v1"
)

// Set specific service as serving
healthServer.SetServingStatus("mypackage.MyService", healthpb.HealthCheckResponse_SERVING)

// Set specific service as not serving
healthServer.SetServingStatus("mypackage.MyService", healthpb.HealthCheckResponse_NOT_SERVING)

// Set overall server health
healthServer.SetServingStatus("", healthpb.HealthCheckResponse_SERVING)

Graceful Shutdown

import (
    "os"
    "os/signal"
    "syscall"
    "google.golang.org/grpc"
    "google.golang.org/grpc/health"
    healthpb "google.golang.org/grpc/health/grpc_health_v1"
)

server := grpc.NewServer()
healthServer := health.NewServer()
healthpb.RegisterHealthServer(server, healthServer)

// Set initial status
healthServer.SetServingStatus("", healthpb.HealthCheckResponse_SERVING)

// Handle signals
sigChan := make(chan os.Signal, 1)
signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM)

go func() {
    <-sigChan
    // Set health status to NOT_SERVING before shutdown
    healthServer.Shutdown()
    // Give clients time to detect unhealthy status
    time.Sleep(5 * time.Second)
    // Graceful stop
    server.GracefulStop()
}()

lis, _ := net.Listen("tcp", ":50051")
server.Serve(lis)

Resume After Maintenance

// During maintenance, shutdown health server
healthServer.Shutdown()

// Perform maintenance...

// Resume after maintenance
healthServer.Resume()

Dynamic Health Status

import (
    "context"
    "time"
    "google.golang.org/grpc/health"
    healthpb "google.golang.org/grpc/health/grpc_health_v1"
)

// Monitor dependency health and update status
func monitorHealth(ctx context.Context, healthServer *health.Server) {
    ticker := time.NewTicker(10 * time.Second)
    defer ticker.Stop()

    for {
        select {
        case <-ctx.Done():
            return
        case <-ticker.C:
            // Check database connection
            if !checkDatabase() {
                healthServer.SetServingStatus("myservice",
                    healthpb.HealthCheckResponse_NOT_SERVING)
                continue
            }

            // Check external API
            if !checkExternalAPI() {
                healthServer.SetServingStatus("myservice",
                    healthpb.HealthCheckResponse_NOT_SERVING)
                continue
            }

            // All dependencies healthy
            healthServer.SetServingStatus("myservice",
                healthpb.HealthCheckResponse_SERVING)
        }
    }
}

func checkDatabase() bool {
    // Check database connectivity
    return true
}

func checkExternalAPI() bool {
    // Check external API availability
    return true
}

Client-Side Health Checking

Manual Health Checking

import (
    "context"
    "time"
    "google.golang.org/grpc"
    healthpb "google.golang.org/grpc/health/grpc_health_v1"
)

conn, err := grpc.NewClient("localhost:50051",
    grpc.WithTransportCredentials(insecure.NewCredentials()))
if err != nil {
    log.Fatal(err)
}
defer conn.Close()

// Create health client
healthClient := healthpb.NewHealthClient(conn)

// Check overall server health
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
defer cancel()

resp, err := healthClient.Check(ctx, &healthpb.HealthCheckRequest{
    Service: "", // Empty for overall health
})
if err != nil {
    log.Printf("Health check failed: %v", err)
} else {
    log.Printf("Server health status: %v", resp.GetStatus())
}

// Check specific service health
resp, err = healthClient.Check(ctx, &healthpb.HealthCheckRequest{
    Service: "mypackage.MyService",
})
if err != nil {
    log.Printf("Service health check failed: %v", err)
} else {
    log.Printf("Service health status: %v", resp.GetStatus())
}

Watch Health Status

import (
    "context"
    "io"
    "log"
    "google.golang.org/grpc"
    healthpb "google.golang.org/grpc/health/grpc_health_v1"
)

conn, err := grpc.NewClient("localhost:50051",
    grpc.WithTransportCredentials(insecure.NewCredentials()))
if err != nil {
    log.Fatal(err)
}
defer conn.Close()

healthClient := healthpb.NewHealthClient(conn)

// Watch service health status
stream, err := healthClient.Watch(context.Background(),
    &healthpb.HealthCheckRequest{Service: "mypackage.MyService"})
if err != nil {
    log.Fatal(err)
}

// Receive health status updates
for {
    resp, err := stream.Recv()
    if err == io.EOF {
        break
    }
    if err != nil {
        log.Printf("Watch error: %v", err)
        break
    }

    log.Printf("Health status changed: %v", resp.GetStatus())

    switch resp.GetStatus() {
    case healthpb.HealthCheckResponse_SERVING:
        log.Println("Service is now healthy")
    case healthpb.HealthCheckResponse_NOT_SERVING:
        log.Println("Service is now unhealthy")
    case healthpb.HealthCheckResponse_SERVICE_UNKNOWN:
        log.Println("Service is unknown")
    }
}

List All Services

import (
    "context"
    "time"
    "google.golang.org/grpc"
    healthpb "google.golang.org/grpc/health/grpc_health_v1"
)

conn, err := grpc.NewClient("localhost:50051",
    grpc.WithTransportCredentials(insecure.NewCredentials()))
if err != nil {
    log.Fatal(err)
}
defer conn.Close()

healthClient := healthpb.NewHealthClient(conn)

// List all services
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
defer cancel()

resp, err := healthClient.List(ctx, &healthpb.HealthListRequest{})
if err != nil {
    log.Printf("List failed: %v", err)
} else {
    log.Println("Available services:")
    for _, service := range resp.Services {
        log.Printf("  - %s: %v", service.Service, service.Status)
    }
}

Automatic Health Checking

Configure automatic client-side health checking:

import (
    "google.golang.org/grpc"
    "google.golang.org/grpc/credentials/insecure"
)

// Enable client-side health checking via service config
serviceConfig := `{
    "healthCheckConfig": {
        "serviceName": "mypackage.MyService"
    }
}`

conn, err := grpc.NewClient("localhost:50051",
    grpc.WithDefaultServiceConfig(serviceConfig),
    grpc.WithTransportCredentials(insecure.NewCredentials()))
if err != nil {
    log.Fatal(err)
}
defer conn.Close()

// gRPC will automatically perform health checks
// and remove unhealthy connections from load balancing

Health Checking with Load Balancing

Per-Connection Health Checking

import (
    "google.golang.org/grpc"
    "google.golang.org/grpc/credentials/insecure"
)

// Service config with health checking and load balancing
serviceConfig := `{
    "loadBalancingPolicy": "round_robin",
    "healthCheckConfig": {
        "serviceName": ""
    }
}`

conn, err := grpc.NewClient("dns:///myservice.example.com:50051",
    grpc.WithDefaultServiceConfig(serviceConfig),
    grpc.WithTransportCredentials(insecure.NewCredentials()))
if err != nil {
    log.Fatal(err)
}
defer conn.Close()

// Each backend connection will be health checked
// Unhealthy backends are removed from load balancing rotation

Best Practices

Server-Side

  1. Register early: Register health server before other services
  2. Set initial status: Set health status after services are ready
  3. Update dynamically: Monitor dependencies and update health status
  4. Graceful shutdown: Set NOT_SERVING before stopping server
  5. Per-service status: Track health of individual services separately

Client-Side

  1. Set deadlines: Always set timeouts for health check RPCs
  2. Handle failures: Treat health check failures as unhealthy
  3. Use Watch: Prefer Watch over periodic Check for real-time updates
  4. Retry logic: Implement retry with backoff for health checks
  5. Service-specific: Check specific services rather than overall health when possible

Integration

import (
    "context"
    "database/sql"
    "time"
    "google.golang.org/grpc"
    "google.golang.org/grpc/health"
    healthpb "google.golang.org/grpc/health/grpc_health_v1"
)

type server struct {
    pb.UnimplementedMyServiceServer
    db           *sql.DB
    healthServer *health.Server
}

func (s *server) startup() error {
    // Initialize dependencies
    if err := s.db.Ping(); err != nil {
        return err
    }

    // Set health status after successful startup
    s.healthServer.SetServingStatus("mypackage.MyService",
        healthpb.HealthCheckResponse_SERVING)
    s.healthServer.SetServingStatus("",
        healthpb.HealthCheckResponse_SERVING)

    // Start health monitoring
    go s.monitorHealth()

    return nil
}

func (s *server) monitorHealth() {
    ticker := time.NewTicker(5 * time.Second)
    defer ticker.Stop()

    for range ticker.C {
        healthy := true

        // Check database
        ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
        if err := s.db.PingContext(ctx); err != nil {
            log.Printf("Database unhealthy: %v", err)
            healthy = false
        }
        cancel()

        // Update health status
        status := healthpb.HealthCheckResponse_SERVING
        if !healthy {
            status = healthpb.HealthCheckResponse_NOT_SERVING
        }

        s.healthServer.SetServingStatus("mypackage.MyService", status)
    }
}

func (s *server) shutdown() {
    // Mark unhealthy before shutdown
    s.healthServer.Shutdown()

    // Give clients time to detect
    time.Sleep(5 * time.Second)

    // Close resources
    s.db.Close()
}

Kubernetes Integration

// In Kubernetes, configure liveness and readiness probes
// to use gRPC health checking

// Deployment YAML:
// livenessProbe:
//   grpc:
//     port: 50051
//     service: ""
//   initialDelaySeconds: 10
//   periodSeconds: 5
//
// readinessProbe:
//   grpc:
//     port: 50051
//     service: "mypackage.MyService"
//   initialDelaySeconds: 5
//   periodSeconds: 3

Testing

import (
    "context"
    "testing"
    "google.golang.org/grpc"
    "google.golang.org/grpc/credentials/insecure"
    "google.golang.org/grpc/health"
    healthpb "google.golang.org/grpc/health/grpc_health_v1"
    "google.golang.org/grpc/test/bufconn"
)

func TestHealthCheck(t *testing.T) {
    // Create in-memory listener
    lis := bufconn.Listen(1024 * 1024)
    server := grpc.NewServer()
    defer server.Stop()

    // Register health server
    healthServer := health.NewServer()
    healthpb.RegisterHealthServer(server, healthServer)
    healthServer.SetServingStatus("", healthpb.HealthCheckResponse_SERVING)

    go server.Serve(lis)

    // Create client
    conn, err := grpc.NewClient("bufnet",
        grpc.WithContextDialer(func(ctx context.Context, s string) (net.Conn, error) {
            return lis.DialContext(ctx)
        }),
        grpc.WithTransportCredentials(insecure.NewCredentials()))
    if err != nil {
        t.Fatal(err)
    }
    defer conn.Close()

    // Test health check
    client := healthpb.NewHealthClient(conn)
    resp, err := client.Check(context.Background(),
        &healthpb.HealthCheckRequest{Service: ""})
    if err != nil {
        t.Fatalf("Check failed: %v", err)
    }

    if resp.GetStatus() != healthpb.HealthCheckResponse_SERVING {
        t.Errorf("Expected SERVING, got %v", resp.GetStatus())
    }
}