or run

npx @tessl/cli init
Log in

Version

Tile

Overview

Evals

Files

docs

admin.mdadvanced.mdclient-server.mdcredentials-security.mderrors-status.mdhealth.mdindex.mdinterceptors.mdload-balancing.mdmetadata-context.mdname-resolution.mdobservability.mdreflection.mdstreaming.mdtesting.mdxds.md
tile.json

observability.mddocs/

Observability Features

This document covers observability features in gRPC-Go, including metrics, stats, channelz, and integration with monitoring systems.

Stats Handler

Overview

Stats handlers provide hooks to collect RPC statistics.

import "google.golang.org/grpc/stats"

type Handler interface {
    // TagRPC attaches tags to the context for this RPC
    TagRPC(context.Context, *RPCTagInfo) context.Context

    // HandleRPC processes RPC stats
    HandleRPC(context.Context, RPCStats)

    // TagConn attaches tags to the context for this connection
    TagConn(context.Context, *ConnTagInfo) context.Context

    // HandleConn processes connection stats
    HandleConn(context.Context, ConnStats)
}

Stats Types

// RPC stats
type RPCStats interface {
    IsClient() bool  // true for client-side stats
}

// InHeader: received response header (client) or request header (server)
type InHeader struct {
    Client        bool
    WireLength    int
    Header        metadata.MD
    Compression   string
    LocalAddr     net.Addr
    RemoteAddr    net.Addr
}

// OutHeader: sent request header (client) or response header (server)
type OutHeader struct {
    Client        bool
    WireLength    int
    Header        metadata.MD
    Compression   string
}

// InPayload: received message
type InPayload struct {
    Client      bool
    Payload     any
    Data        []byte
    Length      int
    WireLength  int
    RecvTime    time.Time
}

// OutPayload: sent message
type OutPayload struct {
    Client      bool
    Payload     any
    Data        []byte
    Length      int
    WireLength  int
    SentTime    time.Time
}

// End: RPC ended
type End struct {
    Client      bool
    BeginTime   time.Time
    EndTime     time.Time
    Error       error
}

Custom Stats Handler

import (
    "context"
    "log"
    "google.golang.org/grpc"
    "google.golang.org/grpc/stats"
)

type myStatsHandler struct{}

func (h *myStatsHandler) TagRPC(ctx context.Context, info *stats.RPCTagInfo) context.Context {
    return ctx
}

func (h *myStatsHandler) HandleRPC(ctx context.Context, s stats.RPCStats) {
    switch st := s.(type) {
    case *stats.InHeader:
        log.Printf("InHeader: %v", st.Header)
    case *stats.OutHeader:
        log.Printf("OutHeader: %v", st.Header)
    case *stats.InPayload:
        log.Printf("InPayload: length=%d", st.Length)
    case *stats.OutPayload:
        log.Printf("OutPayload: length=%d", st.Length)
    case *stats.End:
        log.Printf("End: duration=%v error=%v", st.EndTime.Sub(st.BeginTime), st.Error)
    }
}

func (h *myStatsHandler) TagConn(ctx context.Context, info *stats.ConnTagInfo) context.Context {
    return ctx
}

func (h *myStatsHandler) HandleConn(ctx context.Context, s stats.ConnStats) {
    // Handle connection stats
}

// Use stats handler
conn, err := grpc.NewClient("localhost:50051",
    grpc.WithStatsHandler(&myStatsHandler{}),
    grpc.WithTransportCredentials(insecure.NewCredentials()))

Channelz

Overview

Channelz provides real-time debugging information about channels and connections.

import (
    "google.golang.org/grpc/channelz/service"
    channelzpb "google.golang.org/grpc/channelz/grpc_channelz_v1"
)

// Enable channelz on server
server := grpc.NewServer()
service.RegisterChannelzServiceToServer(server)

Channelz Service

Query channelz information:

# Using grpcurl
grpcurl -plaintext localhost:50051 grpc.channelz.v1.Channelz.GetTopChannels
grpcurl -plaintext localhost:50051 grpc.channelz.v1.Channelz.GetServers
grpcurl -plaintext localhost:50051 grpc.channelz.v1.Channelz.GetChannel

Tracing

OpenTelemetry Integration

import (
    "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc"
    "google.golang.org/grpc"
)

// Client with tracing
conn, err := grpc.NewClient("localhost:50051",
    grpc.WithStatsHandler(otelgrpc.NewClientHandler()),
    grpc.WithTransportCredentials(insecure.NewCredentials()))

// Server with tracing
server := grpc.NewServer(
    grpc.StatsHandler(otelgrpc.NewServerHandler()))

Metrics

Prometheus Integration

import (
    "github.com/grpc-ecosystem/go-grpc-prometheus"
    "github.com/prometheus/client_golang/prometheus"
    "github.com/prometheus/client_golang/prometheus/promhttp"
)

// Server metrics
grpcMetrics := grpc_prometheus.NewServerMetrics()
server := grpc.NewServer(
    grpc.ChainUnaryInterceptor(grpcMetrics.UnaryServerInterceptor()),
    grpc.ChainStreamInterceptor(grpcMetrics.StreamServerInterceptor()))

// Register metrics
prometheus.MustRegister(grpcMetrics)

// Initialize metrics after registration
grpcMetrics.InitializeMetrics(server)

// Expose metrics endpoint
http.Handle("/metrics", promhttp.Handler())
go http.ListenAndServe(":9090", nil)

Logging

Structured Logging

import (
    "context"
    "log/slog"
    "google.golang.org/grpc"
)

func loggingInterceptor(
    ctx context.Context,
    req any,
    info *grpc.UnaryServerInfo,
    handler grpc.UnaryHandler,
) (any, error) {
    logger := slog.Default()

    logger.InfoContext(ctx, "RPC started",
        "method", info.FullMethod,
        "request", req)

    resp, err := handler(ctx, req)

    if err != nil {
        logger.ErrorContext(ctx, "RPC failed",
            "method", info.FullMethod,
            "error", err)
    } else {
        logger.InfoContext(ctx, "RPC completed",
            "method", info.FullMethod)
    }

    return resp, err
}

Best Practices

  1. Use stats handlers: For detailed metrics collection
  2. Enable channelz: For debugging in development
  3. Distributed tracing: Integrate OpenTelemetry for tracing
  4. Structured logging: Use structured logging with context
  5. Metrics exposition: Expose Prometheus metrics endpoint
  6. Performance impact: Monitor overhead of observability features