Karpenter Reference

Unschedulable Pod
       │
       ▼
Karpenter controller
  ├── finds best NodePool match
  ├── selects optimal instance type
  ├── launches EC2 directly (RunInstances API)
  └── creates NodeClaim → Node registers → Pod schedules

NodePool          — scheduling requirements, limits, disruption policy
  └── EC2NodeClass  — AMI, subnet, security group, instance profile
       └── NodeClaim — one per provisioned node, lifecycle-tracked by controller
            └── Node — the actual Kubernetes node

# 1. EKS cluster with OIDC provider (for IRSA) or Pod Identity agent (for EKS Pod Identity)
aws eks describe-cluster --name <cluster> --region eu-north-1 \
  --query 'cluster.identity.oidc.issuer'

# 2. Node IAM role — separate from the EKS worker role, used by Karpenter-provisioned nodes
#    Attach: AmazonEKSWorkerNodePolicy, AmazonEKS_CNI_Policy, AmazonEC2ContainerRegistryReadOnly
#    Plus: EC2 instance profile for the role

# 3. Controller IAM policy — allows Karpenter to call EC2, SQS, SSM, IAM
#    See minimum policy in the IAM section below

# 4. Tag subnets, node security groups, AND the EKS cluster security group for discovery.
#    Missing the cluster SG causes NodeClaim to fail with InvalidParameterValue: securityGroupIds.

# Tag all worker/node subnets
aws ec2 create-tags \
  --resources <subnet-id> <subnet-id-2> \
  --tags Key=karpenter.sh/discovery,Value=<cluster-name>

# Tag the node security group (the one attached to worker nodes)
aws ec2 create-tags \
  --resources <node-security-group-id> \
  --tags Key=karpenter.sh/discovery,Value=<cluster-name>

# Also tag the EKS cluster security group (the one EKS creates automatically)
CLUSTER_SG=$(aws eks describe-cluster --name <cluster-name> --region eu-north-1 \
  --query 'cluster.resourcesVpcConfig.clusterSecurityGroupId' --output text)
aws ec2 create-tags \
  --resources "$CLUSTER_SG" \
  --tags Key=karpenter.sh/discovery,Value=<cluster-name>

# 5. Register the node IAM role so the EKS API server trusts Karpenter-provisioned nodes.
#    Without this, nodes launch and immediately go NotReady with no Karpenter log error.

# EKS 1.29+ — access entry (replaces aws-auth)
aws eks create-access-entry \
  --cluster-name <cluster-name> \
  --principal-arn arn:aws:iam::<account-id>:role/<node-role-name> \
  --type EC2_LINUX \
  --region eu-north-1

# EKS < 1.29 — add to aws-auth ConfigMap instead:
# - groups: [system:bootstrappers, system:nodes]
#   rolearn: arn:aws:iam::<account-id>:role/<node-role-name>
#   username: system:node:{{EC2PrivateDNSName}}

# Create SQS queue
aws sqs create-queue \
  --queue-name karpenter-<cluster-name> \
  --attributes MessageRetentionPeriod=300

# EventBridge rules (one per event type)
aws events put-rule --name KarpenterSpotInterruption \
  --event-pattern '{"source":["aws.ec2"],"detail-type":["EC2 Spot Instance Interruption Warning"]}'

aws events put-targets --rule KarpenterSpotInterruption \
  --targets Id=karpenter-queue,Arn=<sqs-arn>
# Repeat for: EC2 Instance Rebalance Recommendation, EC2 Instance State-change Notification

# Log in to the public ECR registry (no credentials needed for public)
helm registry login public.ecr.aws --username AWS --password $(
  aws ecr-public get-login-password --region us-east-1
)

helm upgrade --install karpenter \
  oci://public.ecr.aws/karpenter/karpenter \
  --version "1.12.1" \
  --namespace karpenter \
  --create-namespace \
  --set "settings.clusterName=<cluster-name>" \
  --set "settings.interruptionQueue=karpenter-<cluster-name>" \
  --set "serviceAccount.annotations.eks\.amazonaws\.com/role-arn=<controller-role-arn>" \
  --set controller.resources.requests.cpu=1 \
  --set controller.resources.requests.memory=1Gi \
  --set controller.resources.limits.cpu=1 \
  --set controller.resources.limits.memory=1Gi \
  --wait

kubectl get pods -n karpenter
# NAME                         READY   STATUS    RESTARTS
# karpenter-xxx                1/1     Running   0

kubectl get nodepool,ec2nodeclass -A
kubectl describe nodepool <name>   # Conditions: Ready=True

{
  "Version": "2012-10-17",
  "Statement": [
    {
      "Sid": "AllowScopedEC2InstanceActions",
      "Effect": "Allow",
      "Action": [
        "ec2:RunInstances",
        "ec2:CreateFleet"
      ],
      "Resource": [
        "arn:aws:ec2:<region>::image/*",
        "arn:aws:ec2:<region>::snapshot/*",
        "arn:aws:ec2:<region>:*:security-group/*",
        "arn:aws:ec2:<region>:*:subnet/*",
        "arn:aws:ec2:<region>:*:launch-template/*"
      ]
    },
    {
      "Sid": "AllowScopedEC2InstanceActionsWithTags",
      "Effect": "Allow",
      "Action": [
        "ec2:RunInstances",
        "ec2:CreateFleet",
        "ec2:CreateLaunchTemplate"
      ],
      "Resource": [
        "arn:aws:ec2:<region>:*:fleet/*",
        "arn:aws:ec2:<region>:*:instance/*",
        "arn:aws:ec2:<region>:*:volume/*",
        "arn:aws:ec2:<region>:*:network-interface/*",
        "arn:aws:ec2:<region>:*:launch-template/*",
        "arn:aws:ec2:<region>:*:spot-instances-request/*"
      ],
      "Condition": {
        "StringEquals": {
          "aws:RequestTag/kubernetes.io/cluster/<cluster-name>": "owned"
        },
        "StringLike": {
          "aws:RequestTag/karpenter.sh/nodepool": "*"
        }
      }
    },
    {
      "Sid": "AllowScopedDeletion",
      "Effect": "Allow",
      "Action": [
        "ec2:TerminateInstances",
        "ec2:DeleteLaunchTemplate"
      ],
      "Resource": [
        "arn:aws:ec2:<region>:*:instance/*",
        "arn:aws:ec2:<region>:*:launch-template/*"
      ],
      "Condition": {
        "StringEquals": {
          "aws:ResourceTag/kubernetes.io/cluster/<cluster-name>": "owned"
        },
        "StringLike": {
          "aws:ResourceTag/karpenter.sh/nodepool": "*"
        }
      }
    },
    {
      "Sid": "AllowRegionalReadActions",
      "Effect": "Allow",
      "Action": [
        "ec2:DescribeAvailabilityZones",
        "ec2:DescribeImages",
        "ec2:DescribeInstances",
        "ec2:DescribeInstanceTypeOfferings",
        "ec2:DescribeInstanceTypes",
        "ec2:DescribeLaunchTemplates",
        "ec2:DescribeSecurityGroups",
        "ec2:DescribeSpotPriceHistory",
        "ec2:DescribeSubnets"
      ],
      "Resource": "*",
      "Condition": {
        "StringEquals": {
          "aws:RequestedRegion": "<region>"
        }
      }
    },
    {
      "Sid": "AllowSSMReadActions",
      "Effect": "Allow",
      "Action": "ssm:GetParameter",
      "Resource": "arn:aws:ssm:<region>::parameter/aws/service/*"
    },
    {
      "Sid": "AllowInterruptionQueueActions",
      "Effect": "Allow",
      "Action": [
        "sqs:DeleteMessage",
        "sqs:GetQueueAttributes",
        "sqs:GetQueueUrl",
        "sqs:ReceiveMessage"
      ],
      "Resource": "arn:aws:sqs:<region>:<account-id>:karpenter-<cluster-name>"
    },
    {
      "Sid": "AllowPassingInstanceRole",
      "Effect": "Allow",
      "Action": "iam:PassRole",
      "Resource": "arn:aws:iam::<account-id>:role/<node-role-name>",
      "Condition": {
        "StringEquals": {
          "iam:PassedToService": "ec2.amazonaws.com"
        }
      }
    },
    {
      "Sid": "AllowScopedInstanceProfileActions",
      "Effect": "Allow",
      "Action": [
        "iam:AddRoleToInstanceProfile",
        "iam:CreateInstanceProfile",
        "iam:DeleteInstanceProfile",
        "iam:GetInstanceProfile",
        "iam:RemoveRoleFromInstanceProfile",
        "iam:TagInstanceProfile"
      ],
      "Resource": "arn:aws:iam::<account-id>:instance-profile/karpenter*"
    },
    {
      "Sid": "AllowAPIServerEndpointDiscovery",
      "Effect": "Allow",
      "Action": "eks:DescribeCluster",
      "Resource": "arn:aws:eks:<region>:<account-id>:cluster/<cluster-name>"
    }
  ]
}

# Install the agent add-on
aws eks create-addon \
  --cluster-name <cluster> \
  --addon-name eks-pod-identity-agent \
  --region eu-north-1

# Create association
aws eks create-pod-identity-association \
  --cluster-name <cluster> \
  --namespace karpenter \
  --service-account karpenter \
  --role-arn arn:aws:iam::<account-id>:role/karpenter-controller

# Annotate the Karpenter service account
kubectl annotate serviceaccount karpenter -n karpenter \
  eks.amazonaws.com/role-arn=arn:aws:iam::<account-id>:role/karpenter-controller

apiVersion: karpenter.k8s.aws/v1
kind: EC2NodeClass
metadata:
  name: default
spec:
  # AMI selection — alias resolves to the latest EKS-optimised AMI for the cluster version
  # Use alias for dev/staging; pin to a specific AMI ID for production stability
  amiSelectorTerms:
    - alias: al2023@latest       # AL2023 latest for this cluster's k8s version
    # - id: ami-0abc1234def56789  # Pin to a specific tested AMI for production

  # Subnet discovery — tags applied during cluster bootstrap
  subnetSelectorTerms:
    - tags:
        karpenter.sh/discovery: my-cluster

  # Security group discovery
  securityGroupSelectorTerms:
    - tags:
        karpenter.sh/discovery: my-cluster

  # Instance profile (for node IAM role)
  instanceProfile: karpenter-node-profile   # EC2 instance profile name

  # Block device — encryption and size
  blockDeviceMappings:
    - deviceName: /dev/xvda
      ebs:
        volumeSize: 50Gi
        volumeType: gp3
        encrypted: true
        iops: 3000
        throughput: 125

  # IMDSv2 required — prevents SSRF attacks from pods accessing instance metadata
  metadataOptions:
    httpEndpoint: enabled
    httpProtocolIPv6: disabled
    httpTokens: required           # IMDSv2 only
    httpPutResponseHopLimit: 1     # Prevents pods from reaching IMDS (hop 1 = instance only)

  # Tags applied to all provisioned nodes and volumes
  tags:
    Environment: production
    ManagedBy: karpenter
    karpenter.sh/discovery: my-cluster

  # Optional: custom user-data (AL2023 uses nodeadm YAML, not bash)
  # userData: |
  #   apiVersion: node.eks.aws/v1alpha1
  #   kind: NodeConfig
  #   spec:
  #     kubelet:
  #       config:
  #         maxPods: 110

apiVersion: karpenter.sh/v1
kind: NodePool
metadata:
  name: default
spec:
  template:
    metadata:
      labels:
        nodepool: default         # Useful for node selectors and log filtering
    spec:
      nodeClassRef:
        group: karpenter.k8s.aws
        kind: EC2NodeClass
        name: default

      requirements:
        # Instance families — list at least 3 for Spot diversity
        - key: karpenter.k8s.aws/instance-family
          operator: In
          values: [m6i, m6a, m7i, m7a, c6i, c6a, r6i]
          minValues: 3            # Enforce diversity — fail if fewer than 3 families available

        # Instance size — allow medium through 2xlarge for flexible bin-packing
        - key: karpenter.k8s.aws/instance-size
          operator: In
          values: [medium, large, xlarge, 2xlarge]

        # Architecture — include Graviton for ~20% cost saving where workloads support it
        - key: kubernetes.io/arch
          operator: In
          values: [amd64, arm64]

        # Capacity type — Spot preferred; On-Demand fallback via NodePool weight
        - key: karpenter.sh/capacity-type
          operator: In
          values: [spot, on-demand]

        # AZ — tie to subnets where you have capacity
        - key: topology.kubernetes.io/zone
          operator: In
          values: [eu-north-1a, eu-north-1b, eu-north-1c]

      # startupTaints: only set when your CNI/device-plugin removes the taint on readiness.
      # karpenter.sh/not-ready has no built-in remover — omit it here.
      # Example (Cilium removes this after CNI is ready):
      # startupTaints:
      #   - key: node.cilium.io/agent-not-ready
      #     effect: NoExecute

      # Taints for workload isolation (optional)
      # taints:
      #   - key: workload-type
      #     value: batch
      #     effect: NoSchedule

      # Maximum time a node can run before Karpenter replaces it (AMI rotation, drift)
      expireAfter: 720h           # 30 days

      # Maximum time to wait for graceful termination before force-terminating
      terminationGracePeriod: 48h

  # Hard resource ceiling — never omit
  limits:
    cpu: "1000"                   # 1000 vCPUs max across all nodes in this NodePool
    memory: 4000Gi

  # NodePool selection when multiple pools exist — higher weight = preferred
  weight: 10

  disruption:
    # consolidationPolicy options:
    #   WhenEmpty               — only remove fully empty nodes
    #   WhenEmptyOrUnderutilized — also consolidate underutilised nodes (default recommendation)
    consolidationPolicy: WhenEmptyOrUnderutilized
    consolidateAfter: 5m          # Minimum safe value — 1m resets on every pod event and causes churn

    # Disruption budgets — protect critical workloads from simultaneous eviction
    budgets:
      - nodes: "10%"              # Default: max 10% of nodes disrupted at once
      - schedule: "0 9 * * 1-5"  # During business hours: disrupt max 0 nodes
        duration: 8h
        nodes: "0"

NodePool: critical-ondemand   weight: 100   On-Demand only, m-family
NodePool: spot-flex           weight: 50    Spot preferred, broad families
NodePool: default             weight: 10    On-Demand fallback, general

requirements:
  - key: karpenter.k8s.aws/instance-family
    operator: In
    values: [m6i, m6a, m7i, m7a, c6i, c6a, c7i, c7a, r6i, r7i]
    minValues: 3    # Hard floor — Karpenter will not provision if fewer than 3 families available
  - key: karpenter.sh/capacity-type
    operator: In
    values: [spot, on-demand]   # spot preferred; on-demand fallback

requirements:
  # Block burstable instances — CPU credit behaviour causes latency spikes under sustained load
  - key: karpenter.k8s.aws/instance-family
    operator: NotIn
    values: [t2, t3, t3a, t4g]

  # Block previous-generation instances — worse price/performance than gen 6+
  - key: karpenter.k8s.aws/instance-generation
    operator: Gt
    values: ["5"]   # Only gen 6 and above (m6i, c6i, r6i, m7i, ...)

  # Combine with an In requirement to allow only specific families
  - key: karpenter.k8s.aws/instance-family
    operator: In
    values: [m6i, m6a, m7i, m7a, c6i, c6a, c7i, r6i, r7i]

# Spot-first NodePool (weight: 100) — Karpenter tries this first
# On-Demand fallback NodePool (weight: 10) — used when Spot unavailable

- key: kubernetes.io/arch
  operator: In
  values: [amd64, arm64]

# DockerHub / public registry
docker manifest inspect <image>:<tag> | grep -i '"architecture"'
# Expected output: "architecture": "amd64" and "architecture": "arm64"

# ECR (private registry) — requires AWS credentials
aws ecr describe-image-scan-findings --repository-name <repo> --image-id imageTag=<tag> \
  --region eu-north-1 --query 'imageScanFindings' 2>/dev/null || \
aws ecr batch-get-image \
  --repository-name <repo> \
  --image-ids imageTag=<tag> \
  --query 'images[*].imageManifest' --output text | \
  python3 -c "import sys,json; m=json.load(sys.stdin); [print(e.get('platform',{}).get('architecture','?')) for e in m.get('manifests',[])]"

# Fastest check — pull the manifest and look for multi-arch index
crane manifest <image>:<tag> | jq '[.manifests[]?.platform.architecture]'
# crane: go install github.com/google/go-containerregistry/cmd/crane@latest

exec /app/server: exec format error

apiVersion: karpenter.k8s.aws/v1
kind: EC2NodeClass
metadata:
  name: reserved-capacity
spec:
  amiSelectorTerms:
    - alias: al2023@latest

  subnetSelectorTerms:
    - tags:
        karpenter.sh/discovery: my-cluster

  securityGroupSelectorTerms:
    - tags:
        karpenter.sh/discovery: my-cluster

  instanceProfile: karpenter-node-profile

  # Target a specific Capacity Reservation or Capacity Block by ID or tag
  capacityReservationSelectorTerms:
    - id: cr-0abc1234def567890        # Specific ODCR ID
    # Or select by tag:
    # - tags:
    #     karpenter.sh/reservation-type: ml-training

  metadataOptions:
    httpTokens: required
    httpPutResponseHopLimit: 1

  blockDeviceMappings:
    - deviceName: /dev/xvda
      ebs:
        volumeSize: 100Gi
        volumeType: gp3
        encrypted: true

---
apiVersion: karpenter.sh/v1
kind: NodePool
metadata:
  name: reserved-gpu
spec:
  template:
    spec:
      nodeClassRef:
        group: karpenter.k8s.aws
        kind: EC2NodeClass
        name: reserved-capacity
      requirements:
        - key: karpenter.sh/capacity-type
          operator: In
          values: [on-demand]       # ODCR uses on-demand capacity type
        - key: karpenter.k8s.aws/instance-family
          operator: In
          values: [p4d, p4de, p5]   # Match your reservation instance type exactly
        - key: topology.kubernetes.io/zone
          operator: In
          values: [eu-north-1a]     # ODCR is AZ-specific — must match reservation AZ
  limits:
    cpu: "768"                      # p4d.24xlarge = 96 vCPU; 8 nodes worth
  disruption:
    consolidationPolicy: WhenEmpty  # Never consolidate reserved capacity nodes

aws ec2 describe-capacity-reservations \
  --capacity-reservation-ids cr-0abc1234def567890 \
  --query 'CapacityReservations[*].{Total:TotalInstanceCount,Available:AvailableInstanceCount,State:State}'

1. Create new NodePool (e.g. default-v2) with updated config and weight: 200
   — higher weight than the existing pool (weight: 10)

2. New pods scheduled on default-v2; existing pods continue running on default

3. Cordon default NodePool nodes to prevent new scheduling:
   kubectl get nodes -l karpenter.sh/nodepool=default -o name | \
     xargs kubectl cordon

4. Drain default NodePool nodes (PDBs respected):
   kubectl get nodes -l karpenter.sh/nodepool=default -o name | \
     xargs -I{} kubectl drain {} \
       --ignore-daemonsets \
       --delete-emptydir-data \
       --grace-period=120 \
       --timeout=300s

5. Verify all workloads running on default-v2 nodes:
   kubectl get pods -A -o wide | grep -v default-v2

6. Delete the old NodePool (Karpenter terminates its now-empty nodes):
   kubectl delete nodepool default

# See allocatable after DaemonSet overhead on a Karpenter node
kubectl describe node <karpenter-node> | grep -A 6 "Allocatable:"

# See what's consuming resources before your workload pod lands
kubectl describe node <karpenter-node> | grep -A 40 "Non-terminated Pods:"

disruption:
  consolidationPolicy: WhenEmptyOrUnderutilized
  consolidateAfter: 5m   # Wait 5 min after underutilisation detected

expireAfter: 720h   # 30 days — rotate monthly

1. Upgrade EKS control plane
   aws eks update-cluster-version --name <cluster> --kubernetes-version 1.31

2. Wait for control plane upgrade to complete
   aws eks wait cluster-active --name <cluster>

3a. If using alias (al2023@latest):
    SSM parameter auto-updates → Karpenter detects AMI drift → nodes replaced rolling
    (no manual action needed, but watch disruption budgets)

3b. If using pinned AMI ID:
    Find the new EKS-optimised AMI for the new version:
    aws ssm get-parameter \
      --name /aws/service/eks/optimized-ami/1.31/amazon-linux-2023/x86_64/standard/recommended/image_id \
      --query Parameter.Value --output text
    Update EC2NodeClass amiSelectorTerms[].id → commit/merge → Karpenter drifts nodes

4. Monitor replacement progress
   kubectl get nodeclaim -A -w
   kubectl get nodes -o wide | grep -v v1.31   # Should go to zero

5. Verify workloads healthy after rotation
   kubectl get pods -A | grep -v Running | grep -v Completed

# 1. PDB — prevents simultaneous eviction during drain (Karpenter and kubectl drain both respect this)
apiVersion: policy/v1
kind: PodDisruptionBudget
metadata:
  name: redis
  namespace: cache
spec:
  minAvailable: 2     # For a 3-replica StatefulSet: always keep 2 running
  selector:
    matchLabels:
      app: redis

---
# 2. Pod annotation — blocks consolidation and expiration on the node hosting this pod.
#    Add to the StatefulSet pod template, not just one pod.
metadata:
  annotations:
    karpenter.sh/do-not-disrupt: "true"

---
# 3. Match terminationGracePeriodSeconds to your actual shutdown time
spec:
  terminationGracePeriodSeconds: 300   # Must be >= your shutdown + checkpoint time

# Remove do-not-disrupt during a maintenance window to allow expiration/drift replacement
kubectl patch statefulset redis -n cache \
  --type=json \
  -p='[{"op":"remove","path":"/spec/template/metadata/annotations/karpenter.sh~1do-not-disrupt"}]'

# Re-add after replacement completes
kubectl patch statefulset redis -n cache \
  --type=merge \
  -p='{"spec":{"template":{"metadata":{"annotations":{"karpenter.sh/do-not-disrupt":"true"}}}}}'

disruption:
  budgets:
    - nodes: "10%"             # Default: max 10% of NodePool nodes disrupted concurrently
    - schedule: "0 8 * * 1-5" # Business hours: no disruption
      duration: 8h
      nodes: "0"
    - schedule: "0 0 * * 6"   # Saturday midnight: allow up to 50% (maintenance window)
      duration: 6h
      nodes: "50%"

# PodDisruptionBudget — protects the deployment during node drain
apiVersion: policy/v1
kind: PodDisruptionBudget
metadata:
  name: orders-api
  namespace: orders
spec:
  minAvailable: 1       # or maxUnavailable: 1
  selector:
    matchLabels:
      app: orders-api

# Per-pod opt-out of disruption (use sparingly — prevents consolidation)
metadata:
  annotations:
    karpenter.sh/do-not-disrupt: "true"

{
  "Version": "2012-10-17",
  "Statement": [{
    "Effect": "Deny",
    "Action": ["ec2:CreateTags", "ec2:DeleteTags"],
    "Resource": "*",
    "Condition": {
      "ForAnyValue:StringLike": {
        "aws:TagKeys": ["karpenter.sh/*"]
      },
      "StringNotEquals": {
        "aws:PrincipalARN": "arn:aws:iam::<account>:role/karpenter-controller"
      }
    }
  }]
}

# In EC2NodeClass:
metadataOptions:
  httpTokens: required
  httpPutResponseHopLimit: 1

# Check which endpoints exist
aws ec2 describe-vpc-endpoints \
  --filters "Name=vpc-id,Values=<vpc-id>" \
  --query 'VpcEndpoints[*].{Service:ServiceName,State:State}'

aws eks create-fargate-profile \
  --cluster-name <cluster> \
  --fargate-profile-name karpenter \
  --pod-execution-role-arn arn:aws:iam::<account>:role/eks-fargate-role \
  --selectors '[{"namespace":"karpenter"}]'

# Fargate profile — selects by namespace label (created via eksctl or Terraform)
# All pods in namespace 'secure-workloads' go to Fargate
aws eks create-fargate-profile \
  --cluster-name my-cluster \
  --fargate-profile-name secure \
  --pod-execution-role-arn arn:aws:iam::<account>:role/eks-fargate-role \
  --selectors '[{"namespace":"secure-workloads"}]'

# NodePool — exclude Fargate-targeted namespaces using nodeSelector
# Pods in 'secure-workloads' namespace will never match this NodePool because
# Karpenter only acts on pods that are not already matched by a Fargate profile.
# Karpenter and the Fargate scheduler do not interfere — the EKS scheduler
# assigns Fargate pods before Karpenter sees them as unschedulable.

# Verify Karpenter controller is on Fargate or a non-Karpenter node
kubectl get pod -n karpenter -l app.kubernetes.io/name=karpenter -o wide
kubectl get node <node-name> -o jsonpath='{.metadata.labels.karpenter\.sh/nodepool}'
# Should be empty (not a Karpenter-managed node) or show "fargate"

KEDA detects queue depth → scales Deployment replicas →
pods Pending (no node) → Karpenter provisions node → pods schedule

# Find pods blocking eviction
kubectl get pods -A -o json | jq '
  .items[] |
  select(.metadata.annotations["cluster-autoscaler.kubernetes.io/safe-to-evict"] == "false") |
  {name: .metadata.name, ns: .metadata.namespace}
'

# NodePool status and limit usage
kubectl get nodepool -A
kubectl describe nodepool <name>

# NodeClaim lifecycle
kubectl get nodeclaim -A
kubectl describe nodeclaim <name>   # Conditions block shows why it's stuck

# Karpenter controller logs (most signal is here)
kubectl logs -n karpenter -l app.kubernetes.io/name=karpenter \
  --since=30m --tail=300

# Pending pods
kubectl get pods -A --field-selector=status.phase=Pending
kubectl describe pod <name> -n <ns>   # Events: FailedScheduling tells you why

# Check what the pod requires
kubectl describe pod <name> -n <ns> | grep -A 20 "Node-Selectors\|Tolerations\|Affinity"

# Check what NodePool offers
kubectl describe nodepool <name> | grep -A 30 "Requirements"

kubectl describe nodeclaim <name>
# Look for Conditions: Launched=False
# Common message: "UnauthorizedOperation" → IAM policy missing
# Common message: "InsufficientCapacityError" → Spot family too narrow
# Common message: "InvalidParameterValue: subnet" → subnet tags missing

# Test from inside the karpenter namespace
kubectl run -it --rm iamtest \
  --image=amazon/aws-cli \
  --serviceaccount=karpenter \
  --namespace=karpenter \
  -- ec2 describe-instance-types --region eu-north-1 --max-items 1

# Get the node's cloud-init logs via SSM (if SSM agent is running)
aws ssm start-session --target <instance-id>
sudo journalctl -u cloud-init -n 200
sudo journalctl -u kubelet -n 200

kubectl describe nodepool <name> | grep -A 5 "Status"
# Resources: cpu=850/1000 (85% used)

# Disruption reason
kubectl get node <name> -o jsonpath='{.annotations.karpenter\.sh/disruption-reason}'

# Events
kubectl get events -n <workload-ns> \
  --field-selector reason=Evicted \
  --sort-by='.lastTimestamp'

aws sqs get-queue-attributes \
  --queue-url <queue-url> \
  --attribute-names NumberOfMessagesReceived \
  --region eu-north-1

# Verify EventBridge rule is enabled
aws events describe-rule --name KarpenterSpotInterruption --region eu-north-1

# Look for this in pod events
kubectl describe pod <name> -n <ns> | grep -A 5 "didn't match pod's topology spread"
# or
kubectl describe pod <name> -n <ns> | grep -A 5 "ErrTopologySpreadConstraintNotSatisfiable"

# infrastructure/karpenter/kustomization.yaml — installs Helm chart + CRDs
apiVersion: kustomize.toolkit.fluxcd.io/v1
kind: Kustomization
metadata:
  name: karpenter
  namespace: flux-system
spec:
  interval: 10m
  path: ./infrastructure/karpenter
  dependsOn:
    - name: eks-addons        # cluster-autoscaler removal happens here
  healthChecks:
    - apiVersion: apps/v1
      kind: Deployment
      name: karpenter
      namespace: karpenter

---
# infrastructure/karpenter-config/kustomization.yaml — NodePool + EC2NodeClass
apiVersion: kustomize.toolkit.fluxcd.io/v1
kind: Kustomization
metadata:
  name: karpenter-config
  namespace: flux-system
spec:
  interval: 10m
  path: ./infrastructure/karpenter-config
  dependsOn:
    - name: karpenter           # CRDs must exist first
  postBuild:
    substitute:
      CLUSTER_NAME: my-cluster
      NODE_ROLE_ARN: arn:aws:iam::123456789012:role/karpenter-node

# Application: karpenter Helm chart (wave -1 — installs CRDs first)
apiVersion: argoproj.io/v1alpha1
kind: Application
metadata:
  name: karpenter
  namespace: argocd
  annotations:
    argocd.argoproj.io/sync-wave: "-1"
spec:
  source:
    repoURL: public.ecr.aws/karpenter
    chart: karpenter
    targetRevision: "1.12.1"
    helm:
      values: |
        settings:
          clusterName: my-cluster
          interruptionQueue: karpenter-my-cluster
  destination:
    server: https://kubernetes.default.svc
    namespace: karpenter
  syncPolicy:
    automated:
      prune: true
      selfHeal: true
    syncOptions:
      - CreateNamespace=true
      - ServerSideApply=true   # Required for Karpenter CRD installation

---
# Application: karpenter-config (NodePool + EC2NodeClass, wave 0 — after CRDs)
apiVersion: argoproj.io/v1alpha1
kind: Application
metadata:
  name: karpenter-config
  namespace: argocd
  annotations:
    argocd.argoproj.io/sync-wave: "0"
spec:
  source:
    repoURL: https://github.com/my-org/gitops
    path: infrastructure/karpenter-config
    targetRevision: main
  destination:
    server: https://kubernetes.default.svc
    namespace: karpenter

# In the karpenter-config Application spec:
spec:
  ignoreDifferences:
    - group: karpenter.sh
      kind: NodePool
      jsonPointers:
        - /status
        - /metadata/annotations/kubectl.kubernetes.io~1last-applied-configuration
    - group: karpenter.k8s.aws
      kind: EC2NodeClass
      jsonPointers:
        - /status
        - /spec/amiSelectorTerms    # Karpenter resolves alias → ID; Argo CD sees drift

# EC2NodeClass tags — applied to all EC2 instances and EBS volumes
tags:
  Environment: production
  Team: platform
  CostCenter: eng-platform-001
  karpenter.sh/discovery: my-cluster

# Check which tags are active as cost allocation tags
aws ce list-cost-allocation-tags \
  --status Active \
  --query 'CostAllocationTags[*].TagKey'

# Activate a new tag (takes up to 24h to appear in reports)
aws ce update-cost-allocation-tags-status \
  --cost-allocation-tags-status TagKey=Team,Status=Active

spec:
  template:
    metadata:
      labels:
        nodepool: spot-flex   # visible in kubectl, Datadog, Kubecost
    spec:
      nodeClassRef:
        name: spot-flex

# See current node cost breakdown by NodePool (requires Kubecost or OpenCost)
kubectl port-forward -n kubecost svc/kubecost-cost-analyzer 9090:9090
# Or query the Kubecost API:
curl "http://localhost:9090/model/allocation?window=1d&aggregate=label:nodepool"

# Average node utilization (higher = better bin-packing)
kubectl top nodes | awk 'NR>1 {print $3}' | sort -n

# Node count trend (should decrease with consolidation)
kubectl get nodes --no-headers | wc -l

# Spot vs On-Demand ratio (Spot ~70-80% cheaper)
kubectl get nodes -o json | jq '[.items[] | .metadata.labels."karpenter.sh/capacity-type"] | group_by(.) | map({type: .[0], count: length})'

kubectl port-forward -n karpenter svc/karpenter 8080:8080
curl localhost:8080/metrics | grep karpenter_

# NodeClaim stuck launching for > 5 minutes
- alert: KarpenterNodeClaimStuck
  expr: karpenter_nodeclaims_total{phase="Pending"} > 0
  for: 5m
  labels:
    severity: warning
  annotations:
    summary: "Karpenter NodeClaim stuck in Pending"
    description: "{{ $value }} NodeClaim(s) have been Pending for > 5 minutes"

# No nodes in NodePool (unexpected drain)
- alert: KarpenterNodePoolEmpty
  expr: karpenter_nodes_total == 0
  for: 2m
  labels:
    severity: critical

# Check for webhook timeout errors in API server audit logs or kube-apiserver pods
kubectl logs -n kube-system -l component=kube-apiserver --tail=200 | \
  grep -i "karpenter\|webhook\|timeout\|deadline"

# Check Karpenter webhook pod responsiveness
kubectl get validatingwebhookconfigurations | grep karpenter
kubectl describe validatingwebhookconfiguration \
  validation.webhook.karpenter.sh | grep -A 5 "Timeout"

helm upgrade karpenter oci://public.ecr.aws/karpenter/karpenter \
  --version "1.12.1" \
  --namespace karpenter \
  --reuse-values \
  --set "webhook.timeoutSeconds=30" \
  --set "controller.resources.requests.cpu=2" \
  --set "controller.resources.requests.memory=2Gi" \
  --set "controller.resources.limits.cpu=4" \
  --set "controller.resources.limits.memory=4Gi"

helm upgrade karpenter ... --set "controller.env[0].name=BATCH_MAX_DURATION" \
  --set "controller.env[0].value=30s"