Retry admission webhooks on transport errors #139105

README.md

1. Create a KIND cluster
2. Compile the webhook go build -o /tmp/webhook webhook.go
3. Build the image docker build -t fake-webhook:latest -f /tmp/Dockerfile /tmp
4. Load the image in a kind cluster kind load docker-image fake-webhook:latest --name kind
5. Certficates and deploy

mkdir -p /tmp/certs
# Generate CA
openssl genrsa -out /tmp/certs/ca.key 2048
openssl req -x509 -new -nodes -key /tmp/certs/ca.key -subj "/CN=Webhook CA" -days 365 -out /tmp/certs/ca.crt
# Generate Server Key & CSR
openssl genrsa -out /tmp/certs/tls.key 2048
openssl req -new -key /tmp/certs/tls.key -subj "/CN=webhook-service.default.svc" -out /tmp/certs/tls.csr
# Sign Server Certificate with SAN
cat <<EOF > /tmp/certs/san.cnf
[req]
req_extensions = v3_req
distinguished_name = req_distinguished_name
[req_distinguished_name]
[ v3_req ]
basicConstraints = CA:FALSE
keyUsage = nonRepudiation, digitalSignature, keyEncipherment
subjectAltName = @alt_names
[alt_names]
DNS.1 = webhook-service.default.svc
EOF
openssl x509 -req -in /tmp/certs/tls.csr -CA /tmp/certs/ca.crt -CAkey /tmp/certs/ca.key -CAcreateserial -out /tmp/certs/tls.crt -days 365 -extfile /tmp/certs/san.cnf -extensions v3_req

6. Upload certificates

kubectl create secret generic webhook-certs \
  --from-file=tls.crt=/tmp/certs/tls.crt \
  --from-file=tls.key=/tmp/certs/tls.key \
  -n default

7. Deploy the webhook kubectl apply -f webhook.yaml
8. Create a namespace and register the webhook

kubectl create namespace demo

CA_BUNDLE=$(cat /tmp/certs/ca.crt | base64 | tr -d '\n')

cat <<EOF | kubectl apply -f -
apiVersion: admissionregistration.k8s.io/v1
kind: ValidatingWebhookConfiguration
metadata:
  name: fake-webhook-config
webhooks:
- name: webhook-service.default.svc
  clientConfig:
    service:
      name: webhook-service
      namespace: default
      path: "/validate"
    caBundle: ${CA_BUNDLE}
  rules:
  - operations: ["CREATE"]
    apiGroups: [""]
    apiVersions: ["v1"]
    resources: ["configmaps"]
    scope: "Namespaced"
  admissionReviewVersions: ["v1"]
  sideEffects: None
  failurePolicy: Fail
  namespaceSelector:
    matchLabels:
      kubernetes.io/metadata.name: demo
EOF

9. Exercise the webhooks

for i in $(seq 1 20); do
  kubectl create configmap test-cm-$i -n demo --from-literal=key=value
  kubectl delete configmap test-cm-$i -n demo
done

10. View the logs to see the load balancing of requests

PODS=$(kubectl get pods -l app=fake-webhook -o jsonpath='{.items[*].metadata.name}')
# View logs for both pods
for pod in $PODS; do
  echo "--- Logs for $pod ---"
  kubectl logs $pod | grep "WEBHOOK"
done
--- Logs for fake-webhook-78c969995d-8wpd8 ---
[WEBHOOK] AdmissionRequest UID 970cfae8-0138-4146-9b26-bfc6bfd3bdf0 successfully processed by Pod: fake-webhook-78c969995d-8wpd8
[WEBHOOK] AdmissionRequest UID 2a51cb81-a88b-4a74-b5d4-91251c0235ef successfully processed by Pod: fake-webhook-78c969995d-8wpd8
[WEBHOOK] AdmissionRequest UID d4350207-4ba3-4daf-9f64-27b357470d7d successfully processed by Pod: fake-webhook-78c969995d-8wpd8
[WEBHOOK] AdmissionRequest UID a2442b6b-1a4c-4d5b-92da-efe9fc53e917 successfully processed by Pod: fake-webhook-78c969995d-8wpd8
[WEBHOOK] AdmissionRequest UID 756d9878-c2bc-43e6-aed6-5a1f6bc4289c successfully processed by Pod: fake-webhook-78c969995d-8wpd8
[WEBHOOK] AdmissionRequest UID 6dceeefe-b2a6-48b4-bc96-611bd5d42d65 successfully processed by Pod: fake-webhook-78c969995d-8wpd8
[WEBHOOK] AdmissionRequest UID 0d86221f-cf85-49bf-99f4-28285f59d3ca successfully processed by Pod: fake-webhook-78c969995d-8wpd8
[WEBHOOK] AdmissionRequest UID 0259331d-a255-4d22-b7fa-86fe5e055e25 successfully processed by Pod: fake-webhook-78c969995d-8wpd8
[WEBHOOK] AdmissionRequest UID 0f70afa2-1a84-4574-8ac4-369bf21fe5d9 successfully processed by Pod: fake-webhook-78c969995d-8wpd8
[WEBHOOK] AdmissionRequest UID 1a4b56e9-3519-42e1-931b-7069a1225052 successfully processed by Pod: fake-webhook-78c969995d-8wpd8
[WEBHOOK] AdmissionRequest UID de336d28-2a17-452a-9e8d-3ca584142497 successfully processed by Pod: fake-webhook-78c969995d-8wpd8
[WEBHOOK] AdmissionRequest UID 861ecb0b-41ee-4d09-84d6-4af81426bbdb successfully processed by Pod: fake-webhook-78c969995d-8wpd8
[WEBHOOK] AdmissionRequest UID 8c441c95-7651-4c31-ad68-6b92aa743387 successfully processed by Pod: fake-webhook-78c969995d-8wpd8
[WEBHOOK] AdmissionRequest UID 000798bc-43f3-45a5-9c6a-cc7988d7d1a1 successfully processed by Pod: fake-webhook-78c969995d-8wpd8
[WEBHOOK] AdmissionRequest UID 177cfb91-8573-40b9-97b5-17e4382265be successfully processed by Pod: fake-webhook-78c969995d-8wpd8
[WEBHOOK] AdmissionRequest UID cef9403f-0e43-4c08-acf2-e2ac33d7d93d successfully processed by Pod: fake-webhook-78c969995d-8wpd8
[WEBHOOK] AdmissionRequest UID 027b3e2b-1f85-47cf-ab2b-4e63dd662fad successfully processed by Pod: fake-webhook-78c969995d-8wpd8
[WEBHOOK] AdmissionRequest UID 960bed55-6626-455e-bb3a-dd882ad9a4be successfully processed by Pod: fake-webhook-78c969995d-8wpd8
[WEBHOOK] AdmissionRequest UID d9d0001b-66cf-4dcd-81d7-99bcffffafba successfully processed by Pod: fake-webhook-78c969995d-8wpd8
[WEBHOOK] AdmissionRequest UID f9c2c0ce-424c-428d-bce1-e7f693065668 successfully processed by Pod: fake-webhook-78c969995d-8wpd8
--- Logs for fake-webhook-78c969995d-f7p4b ---

The same pod receive all the requests

Dockerfile

FROM alpine:latest
COPY webhook /usr/local/bin/webhook
ENTRYPOINT ["/usr/local/bin/webhook"]

webhook.go

package main

import (
	"encoding/json"
	"fmt"
	"io"
	"net/http"
	"os"
)

func main() {
	podName := os.Getenv("POD_NAME")
	http.HandleFunc("/validate", func(w http.ResponseWriter, r *http.Request) {
		defer r.Body.Close()
		body, err := io.ReadAll(r.Body)
		if err != nil {
			w.WriteHeader(http.StatusInternalServerError)
			return
		}
		var review struct {
			Request struct {
				UID string `json:"uid"`
			} `json:"request"`
		}
		if err := json.Unmarshal(body, &review); err != nil {
			w.WriteHeader(http.StatusBadRequest)
			return
		}
		fmt.Printf("[WEBHOOK] AdmissionRequest UID %s successfully processed by Pod: %s\n", review.Request.UID, podName)
		w.Header().Set("Content-Type", "application/json")
		w.Write([]byte(fmt.Sprintf(`{"apiVersion":"admission.k8s.io/v1","kind":"AdmissionReview","response":{"uid":"%s","allowed":true}}`, review.Request.UID)))
	})
	fmt.Printf("Starting secure webhook server on :8443, pod=%s...\n", podName)
	err := http.ListenAndServeTLS(":8443", "/etc/webhook/certs/tls.crt", "/etc/webhook/certs/tls.key", nil)
	if err != nil {
		fmt.Printf("Failed to start server: %v\n", err)
	}
}

webhook.yaml

apiVersion: apps/v1
kind: Deployment
metadata:
  name: fake-webhook
  namespace: default
  labels:
    app: fake-webhook
spec:
  replicas: 2
  selector:
    matchLabels:
      app: fake-webhook
  template:
    metadata:
      labels:
        app: fake-webhook
    spec:
      containers:
      - name: webhook
        image: fake-webhook:latest
        imagePullPolicy: Never
        env:
        - name: POD_NAME
          valueFrom:
            fieldRef:
              fieldPath: metadata.name
        ports:
        - containerPort: 8443
        volumeMounts:
        - name: certs
          mountPath: /etc/webhook/certs
          readOnly: true
      volumes:
      - name: certs
        secret:
          secretName: webhook-certs
---
apiVersion: v1
kind: Service
metadata:
  name: webhook-service
  namespace: default
spec:
  ports:
  - port: 443
    targetPort: 8443
  selector:
    app: fake-webhook