Merge pull request #4039 from crazy-max/kubernetes-dial-backoff-jitter

kubernetes: jitter transient dial retries

Author: crazy-max

driver/kubernetes/driver.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	stderrors "errors"
 	"fmt"
+	"math/rand/v2"
 	"net"
 	"strings"
 	"syscall"
@@ -389,9 +390,17 @@ func isTransientConnectionError(err error) bool {
 	return false
 }
 
-// calculateBackoff calculates the delay for the given attempt with exponential backoff.
+// calculateBackoff returns a randomized exponential backoff delay for attempt,
+// never exceeding maxDelay.
 func calculateBackoff(attempt int, baseDelay, maxDelay time.Duration) time.Duration {
-	return min(time.Duration(1<<uint(attempt))*baseDelay, maxDelay)
+	delay := min(baseDelay<<attempt, maxDelay)
+	// Jitter is additive and clipped to the headroom left under maxDelay, so a
+	// retry never fires sooner than the exponential schedule alone would allow.
+	jitter := min(delay, maxDelay-delay)
+	if jitter <= 0 {
+		return delay
+	}
+	return delay + rand.N(jitter) // #nosec G404 -- no strong randomness required for retry jitter
 }
 
 func (d *Driver) Client(ctx context.Context, opts ...client.ClientOpt) (*client.Client, error) {

driver/kubernetes/driver_test.go

@@ -0,0 +1,41 @@
+package kubernetes
+
+import (
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestCalculateBackoff(t *testing.T) {
+	const (
+		baseDelay = 500 * time.Millisecond
+		maxDelay  = 10 * time.Second
+	)
+	tests := []struct {
+		attempt    int
+		floor, cap time.Duration
+	}{
+		{0, 500 * time.Millisecond, time.Second},
+		{1, time.Second, 2 * time.Second},
+		{2, 2 * time.Second, 4 * time.Second},
+		{3, 4 * time.Second, 8 * time.Second},
+		{4, 8 * time.Second, maxDelay},
+		{5, maxDelay, maxDelay},
+		{20, maxDelay, maxDelay},
+	}
+	for _, tt := range tests {
+		seen := make(map[time.Duration]struct{})
+		for range 500 {
+			got := calculateBackoff(tt.attempt, baseDelay, maxDelay)
+			require.GreaterOrEqual(t, got, tt.floor, "attempt %d must never wait less than the exponential schedule", tt.attempt)
+			require.LessOrEqual(t, got, tt.cap, "attempt %d must not exceed twice the schedule, nor maxDelay", tt.attempt)
+			seen[got] = struct{}{}
+		}
+		if tt.floor < tt.cap {
+			// The narrowest jittered range is 500ms wide, so a single distinct
+			// value across 500 draws would not be chance.
+			require.Greater(t, len(seen), 1, "attempt %d must vary so concurrent builders do not retry in lockstep", tt.attempt)
+		}
+	}
+}