diff --git a/proxy/healthy_endpoints_test.go b/proxy/healthy_endpoints_test.go index 1fa6694761..a8a8cd98f8 100644 --- a/proxy/healthy_endpoints_test.go +++ b/proxy/healthy_endpoints_test.go @@ -101,6 +101,17 @@ func setupServices(t *testing.T, healthy, unhealthy int) []string { return services } +func formatServicesString(services []string) string { + backends := "" + for i, service := range services { + backends += fmt.Sprintf(`"%s"`, service) + if i < len(services)-1 { + backends += ", " + } + } + return backends +} + func TestPHCWithoutRequests(t *testing.T) { services := setupServices(t, 3, 0) @@ -150,142 +161,84 @@ func TestPHCForSingleHealthyEndpoint(t *testing.T) { assert.Equal(t, 0, failedReqs) } -func TestPHCForMultipleHealthyEndpoints(t *testing.T) { - services := setupServices(t, 3, 0) - - for _, algorithm := range []string{"random", "consistentHash", "roundRobin", "powerOfRandomNChoices"} { - t.Run(algorithm, func(t *testing.T) { - _, ps := setupProxy(t, fmt.Sprintf(`* -> consistentHashKey("${request.header.ConsistentHashKey}") -> <%s, "%s", "%s", "%s">`, - algorithm, services[0], services[1], services[2])) - va := fireVegeta(t, ps, 3000, 1*time.Second, 5*time.Second) - count200, ok := va.CountStatus(200) - assert.True(t, ok) - assert.InDelta(t, count200, 15000, 50) // 3000*5s, the delta is for CI - assert.Equal(t, count200, int(va.TotalRequests())) - }) - } - - t.Run("consistent hash with balance factor", func(t *testing.T) { - _, ps := setupProxy(t, fmt.Sprintf(`* -> consistentHashKey("${request.header.ConsistentHashKey}") -> consistentHashBalanceFactor(1.25) -> `, - services[0], services[1], services[2])) - va := fireVegeta(t, ps, 3000, 1*time.Second, 5*time.Second) - count200, ok := va.CountStatus(200) - assert.True(t, ok) - assert.Equal(t, count200, int(va.TotalRequests())) - }) -} - -func TestPHCForMultipleHealthyAndOneUnhealthyEndpoints(t *testing.T) { - services := setupServices(t, 2, 1) - for _, algorithm := range []string{"random", "roundRobin", "powerOfRandomNChoices"} { - t.Run(algorithm, func(t *testing.T) { - mockMetrics, ps := setupProxy(t, fmt.Sprintf(`* -> backendTimeout("5ms") -> consistentHashKey("${request.header.ConsistentHashKey}") -> <%s, "%s", "%s", "%s">`, - algorithm, services[0], services[1], services[2])) - - va := fireVegeta(t, ps, 3000, 1*time.Second, 5*time.Second) - - count200, ok := va.CountStatus(200) - assert.True(t, ok) - - count504, ok := va.CountStatus(504) - assert.True(t, ok) - - failedRequests := math.Abs(float64(va.TotalRequests())) - float64(count200) - t.Logf("total requests: %d, count200: %d, count504: %d, failedRequests: %f", va.TotalRequests(), count200, count504, failedRequests) - - assert.InDelta(t, float64(count504), failedRequests, 5) - assert.InDelta(t, 0, float64(failedRequests), 0.1*float64(va.TotalRequests())) - mockMetrics.WithCounters(func(counters map[string]int64) { - assert.InDelta(t, float64(va.TotalRequests()), float64(counters["passive-health-check.endpoints.dropped"]), 0.3*float64(va.TotalRequests())) // allow 30% error +func TestPHC(t *testing.T) { + for _, tt := range []struct { + name string + healthy, unhealthy int + }{ + {"single healthy", 1, 0}, + {"multiple healthy", 2, 0}, + {"multiple healthy and one unhealthy", 2, 1}, + {"multiple healthy and multiple unhealthy", 2, 2}, + } { + t.Run(tt.name, func(t *testing.T) { + services := setupServices(t, tt.healthy, tt.unhealthy) + servicesString := formatServicesString(services) + for _, algorithm := range []string{"random", "roundRobin", "powerOfRandomNChoices"} { + t.Run(algorithm, func(t *testing.T) { + mockMetrics, ps := setupProxy(t, fmt.Sprintf(`* -> backendTimeout("5ms") -> consistentHashKey("${request.header.ConsistentHashKey}") -> <%s, %s>`, + algorithm, servicesString)) + + va := fireVegeta(t, ps, 3000, 1*time.Second, 5*time.Second) + + count200, ok := va.CountStatus(200) + assert.True(t, ok) + + count504, ok := va.CountStatus(504) + assert.Condition(t, func() bool { + if tt.unhealthy == 0 && (ok || count504 == 0) { + return true + } else if tt.unhealthy > 0 && ok { + return true + } else { + return false + } + }) + + failedRequests := math.Abs(float64(va.TotalRequests())) - float64(count200) + t.Logf("total requests: %d, count200: %d, count504: %d, failedRequests: %f", va.TotalRequests(), count200, count504, failedRequests) + + assert.InDelta(t, float64(count504), failedRequests, 5) + assert.InDelta(t, 0, float64(failedRequests), 0.3*float64(va.TotalRequests())) + mockMetrics.WithCounters(func(counters map[string]int64) { + assert.InDelta(t, float64(tt.unhealthy)*float64(va.TotalRequests()), float64(counters["passive-health-check.endpoints.dropped"]), 0.6*float64(va.TotalRequests())) + }) + }) + } + + t.Run("consistentHash", func(t *testing.T) { + endpointRegistry := routing.NewEndpointRegistry(routing.RegistryOptions{ + PassiveHealthCheckEnabled: true, + StatsResetPeriod: 1 * time.Second, + MinRequests: 1, // with 2 test case fails on github actions + MaxHealthCheckDropProbability: 0.95, + MinHealthCheckDropProbability: 0.01, + }) + mockMetrics, ps := setupProxyWithCustomEndpointRegisty(t, fmt.Sprintf(`* -> backendTimeout("5ms") -> consistentHashKey("${request.header.ConsistentHashKey}") -> `, + servicesString), endpointRegistry) + failedReqs := sendGetRequests(t, ps) + assert.InDelta(t, 0, failedReqs, 0.2*float64(nRequests)) + mockMetrics.WithCounters(func(counters map[string]int64) { + assert.InDelta(t, float64(tt.unhealthy*nRequests), float64(counters["passive-health-check.endpoints.dropped"]), 0.3*float64(tt.unhealthy*nRequests)) + }) }) - }) - } - t.Run("consistentHash", func(t *testing.T) { - endpointRegistry := routing.NewEndpointRegistry(routing.RegistryOptions{ - PassiveHealthCheckEnabled: true, - StatsResetPeriod: 1 * time.Second, - MinRequests: 1, // with 2 test case fails on github actions - MaxHealthCheckDropProbability: 0.95, - MinHealthCheckDropProbability: 0.01, - }) - mockMetrics, ps := setupProxyWithCustomEndpointRegisty(t, fmt.Sprintf(`* -> backendTimeout("5ms") -> consistentHashKey("${request.header.ConsistentHashKey}") -> `, - services[0], services[1], services[2]), endpointRegistry) - failedReqs := sendGetRequests(t, ps) - assert.InDelta(t, 0, failedReqs, 0.1*float64(nRequests)) - mockMetrics.WithCounters(func(counters map[string]int64) { - assert.InDelta(t, float64(nRequests), float64(counters["passive-health-check.endpoints.dropped"]), 0.3*float64(nRequests)) // allow 30% error - }) - }) - - t.Run("consistent hash with balance factor", func(t *testing.T) { - mockMetrics, ps := setupProxy(t, fmt.Sprintf(`* -> backendTimeout("5ms") -> consistentHashKey("${request.header.ConsistentHashKey}") -> consistentHashBalanceFactor(1.25) -> `, - services[0], services[1], services[2])) - failedReqs := sendGetRequests(t, ps) - assert.InDelta(t, 0, failedReqs, 0.1*float64(nRequests)) - mockMetrics.WithCounters(func(counters map[string]int64) { - assert.InDelta(t, float64(nRequests), float64(counters["passive-health-check.endpoints.dropped"]), 0.3*float64(nRequests)) // allow 30% error - }) - }) -} - -func TestPHCForMultipleHealthyAndMultipleUnhealthyEndpoints(t *testing.T) { - services := setupServices(t, 2, 2) - for _, algorithm := range []string{"random", "roundRobin", "powerOfRandomNChoices"} { - t.Run(algorithm, func(t *testing.T) { - mockMetrics, ps := setupProxy(t, fmt.Sprintf(`* -> backendTimeout("5ms") -> consistentHashKey("${request.header.ConsistentHashKey}") -> <%s, "%s", "%s", "%s", "%s">`, - algorithm, services[0], services[1], services[2], services[3])) - - va := fireVegeta(t, ps, 3000, 1*time.Second, 5*time.Second) - - count200, ok := va.CountStatus(200) - assert.True(t, ok) - - count504, ok := va.CountStatus(504) - assert.True(t, ok) - - failedRequests := math.Abs(float64(va.TotalRequests())) - float64(count200) - t.Logf("total requests: %d, count200: %d, count504: %d, failedRequests: %f", va.TotalRequests(), count200, count504, failedRequests) - - assert.InDelta(t, float64(count504), failedRequests, 5) - assert.InDelta(t, 0, float64(failedRequests), 0.3*float64(va.TotalRequests())) - mockMetrics.WithCounters(func(counters map[string]int64) { - assert.InDelta(t, 2*float64(va.TotalRequests()), float64(counters["passive-health-check.endpoints.dropped"]), 0.6*float64(va.TotalRequests())) + t.Run("consistent hash with balance factor", func(t *testing.T) { + endpointRegistery := routing.NewEndpointRegistry(routing.RegistryOptions{ + PassiveHealthCheckEnabled: true, + StatsResetPeriod: 1 * time.Second, + MinRequests: 1, // with 2 test case fails on github actions + MaxHealthCheckDropProbability: 0.95, + MinHealthCheckDropProbability: 0.01, + }) + mockMetrics, ps := setupProxyWithCustomEndpointRegisty(t, fmt.Sprintf(`* -> backendTimeout("5ms") -> consistentHashKey("${request.header.ConsistentHashKey}") -> consistentHashBalanceFactor(1.25) -> `, + servicesString), endpointRegistery) + failedReqs := sendGetRequests(t, ps) + assert.InDelta(t, 0, failedReqs, 0.2*float64(nRequests)) + mockMetrics.WithCounters(func(counters map[string]int64) { + assert.InDelta(t, float64(tt.unhealthy*nRequests), float64(counters["passive-health-check.endpoints.dropped"]), 0.3*float64(nRequests)*float64(tt.unhealthy)) + }) }) }) } - - t.Run("consistentHash", func(t *testing.T) { - endpointRegistry := routing.NewEndpointRegistry(routing.RegistryOptions{ - PassiveHealthCheckEnabled: true, - StatsResetPeriod: 1 * time.Second, - MinRequests: 1, // with 2 test case fails on github actions and -race - MaxHealthCheckDropProbability: 0.95, - MinHealthCheckDropProbability: 0.01, - }) - mockMetrics, ps := setupProxyWithCustomEndpointRegisty(t, fmt.Sprintf(`* -> backendTimeout("5ms") -> consistentHashKey("${request.header.ConsistentHashKey}") -> `, - services[0], services[1], services[2], services[3]), endpointRegistry) - failedReqs := sendGetRequests(t, ps) - assert.InDelta(t, 0, failedReqs, 0.2*float64(nRequests)) - mockMetrics.WithCounters(func(counters map[string]int64) { - assert.InDelta(t, 2*float64(nRequests), float64(counters["passive-health-check.endpoints.dropped"]), 0.6*float64(nRequests)) - }) - }) - - t.Run("consistent hash with balance factor", func(t *testing.T) { - endpointRegistry := routing.NewEndpointRegistry(routing.RegistryOptions{ - PassiveHealthCheckEnabled: true, - StatsResetPeriod: 1 * time.Second, - MinRequests: 1, // with 2 test case fails on github actions and -race - MaxHealthCheckDropProbability: 0.95, - MinHealthCheckDropProbability: 0.01, - }) - mockMetrics, ps := setupProxyWithCustomEndpointRegisty(t, fmt.Sprintf(`* -> backendTimeout("5ms") -> consistentHashKey("${request.header.ConsistentHashKey}") -> consistentHashBalanceFactor(1.25) -> `, - services[0], services[1], services[2], services[3]), endpointRegistry) - failedReqs := sendGetRequests(t, ps) - assert.InDelta(t, 0, failedReqs, 0.2*float64(nRequests)) - mockMetrics.WithCounters(func(counters map[string]int64) { - assert.InDelta(t, 2*float64(nRequests), float64(counters["passive-health-check.endpoints.dropped"]), 0.6*float64(nRequests)) - }) - }) }