Integration test enhancements

.github/workflows/tests.yaml

@@ -26,6 +26,13 @@ jobs:
   integration:
     name: Run integration tests
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        arg:
+          - '1.28'
+          - '1.29'
+          - '1.30'
+          - '1.31'
 
     steps:
       - name: Checkout repository
@@ -45,10 +52,10 @@ jobs:
           version: v0.20.0
           kubectl_version: v1.27.2
 
-      - name: Test
+      - name: Test with Kubernetes ${{ matrix.arg }}
         env:
-          MAX_PARALLELISM: 1 # Constrained to 2 CPUs on ubuntu-latest
-        run: make test-run-int-verbose
+          MAX_PARALLELISM: 2 # Constrained to 2 CPUs on ubuntu-latest
+        run: make test-run-int-verbose KUBE_VERSION=${{ matrix.arg }}
 
   helm:
     name: Run Helm tests

CHANGELOG.md

@@ -14,6 +14,8 @@
 ### Kubernetes Compatibility
 | Kube Version | Compatible? | `In-place Update of Pod Resources` Maturity |
 |:------------:|:-----------:|:-------------------------------------------:|
+|     1.31     |     ✔️      |                    Alpha                    |
+|     1.30     |     ✔️      |                    Alpha                    |
 |     1.29     |     ✔️      |                    Alpha                    |
 |     1.28     |     ✔️      |                    Alpha                    |
 |     1.27     |      ❌      |                    Alpha                    |
@@ -49,4 +51,3 @@
 |     1.29     |     ✔️      |                    Alpha                    |
 |     1.28     |     ✔️      |                    Alpha                    |
 |     1.27     |      ❌      |                    Alpha                    |
-

Makefile

@@ -34,20 +34,28 @@ test-run-unit: ## Runs unit tests
 	go test -count=1 ./internal/...
 
 .PHONY: test-run-int
-test-run-int: ## Runs integration tests
+test-run-int: ## Runs integration tests for a specific major.minor version of Kube
+	@if [ -z "${KUBE_VERSION}" ]; then \
+		echo "KUBE_VERSION is required - run 'make test-run-int KUBE_VERSION=x.y'"; \
+		exit 1; \
+	fi
 	go test -count=1 -timeout ${INT_TESTS_TIMEOUT} ./test/integration/...
 
 .PHONY: test-run-int-verbose
-test-run-int-verbose: ## Runs integration tests with verbose logging
+test-run-int-verbose: ## Runs integration tests for a specific major.minor version of Kube, with verbose logging
+	@if [ -z "${KUBE_VERSION}" ]; then \
+		echo "KUBE_VERSION is required - run 'make test-run-int KUBE_VERSION=x.y'"; \
+		exit 1; \
+	fi
 	go test -count=1 -timeout ${INT_TESTS_TIMEOUT} -v ./test/integration/...
 
 .PHONY: test-run-helm
 test-run-helm: ## Runs Helm tests
-	rm -rf ${HELM_TESTS_SNAPSHOT_DIR}
-	mkdir ${HELM_TESTS_SNAPSHOT_DIR}
-	chmod 777 ${HELM_TESTS_SNAPSHOT_DIR}
+	@rm -rf ${HELM_TESTS_SNAPSHOT_DIR}
+	@mkdir ${HELM_TESTS_SNAPSHOT_DIR}
+	@chmod 777 ${HELM_TESTS_SNAPSHOT_DIR}
 	docker run -t --rm -v ${ROOT_DIR}charts:/apps helmunittest/helm-unittest:3.12.3-0.3.5 container-startup-autoscaler
-	rm -rf ${HELM_TESTS_SNAPSHOT_DIR}
+	@rm -rf ${HELM_TESTS_SNAPSHOT_DIR}
 
 ## ------------------
 ## Go Modules

README.md

@@ -9,7 +9,7 @@ works with deployments, statefulsets, daemonsets and other workload management A
 CSA is implemented using [controller-runtime](https://github.com/kubernetes-sigs/controller-runtime).
 
 CSA is built around Kube's [In-place Update of Pod Resources](https://github.com/kubernetes/enhancements/tree/master/keps/sig-node/1287-in-place-update-pod-resources)
-feature, which is currently in alpha state as of Kubernetes 1.29 and therefore requires the `InPlacePodVerticalScaling`
+feature, which is currently in alpha state as of Kubernetes 1.31 and therefore requires the `InPlacePodVerticalScaling`
 feature gate to be enabled. Beta/stable targets are indicated [here](https://github.com/kubernetes/enhancements/issues/1287).
 The feature implementation (along with the corresponding implementation of CSA) is likely to change until it reaches
 stable status. See [CHANGELOG.md](CHANGELOG.md) for details of CSA versions and Kubernetes version compatibility.
@@ -517,14 +517,15 @@ execution might take some time to complete.
 
 A number of environment variable-based configuration options are available:
 
-| Name                     | Default | Description                                                           |
-|--------------------------|---------|-----------------------------------------------------------------------|
-| `MAX_PARALLELISM`        | `4`     | The maximum number of tests that can run in parallel.                 |
-| `REUSE_CLUSTER`          | `false` | Whether to reuse an existing CSA kind cluster (if it already exists). |
-| `INSTALL_METRICS_SERVER` | `false` | Whether to install metrics-server.                                    |
-| `KEEP_CSA`               | `false` | Whether to keep the CSA installation after tests finish.              |
-| `KEEP_CLUSTER`           | `false` | Whether to keep the CSA kind cluster after tests finish.              |
-| `DELETE_NS_AFTER_TEST`   | `true`  | Whether to delete namespaces created by tests after they conclude.    |
+| Name                     | Default | Description                                                                                                                          |
+|--------------------------|---------|--------------------------------------------------------------------------------------------------------------------------------------|
+| `KUBE_VERSION`           | -       | The _major.minor_ version of Kube to run tests against e.g. `1.31`.                                                                  |
+| `MAX_PARALLELISM`        | `4`     | The maximum number of tests that can run in parallel.                                                                                |
+| `REUSE_CLUSTER`          | `false` | Whether to reuse an existing CSA kind cluster (if it already exists). `KUBE_VERSION` has no effect if an existing cluster is reused. |
+| `INSTALL_METRICS_SERVER` | `false` | Whether to install metrics-server.                                                                                                   |
+| `KEEP_CSA`               | `false` | Whether to keep the CSA installation after tests finish.                                                                             |
+| `KEEP_CLUSTER`           | `false` | Whether to keep the CSA kind cluster after tests finish.                                                                             |
+| `DELETE_NS_AFTER_TEST`   | `true`  | Whether to delete namespaces created by tests after they conclude.                                                                   |
 
 Integration tests are executed in parallel due to their long-running nature. Each test operates within a separate Kube
 namespace (but using the same single CSA installation). If local resources are limited, reduce `MAX_PARALLELISM`
@@ -542,7 +543,7 @@ exist in parallel, if desired.
 ### Cluster/CSA Installation
 Executing `csa-install.sh`:
 - Removes any pre-existing CSA kind cluster.
-- Installs a CSA kind cluster.
+- Installs a CSA kind cluster with the latest version of Kubernetes [certified as compatible with CSA](CHANGELOG.md).
 - Creates a new, separate CSA kind cluster kubeconfig file under `$HOME/.kube/`.
 - Pulls metrics-server, loads the image into the CSA kind cluster and installs.
 - Pulls echo-server and loads the image into the CSA kind cluster.

scripts/sandbox/config/vars.sh

@@ -22,11 +22,11 @@ arch=$(uname -m)
 case $arch in
   x86_64)
     # shellcheck disable=SC2034
-    kind_image="kindest/node:v1.29.0@sha256:54a50c9354f11ce0aa56a85d2cacb1b950f85eab3fe1caf988826d1f89bf37eb"
+    kind_image="kindest/node:v1.31.0@sha256:919a65376fd11b67df05caa2e60802ad5de2fca250c9fe0c55b0dce5c9591af3"
     ;;
   arm64)
     # shellcheck disable=SC2034
-    kind_image="kindest/node:v1.29.0@sha256:8ccbd8bc4d52c467f3c79eeeb434827c225600a1d7385a4b1c19d9e038c9e0c0"
+    kind_image="kindest/node:v1.31.0@sha256:0ccfb11dc66eae4abc20c30ee95687bab51de8aeb04e325e1c49af0890646548"
     ;;
   *)
     echo "Error: architecture '$arch' not supported"

test/integration/command.go

@@ -17,22 +17,22 @@ limitations under the License.
 package integration
 
 import (
-	"fmt"
 	"os"
 	"os/exec"
 	"strings"
+	"testing"
 
 	"github.com/ExpediaGroup/container-startup-autoscaler/internal/common"
 )
 
-func cmdRun(cmd *exec.Cmd, info string, coreErrMsg string, fatalOnErr bool, suppressInfo ...bool) (string, error) {
+func cmdRun(t *testing.T, cmd *exec.Cmd, info string, coreErrMsg string, fatalOnErr bool, suppressInfo ...bool) (string, error) {
 	suppress := false
 	if len(suppressInfo) > 0 && suppressInfo[0] {
 		suppress = true
 	}
 
 	if info != "" && !suppress {
-		fmt.Println(info)
+		logMessage(t, info)
 	}
 
 	combinedOutput, err := cmd.CombinedOutput()
@@ -41,7 +41,7 @@ func cmdRun(cmd *exec.Cmd, info string, coreErrMsg string, fatalOnErr bool, supp
 		wrappedErr := common.WrapErrorf(err, "%s (output: %s)", coreErrMsg, trimmedOutput)
 
 		if fatalOnErr {
-			fmt.Println(wrappedErr)
+			logMessage(t, wrappedErr)
 			os.Exit(1)
 		}
 		return trimmedOutput, wrappedErr

test/integration/config/kind.yaml

@@ -16,3 +16,10 @@ kind: Cluster
 apiVersion: kind.x-k8s.io/v1alpha4
 featureGates:
   InPlacePodVerticalScaling: true
+nodes:
+  - role: control-plane
+    kubeadmConfigPatches: # TODO(wt) temporary to work around https://github.com/kubernetes/kubernetes/issues/112264
+      - |
+        apiVersion: kubelet.config.k8s.io/v1beta1
+        kind: KubeletConfiguration
+        syncFrequency: "3s"

test/integration/consts.go

@@ -0,0 +1,112 @@
+/*
+Copyright 2024 Expedia Group, Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package integration
+
+import "os"
+
+// Path ----------------------------------------------------------------------------------------------------------------
+const (
+	pathSeparator        = string(os.PathSeparator)
+	pathIntTestRelPath   = "test" + pathSeparator + "integration"
+	pathConfigDirRelPath = pathIntTestRelPath + pathSeparator + "config"
+)
+
+// kind ----------------------------------------------------------------------------------------------------------------
+const (
+	kindClusterName       = "csa-int-cluster"
+	kindConfigFileRelPath = pathConfigDirRelPath + pathSeparator + "kind.yaml"
+)
+
+var k8sVersionToImage = map[string]map[string]string{
+	"1.31": {
+		"amd64": "kindest/node:v1.31.0@sha256:919a65376fd11b67df05caa2e60802ad5de2fca250c9fe0c55b0dce5c9591af3",
+		"arm64": "kindest/node:v1.31.0@sha256:0ccfb11dc66eae4abc20c30ee95687bab51de8aeb04e325e1c49af0890646548",
+	},
+	"1.30": {
+		"amd64": "kindest/node:v1.30.4@sha256:34cb98a38a57a3357fde925a41d61232bbbbeb411b45a25c0d766635d6c3b975",
+		"arm64": "kindest/node:v1.30.4@sha256:6becd630a18e77730e31f3833f0b129bbcc9c09ee49c3b88429b3c1fdc30bfc4",
+	},
+	"1.29": {
+		"amd64": "kindest/node:v1.29.8@sha256:b69a150f9951ef41158ec76de381a920df2be3582fd16fc19cf4757eef0dded9",
+		"arm64": "kindest/node:v1.29.8@sha256:0d5623800cf6290edbc1007ca8a33a5f7e2ad92b41dc7022b4d20a66447db23c",
+	},
+	"1.28": {
+		"amd64": "kindest/node:v1.28.13@sha256:d97df9fff48099bf9a94c92fdc39adde65bec2aa1d011f84233b96172c1003c9",
+		"arm64": "kindest/node:v1.28.13@sha256:ddef612bb93a9aa3a989f9d3d4e01c0a7c4d866a4b949264146c182cd202d738",
+	},
+}
+
+// metrics-server ------------------------------------------------------------------------------------------------------
+const (
+	metricsServerImageTag            = "registry.k8s.io/metrics-server/metrics-server:v0.6.4"
+	metricsServerKustomizeDirRelPath = pathConfigDirRelPath + pathSeparator + "metricsserver"
+	metricsServerReadyTimeout        = "60s"
+)
+
+// CSA -----------------------------------------------------------------------------------------------------------------
+const (
+	csaDockerImage    = "csa"
+	csaDockerTag      = "test"
+	csaDockerImageTag = csaDockerImage + ":" + csaDockerTag
+)
+
+const (
+	csaHelmChartRelPath = "charts" + pathSeparator + "container-startup-autoscaler"
+	csaHelmName         = "csa-int"
+	csaHelmTimeout      = "60s"
+)
+
+const (
+	csaStatusWaitMillis                            = 1000
+	csaStatusMessageStartupCommanded               = "Startup resources commanded"
+	csaStatusMessageStartupCommandedUnknownRes     = "Startup resources commanded (unknown resources applied)"
+	csaStatusMessagePostStartupCommanded           = "Post-startup resources commanded"
+	csaStatusMessagePostStartupCommandedUnknownRes = "Post-startup resources commanded (unknown resources applied)"
+	csaStatusMessageStartupEnacted                 = "Startup resources enacted"
+	csaStatusMessagePostStartupEnacted             = "Post-startup resources enacted"
+	csaStatusMessageValidationError                = "Validation error"
+)
+
+const (
+	csaEventReasonScaling    = "Scaling"
+	csaEventReasonValidation = "Validation"
+)
+
+// echo-server ---------------------------------------------------------------------------------------------------------
+const (
+	echoServerDockerImageTag = "ealen/echo-server:0.7.0"
+	echoServerName           = "echo-server"
+)
+
+const (
+	echoServerNonTargetContainerName           = echoServerName + "-non-target"
+	echoServerNonTargetContainerCpuRequests    = "50m"
+	echoServerNonTargetContainerCpuLimits      = "50m"
+	echoServerNonTargetContainerMemoryRequests = "150M"
+	echoServerNonTargetContainerMemoryLimits   = "150M"
+)
+
+const (
+	echoServerDefaultProbeInitialDelaySeconds = 15
+	echoServerProbePeriodSeconds              = 1
+	echoServerProbeFailureThreshold           = echoServerDefaultProbeInitialDelaySeconds
+)
+
+// Tests ---------------------------------------------------------------------------------------------------------------
+const (
+	testsDefaultWaitStatusTimeoutSecs = echoServerDefaultProbeInitialDelaySeconds * 2
+)