Skip to content

Commit

Permalink
[e2e] Support to provide e2e testing configuration
Browse files Browse the repository at this point in the history
  • Loading branch information
eromanova committed Dec 10, 2024
1 parent 4822e0f commit bb3bffa
Show file tree
Hide file tree
Showing 18 changed files with 711 additions and 414 deletions.
6 changes: 5 additions & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -108,14 +108,18 @@ tidy:
test: generate-all fmt vet envtest tidy external-crd ## Run tests.
KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test $$(go list ./... | grep -v /e2e) -coverprofile cover.out

# E2E_CONFIG_B64 contains the configuration for e2e testing.
E2E_CONFIG_B64 ?= ""

# Utilize Kind or modify the e2e tests to load the image locally, enabling
# compatibility with other vendors.
.PHONY: test-e2e # Run the e2e tests using a Kind k8s instance as the management cluster.
test-e2e: cli-install
@if [ "$$GINKGO_LABEL_FILTER" ]; then \
ginkgo_label_flag="-ginkgo.label-filter=$$GINKGO_LABEL_FILTER"; \
fi; \
KIND_CLUSTER_NAME="hmc-test" KIND_VERSION=$(KIND_VERSION) go test ./test/e2e/ -v -ginkgo.v -ginkgo.timeout=3h -timeout=3h $$ginkgo_label_flag
KIND_CLUSTER_NAME="hmc-test" KIND_VERSION=$(KIND_VERSION) E2E_CONFIG_B64=$(E2E_CONFIG_B64) \
go test ./test/e2e/ -v -ginkgo.v -ginkgo.timeout=3h -timeout=3h $$ginkgo_label_flag

.PHONY: lint
lint: golangci-lint ## Run golangci-lint linter & yamllint
Expand Down
2 changes: 1 addition & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,6 @@ require (
github.com/fluxcd/pkg/apis/meta v1.7.0
github.com/fluxcd/pkg/runtime v0.50.0
github.com/fluxcd/source-controller/api v1.4.1
github.com/google/uuid v1.6.0
github.com/hashicorp/go-retryablehttp v0.7.7
github.com/onsi/ginkgo/v2 v2.22.0
github.com/onsi/gomega v1.36.0
Expand Down Expand Up @@ -101,6 +100,7 @@ require (
github.com/google/gofuzz v1.2.0 // indirect
github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db // indirect
github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
github.com/google/uuid v1.6.0 // indirect
github.com/gorilla/mux v1.8.1 // indirect
github.com/gorilla/websocket v1.5.3 // indirect
github.com/gosuri/uitable v0.0.4 // indirect
Expand Down
146 changes: 146 additions & 0 deletions test/e2e/config/config.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,146 @@
// Copyright 2024
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package config

import (
"encoding/base64"
"fmt"
"os"
"strings"

"github.com/google/uuid"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
"gopkg.in/yaml.v3"
)

type TestingProvider string

const (
envVarE2EConfig = "E2E_CONFIG_B64"

TestingProviderAWS TestingProvider = "aws"
TestingProviderAzure TestingProvider = "azure"
TestingProviderVsphere TestingProvider = "vsphere"
)

var (
Config TestingConfig

defaultConfig = map[TestingProvider][]ProviderTestingConfig{
TestingProviderAWS: {
{
Standalone: &ClusterTestingConfig{
Template: "aws-standalone-cp-0-0-3",
},
Hosted: &ClusterTestingConfig{
Template: "aws-hosted-cp-0-0-3",
},
},
},
TestingProviderAzure: {
{
Standalone: &ClusterTestingConfig{
Template: "azure-standalone-cp-0-0-3",
},
Hosted: &ClusterTestingConfig{
Template: "azure-hosted-cp-0-0-3",
},
},
},
TestingProviderVsphere: {
{
Standalone: &ClusterTestingConfig{
Template: "vsphere-standalone-cp-0-0-3",
},
Hosted: &ClusterTestingConfig{
Template: "vsphere-hosted-cp-0-0-3",
},
},
},
}
)

type TestingConfig = map[TestingProvider][]ProviderTestingConfig

type ProviderTestingConfig struct {
// Standalone contains the testing configuration for the standalone cluster deployment.
Standalone *ClusterTestingConfig `yaml:"standalone,omitempty"`
// Standalone contains the testing configuration for the hosted cluster deployment.
Hosted *ClusterTestingConfig `yaml:"hosted,omitempty"`
}

type ClusterTestingConfig struct {
// Name is the name of the managed cluster to deploy. Will be generated if unset.
Name string `yaml:"name,omitempty"`
// Upgrade is a boolean parameter that specifies whether the managed cluster upgrade should be tested.
Upgrade bool `yaml:"upgrade,omitempty"`
// Template is the name of the template to use when deploying a managed cluster.
// If unset:
// * The latest available template will be chosen
// * If upgrade is triggered, the latest available template with available upgrades will be chosen.
Template string `yaml:"template,omitempty"`
// UpgradeTemplate specifies the name of the template to upgrade to. Ignored if upgrade is set to false.
// If unset, the latest template available for the upgrade will be chosen.
UpgradeTemplate string `yaml:"upgradeTemplate,omitempty"`
}

func Parse() error {
decodedConfig, err := base64.StdEncoding.DecodeString(os.Getenv(envVarE2EConfig))
if err != nil {
return err
}
_, _ = fmt.Fprintf(GinkgoWriter, "E2e testing configuration:\n%s\n", decodedConfig)

if len(decodedConfig) > 0 {
err = yaml.Unmarshal(decodedConfig, &Config)
if err != nil {
return err
}
} else {
Config = defaultConfig
}

for provider, providerConfigs := range Config {
for _, providerConfig := range providerConfigs {
providerConfig.setDefaults(string(provider))
}
}
return nil
}

func generateClusterName(provider string, template string) string {
clusterType := "standalone"
if strings.Contains(template, "hosted") {
clusterType = "hosted"
}
return "e2e-test-" + provider + "-" + clusterType + "-" + uuid.New().String()[:8]
}

func (c *ProviderTestingConfig) setDefaults(provider string) {
if c.Standalone.Name == "" {
c.Standalone.Name = generateClusterName(provider, c.Standalone.Template)
}
if c.Hosted.Name == "" {
c.Hosted.Name = generateClusterName(provider, c.Hosted.Template)
}
}

func (c *ProviderTestingConfig) String() string {
prettyConfig, err := yaml.Marshal(c)
Expect(err).NotTo(HaveOccurred())

return string(prettyConfig)
}
97 changes: 10 additions & 87 deletions test/e2e/e2e_suite_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,26 +15,24 @@
package e2e

import (
"bufio"
"context"
"fmt"
"net/url"
"os"
"os/exec"
"path/filepath"
"strings"
"testing"
"time"

. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/utils/ptr"

internalutils "github.com/Mirantis/hmc/internal/utils"
"github.com/Mirantis/hmc/test/e2e/config"
"github.com/Mirantis/hmc/test/e2e/kubeclient"
"github.com/Mirantis/hmc/test/e2e/logs"
"github.com/Mirantis/hmc/test/e2e/managedcluster"
"github.com/Mirantis/hmc/test/e2e/templates"
"github.com/Mirantis/hmc/test/utils"
)

Expand All @@ -46,12 +44,15 @@ func TestE2E(t *testing.T) {
}

var _ = BeforeSuite(func() {
GinkgoT().Setenv(managedcluster.EnvVarNamespace, internalutils.DefaultSystemNamespace)
err := config.Parse()
Expect(err).NotTo(HaveOccurred())

GinkgoT().Setenv(managedcluster.EnvVarNamespace, internalutils.DefaultSystemNamespace)
By("building and deploying the controller-manager")
cmd := exec.Command("make", "kind-deploy")
_, err := utils.Run(cmd)
_, err = utils.Run(cmd)
Expect(err).NotTo(HaveOccurred())

cmd = exec.Command("make", "test-apply")
_, err = utils.Run(cmd)
Expect(err).NotTo(HaveOccurred())
Expand All @@ -72,7 +73,7 @@ var _ = AfterSuite(func() {
if !noCleanup() {
By("collecting logs from local controllers")
kc := kubeclient.NewFromLocal(internalutils.DefaultSystemNamespace)
collectLogArtifacts(kc, "")
logs.Collector{Client: kc}.CollectProvidersLogs()

By("removing the controller-manager")
cmd := exec.Command("make", "dev-destroy")
Expand Down Expand Up @@ -144,89 +145,11 @@ func validateController(kc *kubeclient.KubeClient, labelSelector, name string) e

// templateBy wraps a Ginkgo By with a block describing the template being
// tested.
func templateBy(t managedcluster.Template, description string) {
func templateBy(t templates.Type, description string) {
GinkgoHelper()
By(fmt.Sprintf("[%s] %s", t, description))
}

// collectLogArtifacts collects log output from each the HMC controller,
// CAPI controller and the provider controller(s) as well as output from clusterctl
// and stores them in the test/e2e directory as artifacts. clusterName can be
// optionally provided, passing an empty string will prevent clusterctl output
// from being fetched. If collectLogArtifacts fails it produces a warning
// message to the GinkgoWriter, but does not fail the test.
func collectLogArtifacts(kc *kubeclient.KubeClient, clusterName string, providerTypes ...managedcluster.ProviderType) {
GinkgoHelper()

filterLabels := []string{utils.HMCControllerLabel}

var host string
hostURL, err := url.Parse(kc.Config.Host)
if err != nil {
utils.WarnError(fmt.Errorf("failed to parse host from kubeconfig: %w", err))
} else {
host = strings.ReplaceAll(hostURL.Host, ":", "_")
}

if providerTypes == nil {
filterLabels = managedcluster.FilterAllProviders()
} else {
for _, providerType := range providerTypes {
filterLabels = append(filterLabels, managedcluster.GetProviderLabel(providerType))
}
}

for _, label := range filterLabels {
pods, _ := kc.Client.CoreV1().Pods(kc.Namespace).List(context.Background(), metav1.ListOptions{
LabelSelector: label,
})

for _, pod := range pods.Items {
req := kc.Client.CoreV1().Pods(kc.Namespace).GetLogs(pod.Name, &corev1.PodLogOptions{
TailLines: ptr.To(int64(1000)),
})
podLogs, err := req.Stream(context.Background())
if err != nil {
utils.WarnError(fmt.Errorf("failed to get log stream for pod %s: %w", pod.Name, err))
continue
}

output, err := os.Create(fmt.Sprintf("./test/e2e/%s.log", host+"-"+pod.Name))
if err != nil {
utils.WarnError(fmt.Errorf("failed to create log file for pod %s: %w", pod.Name, err))
continue
}

r := bufio.NewReader(podLogs)
_, err = r.WriteTo(output)
if err != nil {
utils.WarnError(fmt.Errorf("failed to write log file for pod %s: %w", pod.Name, err))
}

if err = podLogs.Close(); err != nil {
utils.WarnError(fmt.Errorf("failed to close log stream for pod %s: %w", pod.Name, err))
}
if err = output.Close(); err != nil {
utils.WarnError(fmt.Errorf("failed to close log file for pod %s: %w", pod.Name, err))
}
}
}

if clusterName != "" {
cmd := exec.Command("./bin/clusterctl",
"describe", "cluster", clusterName, "--namespace", internalutils.DefaultSystemNamespace, "--show-conditions=all")
output, err := utils.Run(cmd)
if err != nil {
utils.WarnError(fmt.Errorf("failed to get clusterctl log: %w", err))
return
}
err = os.WriteFile(filepath.Join("test/e2e", host+"-"+"clusterctl.log"), output, 0o644)
if err != nil {
utils.WarnError(fmt.Errorf("failed to write clusterctl log: %w", err))
}
}
}

func noCleanup() bool {
noCleanup := os.Getenv(managedcluster.EnvVarNoCleanup)
if noCleanup != "" {
Expand Down
Loading

0 comments on commit bb3bffa

Please sign in to comment.