Skip to content

Commit

Permalink
deploy
Browse files Browse the repository at this point in the history
  • Loading branch information
@TheSpiritXIII
TheSpiritXIII committed Dec 27, 2023
1 parent 2e0ea01 commit 3202cac
Show file tree
Hide file tree
Showing 4 changed files with 302 additions and 141 deletions.
2 changes: 1 addition & 1 deletion e2e/kubeutil/network.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -107,7 +107,7 @@ func PortForwardClient(t testing.TB, restConfig *rest.Config, kubeClient client.
if err != nil {
return nil, fmt.Errorf("unable to get pod from IP %s: %w", addr.IP, err)
}
if err := WaitForPodContainerReady(ctx, t, restConfig, kubeClient, pod, container); err != nil {
if err := WaitForPodContainerReady(ctx, restConfig, kubeClient, pod, container); err != nil {
return nil, fmt.Errorf("failed waiting for pod from IP %s: %w", addr.IP, err)
}
resourceURL := restClient.
Expand Down
8 changes: 4 additions & 4 deletions e2e/kubeutil/pod.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,13 +48,13 @@ func IsPodContainerReady(ctx context.Context, restConfig *rest.Config, pod *core
return fmt.Errorf("no container named %s found in pod %s", container, key)
}

func WaitForPodContainerReady(ctx context.Context, t testing.TB, restConfig *rest.Config, kubeClient client.Client, pod *corev1.Pod, container string) error {
func WaitForPodContainerReady(ctx context.Context, restConfig *rest.Config, kubeClient client.Client, pod *corev1.Pod, container string) error {
return waitForResourceReady(ctx, kubeClient, pod, 1*time.Minute, func(pod *corev1.Pod) error {
return IsPodContainerReady(ctx, restConfig, pod, container)
})
}

func IsPodReady(ctx context.Context, restConfig *rest.Config, pod *corev1.Pod) error {
func IsPodReady(ctx context.Context, pod *corev1.Pod) error {
var errs []error
for _, status := range pod.Status.ContainerStatuses {
if !status.Ready {
Expand All @@ -65,9 +65,9 @@ func IsPodReady(ctx context.Context, restConfig *rest.Config, pod *corev1.Pod) e
return errors.Join(errs...)
}

func WaitForPodReady(ctx context.Context, t *testing.T, restConfig *rest.Config, kubeClient client.Client, pod *corev1.Pod) error {
func WaitForPodReady(ctx context.Context, t *testing.T, kubeClient client.Client, pod *corev1.Pod) error {
return waitForResourceReady(ctx, kubeClient, pod, 30*time.Second, func(pod *corev1.Pod) error {
return IsPodReady(ctx, restConfig, pod)
return IsPodReady(ctx, pod)
})
}

Expand Down
131 changes: 49 additions & 82 deletions e2e/operator_context_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,17 +23,14 @@ import (
"errors"
"flag"
"fmt"
"net/http"
"os"
"os/signal"
"strings"
"syscall"
"testing"
"time"

"github.com/prometheus/client_golang/prometheus"
"go.uber.org/zap/zapcore"
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
rbacv1 "k8s.io/api/rbac/v1"
apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
Expand Down Expand Up @@ -249,39 +246,39 @@ func newOperatorContext(t *testing.T) *OperatorContext {
cancel()
})

if err := createBaseResources(ctx, tctx.Client(), namespace, pubNamespace, userNamespace, tctx.GetOperatorTestLabelValue()); err != nil {
if err := createBaseResources(tctx, ctx, tctx.Client(), namespace, pubNamespace, userNamespace, tctx.GetOperatorTestLabelValue()); err != nil {
t.Fatalf("create resources: %s", err)
}

var httpClient *http.Client
if portForward {
var err error
httpClient, err = kubeutil.PortForwardClient(t, kubeconfig, tctx.Client())
if err != nil {
t.Fatalf("creating HTTP client: %s", err)
}
}

op, err := operator.New(globalLogger, kubeconfig, operator.Options{
ProjectID: projectID,
Cluster: cluster,
Location: location,
OperatorNamespace: tctx.namespace,
PublicNamespace: tctx.pubNamespace,
// Pick a random available port.
ListenAddr: ":0",
CollectorHTTPClient: httpClient,
})
if err != nil {
t.Fatalf("instantiating operator: %s", err)
}

go func() {
if err := op.Run(ctx, prometheus.NewRegistry()); err != nil {
// Since we aren't in the main test goroutine we cannot fail with Fatal here.
t.Errorf("running operator: %s", err)
}
}()
// var httpClient *http.Client
// if portForward {
// var err error
// httpClient, err = kubeutil.PortForwardClient(t, kubeconfig, tctx.Client())
// if err != nil {
// t.Fatalf("creating HTTP client: %s", err)
// }
// }

// op, err := operator.New(globalLogger, kubeconfig, operator.Options{
// ProjectID: projectID,
// Cluster: cluster,
// Location: location,
// OperatorNamespace: tctx.namespace,
// PublicNamespace: tctx.pubNamespace,
// // Pick a random available port.
// ListenAddr: ":0",
// CollectorHTTPClient: httpClient,
// })
// if err != nil {
// t.Fatalf("instantiating operator: %s", err)
// }

// go func() {
// if err := op.Run(ctx, prometheus.NewRegistry()); err != nil {
// // Since we aren't in the main test goroutine we cannot fail with Fatal here.
// t.Errorf("running operator: %s", err)
// }
// }()

return tctx
}
Expand Down Expand Up @@ -333,7 +330,7 @@ func (tctx *OperatorContext) GetOperatorTestLabelValue() string {
// createBaseResources creates resources the operator requires to exist already.
// These are resources which don't depend on runtime state and can thus be deployed
// statically, allowing to run the operator without critical write permissions.
func createBaseResources(ctx context.Context, kubeClient client.Client, opNamespace, publicNamespace, userNamespace, labelValue string) error {
func createBaseResources(t testing.TB, ctx context.Context, kubeClient client.Client, opNamespace, publicNamespace, userNamespace, labelValue string) error {
if err := createNamespaces(ctx, kubeClient, opNamespace, publicNamespace, userNamespace); err != nil {
return err
}
Expand All @@ -344,31 +341,34 @@ func createBaseResources(ctx context.Context, kubeClient client.Client, opNamesp
if err := createCollectorResources(ctx, kubeClient, opNamespace, labelValue); err != nil {
return err
}
if err := operatorutil.DeployOperator(ctx, kubeClient,
if err := operatorutil.DeployOperator(t, ctx, kubeconfig, kubeClient,
operatorutil.WithOperatorNamespace(opNamespace),
operatorutil.WithPublicNamespace(publicNamespace),
operatorutil.WithUserNamespace(userNamespace),
operatorutil.WithLabel(testLabel, labelValue),
operatorutil.WithMeta(projectID, cluster, location),
operatorutil.WithPortForward(portForward),
); err != nil {
return err
}
return nil
}

func createNamespaces(ctx context.Context, kubeClient client.Client, opNamespace, publicNamespace, userNamespace string) error {
if err := kubeClient.Create(ctx, &corev1.Namespace{
ObjectMeta: metav1.ObjectMeta{
Name: opNamespace,
},
}); err != nil {
return err
}
if err := kubeClient.Create(ctx, &corev1.Namespace{
ObjectMeta: metav1.ObjectMeta{
Name: publicNamespace,
},
}); err != nil {
return err
}
// if err := kubeClient.Create(ctx, &corev1.Namespace{
// ObjectMeta: metav1.ObjectMeta{
// Name: opNamespace,
// },
// }); err != nil {
// return err
// }
// if err := kubeClient.Create(ctx, &corev1.Namespace{
// ObjectMeta: metav1.ObjectMeta{
// Name: publicNamespace,
// },
// }); err != nil {
// return err
// }
return kubeClient.Create(ctx, &corev1.Namespace{
ObjectMeta: metav1.ObjectMeta{
Name: userNamespace,
Expand Down Expand Up @@ -398,15 +398,6 @@ func createGCPSecretResources(ctx context.Context, kubeClient client.Client, nam
}

func createCollectorResources(ctx context.Context, kubeClient client.Client, namespace, labelValue string) error {
if err := kubeClient.Create(ctx, &corev1.ServiceAccount{
ObjectMeta: metav1.ObjectMeta{
Name: operator.NameCollector,
Namespace: namespace,
},
}); err != nil {
return err
}

// The cluster role expected to exist already.
const clusterRoleName = operator.DefaultOperatorNamespace + ":" + operator.NameCollector

Expand All @@ -430,30 +421,6 @@ func createCollectorResources(ctx context.Context, kubeClient client.Client, nam
}); err != nil {
return err
}

obj, err := kubeutil.ResourceFromFile(kubeClient.Scheme(), collectorManifest)
if err != nil {
return fmt.Errorf("decode collector: %w", err)
}
collector := obj.(*appsv1.DaemonSet)
collector.Namespace = namespace
if collector.Spec.Template.Labels == nil {
collector.Spec.Template.Labels = map[string]string{}
}
collector.Spec.Template.Labels[testLabel] = labelValue
if skipGCM {
for i := range collector.Spec.Template.Spec.Containers {
container := &collector.Spec.Template.Spec.Containers[i]
if container.Name == "prometheus" {
container.Args = append(container.Args, "--export.debug.disable-auth")
break
}
}
}

if err = kubeClient.Create(ctx, collector); err != nil {
return fmt.Errorf("create collector DaemonSet: %w", err)
}
return nil
}

Expand Down
Loading

0 comments on commit 3202cac

Please sign in to comment.