Skip to content

Commit

Permalink
add option to deploy operator
Browse files Browse the repository at this point in the history
  • Loading branch information
@TheSpiritXIII
TheSpiritXIII committed Dec 27, 2023
1 parent 3202cac commit 429cf61
Show file tree
Hide file tree
Showing 2 changed files with 102 additions and 47 deletions.
46 changes: 24 additions & 22 deletions e2e/operator_context_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,6 @@ import (

"go.uber.org/zap/zapcore"
corev1 "k8s.io/api/core/v1"
rbacv1 "k8s.io/api/rbac/v1"
apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/api/meta"
Expand Down Expand Up @@ -74,6 +73,7 @@ var (
portForward bool
leakResources bool
cleanup bool
deployOperator bool
)

func init() {
Expand Down Expand Up @@ -127,6 +127,7 @@ func TestMain(m *testing.M) {
flag.BoolVar(&leakResources, "leak-resources", true, "If set, prevents deleting resources. Useful for debugging.")
flag.BoolVar(&cleanup, "cleanup-resources", true, "If set, cleans resources before running tests.")
flag.BoolVar(&cleanupOnly, "cleanup-resources-only", cleanupOnly, "If set, cleans resources and then exits.")
flag.BoolVar(&deployOperator, "deploy-operator", true, "If set, deploys the operator image in the manifest.")

flag.Parse()

Expand Down Expand Up @@ -348,6 +349,7 @@ func createBaseResources(t testing.TB, ctx context.Context, kubeClient client.Cl
operatorutil.WithLabel(testLabel, labelValue),
operatorutil.WithMeta(projectID, cluster, location),
operatorutil.WithPortForward(portForward),
operatorutil.WithDeployOperator(deployOperator),
); err != nil {
return err
}
Expand Down Expand Up @@ -399,28 +401,28 @@ func createGCPSecretResources(ctx context.Context, kubeClient client.Client, nam

func createCollectorResources(ctx context.Context, kubeClient client.Client, namespace, labelValue string) error {
// The cluster role expected to exist already.
const clusterRoleName = operator.DefaultOperatorNamespace + ":" + operator.NameCollector
// const clusterRoleName = operator.DefaultOperatorNamespace + ":" + operator.NameCollector

if err := kubeClient.Create(ctx, &rbacv1.ClusterRoleBinding{
ObjectMeta: metav1.ObjectMeta{
Name: clusterRoleName + ":" + namespace,
},
RoleRef: rbacv1.RoleRef{
APIGroup: "rbac.authorization.k8s.io",
Kind: "ClusterRole",
// The ClusterRole is expected to exist in the cluster already.
Name: clusterRoleName,
},
Subjects: []rbacv1.Subject{
{
Kind: "ServiceAccount",
Namespace: namespace,
Name: operator.NameCollector,
},
},
}); err != nil {
return err
}
// if err := kubeClient.Create(ctx, &rbacv1.ClusterRoleBinding{
// ObjectMeta: metav1.ObjectMeta{
// Name: clusterRoleName + ":" + namespace,
// },
// RoleRef: rbacv1.RoleRef{
// APIGroup: "rbac.authorization.k8s.io",
// Kind: "ClusterRole",
// // The ClusterRole is expected to exist in the cluster already.
// Name: clusterRoleName,
// },
// Subjects: []rbacv1.Subject{
// {
// Kind: "ServiceAccount",
// Namespace: namespace,
// Name: operator.NameCollector,
// },
// },
// }); err != nil {
// return err
// }
return nil
}

Expand Down
103 changes: 78 additions & 25 deletions e2e/operatorutil/deploy.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ import (
"path/filepath"
"strings"
"testing"
"time"

"github.com/GoogleCloudPlatform/prometheus-engine/e2e/kubeutil"
"github.com/GoogleCloudPlatform/prometheus-engine/pkg/operator"
Expand Down Expand Up @@ -113,6 +114,7 @@ type deployOptions struct {
projectID string
cluster string
location string
deployOperator bool
}

type DeployOption func(*deployOptions)
Expand Down Expand Up @@ -156,6 +158,12 @@ func WithPortForward(portForward bool) DeployOption {
}
}

func WithDeployOperator(deployOperator bool) DeployOption {
return func(do *deployOptions) {
do.deployOperator = deployOperator
}
}

func DeployOperator(t testing.TB, ctx context.Context, restConfig *rest.Config, kubeClient client.Client, deployOpts ...DeployOption) error {
opts := &deployOptions{}
for _, opt := range deployOpts {
Expand Down Expand Up @@ -185,7 +193,9 @@ func createResources(t testing.TB, ctx context.Context, restConfig *rest.Config,
// TODO(pintohutch): find a way to incorporate webhooks back into our kind tests.
// This is a workaround for now.
case *admissionregistrationv1.MutatingWebhookConfiguration, *admissionregistrationv1.ValidatingWebhookConfiguration:
continue
if !opts.deployOperator {
continue
}
case *corev1.Namespace:
if obj.GetName() == opts.operatorNamespace || obj.GetName() == opts.publicNamespace {
if obj.Labels == nil {
Expand All @@ -209,43 +219,62 @@ func createResources(t testing.TB, ctx context.Context, restConfig *rest.Config,
}
case *appsv1.Deployment:
if obj.GetName() == operator.NameOperator {
var httpClient *http.Client
if opts.portForward {
var err error
httpClient, err = kubeutil.PortForwardClient(t, restConfig, kubeClient)
if !opts.deployOperator {
var httpClient *http.Client
if opts.portForward {
var err error
httpClient, err = kubeutil.PortForwardClient(t, restConfig, kubeClient)
if err != nil {
return fmt.Errorf("creating HTTP client: %s", err)
}
}

op, err := operator.New(opts.logger, restConfig, operator.Options{
ProjectID: opts.projectID,
Cluster: opts.cluster,
Location: opts.location,
OperatorNamespace: opts.operatorNamespace,
PublicNamespace: opts.publicNamespace,
// Pick a random available port.
ListenAddr: ":0",
CollectorHTTPClient: httpClient,
})
if err != nil {
return fmt.Errorf("creating HTTP client: %s", err)
t.Fatalf("instantiating operator: %s", err)
}

go func() {
if err := op.Run(ctx, prometheus.NewRegistry()); err != nil {
// Since we aren't in the main test goroutine we cannot fail with Fatal here.
t.Errorf("running operator: %s", err)
}
}()
continue
}

op, err := operator.New(opts.logger, restConfig, operator.Options{
ProjectID: opts.projectID,
Cluster: opts.cluster,
Location: opts.location,
OperatorNamespace: opts.operatorNamespace,
PublicNamespace: opts.publicNamespace,
// Pick a random available port.
ListenAddr: ":0",
CollectorHTTPClient: httpClient,
})
container, err := kubeutil.DeploymentContainer(obj, "operator")
if err != nil {
t.Fatalf("instantiating operator: %s", err)
t.Fatalf("unable to find operator container: %s", err)
}

go func() {
if err := op.Run(ctx, prometheus.NewRegistry()); err != nil {
// Since we aren't in the main test goroutine we cannot fail with Fatal here.
t.Errorf("running operator: %s", err)
}
}()
continue
container.Args = append(container.Args, fmt.Sprintf("--project-id=%s", opts.projectID))
container.Args = append(container.Args, fmt.Sprintf("--location=%s", opts.location))
container.Args = append(container.Args, fmt.Sprintf("--cluster=%s", opts.cluster))
container.Args = append(container.Args, fmt.Sprintf("--operator-namespace=%s", opts.operatorNamespace))
container.Args = append(container.Args, fmt.Sprintf("--public-namespace=%s", opts.publicNamespace))
}
}

if err := kubeClient.Create(ctx, obj); err != nil {
return err
}
}
if opts.deployOperator {
t.Log("waiting for operator to be ready...")
if err := kubeutil.WaitForDeploymentReady(ctx, kubeClient, opts.operatorNamespace, operator.NameOperator); err != nil {
t.Fatalf("waiting for operator to be ready: %s", err)
}
time.Sleep(time.Minute * 1)
}
return err
}

Expand All @@ -272,6 +301,30 @@ func normalizeResource(obj client.Object, operatorNamespace, publicNamespace, la
"gmp-operator.gmp-system": "gmp-operator." + operatorNamespace,
}
switch obj := obj.(type) {
case *rbacv1.Role:
for i := range obj.Rules {
rule := &obj.Rules[i]
for j := range rule.ResourceNames {
resource := &rule.ResourceNames[j]
for prefix, result := range replacements {
if strings.HasPrefix(*resource, prefix) {
*resource = result + strings.TrimPrefix(*resource, prefix)
}
}
}
}
case *rbacv1.ClusterRole:
for i := range obj.Rules {
rule := &obj.Rules[i]
for j := range rule.ResourceNames {
resource := &rule.ResourceNames[j]
for prefix, result := range replacements {
if strings.HasPrefix(*resource, prefix) {
*resource = result + strings.TrimPrefix(*resource, prefix)
}
}
}
}
case *rbacv1.RoleBinding:
for i := range obj.Subjects {
subject := &obj.Subjects[i]
Expand Down

0 comments on commit 429cf61

Please sign in to comment.