From e8997090c78a2415f11752899d897a85580d8cb8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Carlos=20Ch=C3=A1vez?= Date: Sun, 13 Nov 2022 13:59:09 +0100 Subject: [PATCH 1/3] wip --- e2e/istio/cluster.yaml | 8 + e2e/istio/httpbin.yaml | 47 ++++++ e2e/istio/ingressgateway.yaml | 30 ++++ e2e/istio/patch-ingressgateway-nodeport.yaml | 9 ++ e2e/istio/wasmplugin.yaml | 14 ++ magefile.go | 151 ++++++++++++++++++- 6 files changed, 258 insertions(+), 1 deletion(-) create mode 100644 e2e/istio/cluster.yaml create mode 100644 e2e/istio/httpbin.yaml create mode 100644 e2e/istio/ingressgateway.yaml create mode 100644 e2e/istio/patch-ingressgateway-nodeport.yaml create mode 100644 e2e/istio/wasmplugin.yaml diff --git a/e2e/istio/cluster.yaml b/e2e/istio/cluster.yaml new file mode 100644 index 0000000..42e9bcc --- /dev/null +++ b/e2e/istio/cluster.yaml @@ -0,0 +1,8 @@ +# https://www.arthurkoziel.com/running-knative-with-istio-in-kind/ +kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 +nodes: + - role: control-plane + extraPortMappings: + - containerPort: 32000 + hostPort: 80 diff --git a/e2e/istio/httpbin.yaml b/e2e/istio/httpbin.yaml new file mode 100644 index 0000000..c83f7db --- /dev/null +++ b/e2e/istio/httpbin.yaml @@ -0,0 +1,47 @@ +apiVersion: v1 +kind: Service +metadata: + name: httpbin + labels: + app: httpbin + service: httpbin +spec: + ports: + - port: 9080 + name: http + selector: + app: httpbin +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: httpbin-sa + labels: + account: httpbin +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: httpbin-v1 + labels: + app: httpbin + version: v1 +spec: + replicas: 1 + selector: + matchLabels: + app: httpbin + version: v1 + template: + metadata: + labels: + app: httpbin + version: v1 + spec: + serviceAccountName: httpbin-sa + containers: + - name: httpbin + image: kennethreitz/httpbin:latest + imagePullPolicy: IfNotPresent + ports: + - containerPort: 9080 diff --git a/e2e/istio/ingressgateway.yaml b/e2e/istio/ingressgateway.yaml new file mode 100644 index 0000000..8e5d628 --- /dev/null +++ b/e2e/istio/ingressgateway.yaml @@ -0,0 +1,30 @@ +apiVersion: networking.istio.io/v1alpha3 +kind: Gateway +metadata: + name: httpbin-gateway +spec: + selector: + istio: ingressgateway # use istio default controller + servers: + - port: + number: 80 + name: http + protocol: HTTP + hosts: + - "*" +--- +apiVersion: networking.istio.io/v1alpha3 +kind: VirtualService +metadata: + name: httpbin +spec: + hosts: + - "*" + gateways: + - httpbin-gateway + http: + - route: + - destination: + host: httpbin + port: + number: 9080 diff --git a/e2e/istio/patch-ingressgateway-nodeport.yaml b/e2e/istio/patch-ingressgateway-nodeport.yaml new file mode 100644 index 0000000..cf7aac3 --- /dev/null +++ b/e2e/istio/patch-ingressgateway-nodeport.yaml @@ -0,0 +1,9 @@ +# https://www.arthurkoziel.com/running-knative-with-istio-in-kind/ +spec: + type: NodePort + ports: + - name: http2 + nodePort: 32000 + port: 80 + protocol: TCP + targetPort: 80 diff --git a/e2e/istio/wasmplugin.yaml b/e2e/istio/wasmplugin.yaml new file mode 100644 index 0000000..07b5c81 --- /dev/null +++ b/e2e/istio/wasmplugin.yaml @@ -0,0 +1,14 @@ +apiVersion: extensions.istio.io/v1alpha1 +kind: WasmPlugin +metadata: + name: coraza-waf +spec: + selector: + matchLabels: + istio: ingressgateway + url: oci://corazawaf/coraza-proxy-wasm:latest + imagePullPolicy: IfNotPresent + phase: AUTHN + pluginConfig: + rules: + - "Include crs/*.conf" diff --git a/magefile.go b/magefile.go index 86c2f8c..54772bc 100644 --- a/magefile.go +++ b/magefile.go @@ -15,6 +15,7 @@ import ( "regexp" "strconv" "strings" + "time" "github.com/magefile/mage/mg" "github.com/magefile/mage/sh" @@ -222,11 +223,159 @@ func UpdateLibs() error { return nil } -// E2e runs e2e tests with a built plugin against the example deployment. Requires docker-compose. func E2e() error { + mg.SerialDeps(E2eEnvoy, E2eIstio) + return nil +} + +// E2e runs e2e tests with a built plugin against the example deployment. Requires docker-compose. +func E2eEnvoy() error { return sh.RunV("docker-compose", "-f", "e2e/docker-compose.yml", "up", "--abort-on-container-exit", "tests") } +func runK8sApply(file string, replacementKV ...string) error { + fmt.Printf("Applying %q\n", file) + if len(replacementKV) == 0 { + return sh.RunV("kubectl", "apply", "-f", file) + } + + if len(replacementKV)%2 != 0 { + return errors.New("missing value for a replacement pair") + } + + manifest, err := os.ReadFile(file) + if err != nil { + return err + } + + patchedManifest := string(manifest) + for i := 0; i < len(replacementKV)/2; i++ { + patchedManifest = strings.Replace(patchedManifest, replacementKV[2*i], replacementKV[2*i+1], 1) + } + + f, err := os.CreateTemp("", filepath.Base(file)) + if err != nil { + return err + } + f.Write([]byte(patchedManifest)) + defer os.Remove(f.Name()) + + if err := sh.RunV("kubectl", "apply", "-f", f.Name()); err != nil { + return err + } + + return nil +} + +// References +// - https://kind.sigs.k8s.io/docs/user/loadbalancer/ +func E2eIstio() error { + const ( + clusterName = "coraza-proxy-wasm-e2e" + ) + + var ( + kind = "kind" + //istioCTL = "/Users/jcchavezs/.getmesh/bin/getmesh istioctl" + //kubeCTL = "kubectl" + dockerImage = fmt.Sprintf("corazawaf/coraza-proxy-wasm:%d", time.Now().Unix()) + ) + + if clusters, err := sh.Output(kind, "get", "clusters"); err != nil { + return err + } else if !strings.Contains(clusters, clusterName) { + err := sh.RunV(kind, "create", "cluster", "--name", clusterName, "--config", "./e2e/istio/cluster.yaml") + if err != nil { + return err + } + } + //defer sh.RunV("kind", "delete", "cluster", "--name", clusterName) + + if err := sh.RunV("/Users/jcchavezs/.getmesh/bin/getmesh", "istioctl", "install", "--set", "profile=demo", "-y"); err != nil { + return err + } + + if err := sh.Run("kubectl", "label", "namespace", "default", "istio-injection=enabled", "--overwrite=true"); err != nil { + return err + } + /* + if err := runK8sApply("https://raw.githubusercontent.com/metallb/metallb/v0.13.7/config/manifests/metallb-native.yaml"); err != nil { + return err + } + + sh.RunV(kubeCTL, "wait", "--namespace", "metallb-system", + "--for=condition=ready", "pod", + "--selector=app=metallb", + "--timeout=90s", + ) + + cidr, err := sh.Output("docker", "network", "inspect", "-f", "'{{.IPAM.Config}}'", "kind") + if err != nil { + return err + } + + fmt.Println(cidr) + + if err := runK8sApply( + "./e2e/istio/metallb-config.yaml", + "${IP_START}", "172.19.255.200", + "${IP_END}", "172.19.255.250", + ); err != nil { + return err + } + */ + + if patch, err := os.ReadFile("./e2e/istio/patch-ingressgateway-nodeport.yaml"); err == nil { + if err := sh.RunV("kubectl", "patch", "service", "istio-ingressgateway", "-n", "istio-system", "--patch", string(patch)); err != nil { + return err + } + } else { + return err + } + + if err := sh.Run("docker", "build", "-t", dockerImage, "."); err != nil { + return err + } + + if err := sh.RunV(kind, "load", "docker-image", "kennethreitz/httpbin:latest", "--name", clusterName); err != nil { + return err + } + + if err := sh.RunV(kind, "load", "docker-image", dockerImage, "--name", clusterName); err != nil { + return err + } + + imageName, version, _ := strings.Cut(dockerImage, ":") + fmt.Printf("Waiting for %q to be loaded in control plane\n", dockerImage) + for { + images, err := sh.Output("docker", "exec", "-it", clusterName+"-control-plane", "crictl", "images") + if err != nil { + return err + } + + if strings.Contains(images, imageName) && strings.Contains(images, version) { + break + } + } + + /*if err := runK8sApply("./e2e/istio/wasmplugin.yaml", "${IMAGE}", dockerImage); err != nil { + return err + } + + if err := runK8sApply("./e2e/istio/service.yaml"); err != nil { + return err + } + + ip, err := sh.Output(kubeCTL, "get", "svc/foo-service", "-o=jsonpath='{.status.loadBalancer.ingress[0].ip}'") + if err != nil { + return err + } + + fmt.Println(ip) + */ + return nil +} + // Ftw runs ftw tests with a built plugin and Envoy. Requires docker-compose. func Ftw() error { if err := sh.RunV("docker-compose", "--file", "ftw/docker-compose.yml", "build", "--pull"); err != nil { From 25bb40ccc7476c10342d9a995e4505f27f886c87 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Carlos=20Ch=C3=A1vez?= Date: Mon, 14 Nov 2022 21:54:45 +0100 Subject: [PATCH 2/3] chore: reenables apply of wasm ext. --- magefile.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/magefile.go b/magefile.go index 54772bc..30dbb35 100644 --- a/magefile.go +++ b/magefile.go @@ -358,11 +358,11 @@ func E2eIstio() error { } } - /*if err := runK8sApply("./e2e/istio/wasmplugin.yaml", "${IMAGE}", dockerImage); err != nil { + if err := runK8sApply("./e2e/istio/wasmplugin.yaml", "${IMAGE}", dockerImage); err != nil { return err } - if err := runK8sApply("./e2e/istio/service.yaml"); err != nil { + /*if err := runK8sApply("./e2e/istio/service.yaml"); err != nil { return err } From a3c2fc81610fbc192b4d2900501e1c69a2f9f459 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Carlos=20Ch=C3=A1vez?= Date: Mon, 14 Nov 2022 21:56:00 +0100 Subject: [PATCH 3/3] chore: replaces image in wasmplugin. --- e2e/istio/wasmplugin.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e2e/istio/wasmplugin.yaml b/e2e/istio/wasmplugin.yaml index 07b5c81..1f66f39 100644 --- a/e2e/istio/wasmplugin.yaml +++ b/e2e/istio/wasmplugin.yaml @@ -6,7 +6,7 @@ spec: selector: matchLabels: istio: ingressgateway - url: oci://corazawaf/coraza-proxy-wasm:latest + url: oci://${IMAGE} imagePullPolicy: IfNotPresent phase: AUTHN pluginConfig: