diff --git a/.github/generate-reports.sh b/.github/generate-reports.sh new file mode 100755 index 00000000..e06501ac --- /dev/null +++ b/.github/generate-reports.sh @@ -0,0 +1,69 @@ +#!/bin/bash + +# Copyright 2020 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail +set -x + +(cd "$(dirname "$0")/../test/e2e/images/reports" && make build-image) + +REPORT_BUILDER_IMAGE=local/reports-builder:0.0.1 + +REPORTS_DIR=${REPORTS_DIR:-/tmp/bfe-ingress-reports} + +INGRESS_CONTROLLER="BFE-ingress-controller" +CONTROLLER_VERSION=${CONTROLLER_VERSION:-'N/A'} + +TEMP_CONTENT=$(mktemp -d) + +docker run \ + -e BUILD="$(date -u +'%Y-%m-%dT%H:%M:%SZ')" \ + -e INPUT_DIRECTORY=/input \ + -e OUTPUT_DIRECTORY=/output \ + -e INGRESS_CONTROLLER="${INGRESS_CONTROLLER}" \ + -e CONTROLLER_VERSION="${CONTROLLER_VERSION}" \ + -v "${REPORTS_DIR}":/input:ro \ + -v "${TEMP_CONTENT}":/output \ + -u "$(id -u):$(id -g)" \ + "${REPORT_BUILDER_IMAGE}" + +pushd "${TEMP_WORKTREE}" > /dev/null + +if [[ -d ./e2e-test ]]; then + git rm -r ./e2e-test +else + mkdir -p "${TEMP_WORKTREE}/e2e-test" +fi + +# copy new content +cp -r -a "${TEMP_CONTENT}/." "${TEMP_WORKTREE}/e2e-test/" + +# cleanup HTML +sudo apt-get install tidy +for html_file in e2e-test/*.html;do + tidy -q --break-before-br no --tidy-mark no --show-warnings no --wrap 0 -indent -m "$html_file" || true +done + +# configure git +git config --global user.email "action@github.com" +git config --global user.name "GitHub Action" +# commit changes +git add e2e-test +git commit -m "e2e test report" +git push --force --quiet + +popd > /dev/null diff --git a/.github/workflows/e2e-test.yml b/.github/workflows/e2e-test.yml new file mode 100644 index 00000000..330bd9cb --- /dev/null +++ b/.github/workflows/e2e-test.yml @@ -0,0 +1,87 @@ +# Copyright 2022 The BFE Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# This is a basic workflow to help you get started with Actions + +name: e2e-test + +# Controls when the action will run. Triggers the workflow on push or pull request +# events but only for the master branch +on: + push: + branches: [ main, develop ] + pull_request: + branches: [ develop ] + +jobs: + e2e-test: + runs-on: ubuntu-latest + environment: e2e + + env: + CUCUMBER_FEATURE: ${{ secrets.CUCUMBER_FEATURE }} + WAIT_FOR_STATUS_TIMEOUT: ${{ secrets.WAIT_FOR_STATUS_TIMEOUT }} + TEST_TIMEOUT: ${{ secrets.TEST_TIMEOUT }} + + steps: + - name: Checkout + uses: actions/checkout@v2 + + - uses: dorny/paths-filter@v2 + id: filter + with: + token: ${{ secrets.GITHUB_TOKEN }} + filters: | + go: + - '**/*.go' + - 'go.mod' + - 'go.sum' + - 'Makefile' + - 'Dockerfile' + - 'test/**/*' + + - name: Run e2e test + id: run-e2e-test + continue-on-error: true + shell: bash + if: steps.filter.outputs.go == 'true' + run: | + export RESULTS_DIR=/tmp/bfe-ingress-reports + export CUCUMBER_OUTPUT_FORMAT=cucumber + make e2e-test + + - name: Generate reports + continue-on-error: true + if: steps.filter.outputs.go == 'true' + run: | + # clone the gh-pages repository branch + export TEMP_WORKTREE=$(mktemp -d) + remote_repo="https://${GITHUB_ACTOR}:${{ secrets.GITHUB_TOKEN }}@github.com/${GITHUB_REPOSITORY}.git" + git clone --branch=gh-pages --depth=1 "${remote_repo}" "${TEMP_WORKTREE}" + + export REPORTS_DIR=/tmp/bfe-ingress-reports + export CONTROLLER_VERSION=$(cat VERSION) + .github/generate-reports.sh + + - name: Upload cucumber json files + uses: actions/upload-artifact@v2 + continue-on-error: true + with: + name: cucumber-output + path: /tmp/bfe-ingress-reports/* + + - name: Check on failures + if: steps.filter.outputs.go == 'true' && steps.run-e2e-test.outcome != 'success' + run: exit 1 + diff --git a/.github/workflows/helm.yaml b/.github/workflows/helm.yaml index de05675d..b56aba10 100644 --- a/.github/workflows/helm.yaml +++ b/.github/workflows/helm.yaml @@ -1,3 +1,17 @@ +# Copyright 2022 The BFE Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# name: Helm on: diff --git a/.licenserc.yaml b/.licenserc.yaml new file mode 100644 index 00000000..f627864f --- /dev/null +++ b/.licenserc.yaml @@ -0,0 +1,37 @@ +# Copyright (c) 2022 The BFE Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +header: + license: + spdx-id: Apache-2.0 + copyright-owner: The BFE Authors + + paths-ignore: + - '.idea' + - 'conf' + - 'docs/images' + - '**/go.mod' + - '**/go.sum' + - '**/*.md' + - '**/*.orig' + - 'examples/*' + - 'LICENSE' + - 'VERSION' + - 'test/e2e/*' + + comment: on-failure + + dependency: + files: + - go.mod diff --git a/CHANGELOG.md b/CHANGELOG.md index 38ef06cf..2dd5c811 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,17 @@ All notable changes to this project will be documented in this file. +## v0.3.0 (2021-03-12) + +### Enhancements: +- Record k8s event for status change. [#47](https://github.com/bfenetworks/ingress-bfe/pull/47) +- E2E test support. [#57](https://github.com/bfenetworks/ingress-bfe/pull/57), [#60](https://github.com/bfenetworks/ingress-bfe/pull/60) +- Optimize startup logic. [#40](https://github.com/bfenetworks/ingress-bfe/pull/40), [#61](https://github.com/bfenetworks/ingress-bfe/pull/61) +- Docs improvement. [#38](https://github.com/bfenetworks/ingress-bfe/pull/38), [#39](https://github.com/bfenetworks/ingress-bfe/pull/39), [#41](https://github.com/bfenetworks/ingress-bfe/pull/41), [#42](https://github.com/bfenetworks/ingress-bfe/pull/42), [#52](https://github.com/bfenetworks/ingress-bfe/pull/52), [#56](https://github.com/bfenetworks/ingress-bfe/pull/56), [#58](https://github.com/bfenetworks/ingress-bfe/pull/58), [#59](https://github.com/bfenetworks/ingress-bfe/pull/59) +- License related. [#50](https://github.com/bfenetworks/ingress-bfe/pull/50), [#51](https://github.com/bfenetworks/ingress-bfe/pull/51) + + + ## v0.2.2 (2021-11-17) ### Enhancements: diff --git a/Dockerfile b/Dockerfile index 4e953ccb..eb9823b7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -24,4 +24,4 @@ COPY --from=build /bfe-ingress-controller/output/* / EXPOSE 8080 8443 8421 -ENTRYPOINT ["/start.sh", "/bfe-ingress-controller", "/bfe/bin/bfe"] +ENTRYPOINT ["/bfe-ingress-controller"] diff --git a/Makefile b/Makefile index 2fc158bd..887a1535 100644 --- a/Makefile +++ b/Makefile @@ -30,6 +30,7 @@ GOGEN := $(GO) generate GOCLEAN := $(GO) clean GOFLAGS := -race STATICCHECK := staticcheck +LICENSEEYE := license-eye # init arch ARCH := $(shell getconf LONG_BIT) @@ -70,6 +71,18 @@ check: $(GO) get honnef.co/go/tools/cmd/staticcheck $(STATICCHECK) ./... +# make license-eye-install +license-eye-install: + $(GO) install github.com/apache/skywalking-eyes/cmd/license-eye@latest + +# make license-check, check code file's license declaration +license-check: license-eye-install + $(LICENSEEYE) header check + +# make license-fix, fix code file's license declaration +license-fix: license-eye-install + $(LICENSEEYE) header fix + # make docker docker: docker build \ @@ -82,5 +95,18 @@ clean: $(GOCLEAN) rm -rf $(OUTDIR) +# e2e test + +kind-cluster: + test/script/kind-create-cluster.sh + +test-env: docker kind-cluster + test/script/kind-load-images.sh $(INGRESS_VERSION) + test/script/deploy-controller.sh $(INGRESS_VERSION) + +e2e-test: test-env + test/e2e/run.sh + test/script/kind-delete-cluster.sh + # avoid filename conflict and speed up build -.PHONY: all compile test clean build +.PHONY: all compile test clean build docker e2e-test test-env kind-cluster diff --git a/README-CN.md b/README-CN.md index ad969bc3..240f2c34 100644 --- a/README-CN.md +++ b/README-CN.md @@ -4,9 +4,17 @@ ## 简介 -BFE Ingress Controller 为基于 [BFE][] 实现的[Kubernetes Ingress Controller][]，用于支持在 Kubernetes 中使用 [Ingress][]。 +BFE Ingress Controller 为基于 [BFE][] 实现的[Kubernetes Ingress Controller][]，用于支持在 Kubernetes 中使用 [Ingress][] 进行流量接入，并利用BFE的众多优秀特点和强大能力。 + +## 特性和优势 + +- 路由转发：支持基于Host、Path、Cookie、Header的路由规则 +- 多服务间负载均衡：支持在提供相同服务的多个Service之间进行负载均衡 +- 灵活的模块框架：采用灵活的模块框架设计，支持高效率定制开发扩展功能 +- 配置热加载：支持配置热加载，配置的更新和生效不会影响已存在的长连接 ## 开始使用 + 详见[部署指南](docs/zh_cn/deployment.md) ## 说明文档 @@ -33,7 +41,7 @@ BFE Ingress Controller 为基于 [BFE][] 实现的[Kubernetes Ingress Controller - +
diff --git a/README.md b/README.md index 48eafa01..5d64bcd0 100644 --- a/README.md +++ b/README.md @@ -4,9 +4,17 @@ English | [中文](README-CN.md) ## Overview -BFE Ingress Controller is an implementation of Kubernetes [Ingress Controller][] based on [BFE][], to fulfill [Ingress][] in Kubernetes. +BFE Ingress Controller is an implementation of Kubernetes [Ingress Controller][] based on [BFE][], to fulfill [Ingress][] in Kubernetes. + +## Features and Advantages + +- Traffic routing based on Host, Path, Cookie and Header +- Support for load balancing among multiple Services of the same application +- Flexible plugin framework, based on which developers can add new features efficiently +- Configuration hot reload, avoiding impact on existing long connections ## Quick start + See [Deployment](docs/en_us/deployment.md) for quick start of using BFE Ingress Controller. ## Documentation diff --git a/VERSION b/VERSION index ee1372d3..0d91a54c 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.2.2 +0.3.0 diff --git a/build/build.sh b/build/build.sh index da22d42c..b09cea89 100755 --- a/build/build.sh +++ b/build/build.sh @@ -1,4 +1,3 @@ -#!/bin/sh # Copyright 2021 The BFE Authors # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/charts/bfe-ingress-controller/.helmignore b/charts/bfe-ingress-controller/.helmignore index 0e8a0eb3..e46d1e53 100644 --- a/charts/bfe-ingress-controller/.helmignore +++ b/charts/bfe-ingress-controller/.helmignore @@ -1,3 +1,17 @@ +# Copyright 2022 The BFE Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# # Patterns to ignore when building packages. # This supports shell glob matching, relative path matching, and # negation (prefixed with !). Only one pattern per line. diff --git a/charts/bfe-ingress-controller/Chart.yaml b/charts/bfe-ingress-controller/Chart.yaml index 7cd70b33..8dad6ee6 100644 --- a/charts/bfe-ingress-controller/Chart.yaml +++ b/charts/bfe-ingress-controller/Chart.yaml @@ -1,3 +1,17 @@ +# Copyright 2022 The BFE Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# apiVersion: v2 name: bfe-ingress-controller description: Helm chart for BFE Ingress Controller diff --git a/charts/bfe-ingress-controller/templates/_helpers.tpl b/charts/bfe-ingress-controller/templates/_helpers.tpl index 9fd5a56d..d6c2f714 100644 --- a/charts/bfe-ingress-controller/templates/_helpers.tpl +++ b/charts/bfe-ingress-controller/templates/_helpers.tpl @@ -1,3 +1,19 @@ +{* + Copyright 2022 The BFE Authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +*} {{/* Expand the name of the chart. */}} diff --git a/charts/bfe-ingress-controller/templates/deployment.yaml b/charts/bfe-ingress-controller/templates/deployment.yaml index ebbfa5d0..ee8f2691 100644 --- a/charts/bfe-ingress-controller/templates/deployment.yaml +++ b/charts/bfe-ingress-controller/templates/deployment.yaml @@ -1,3 +1,17 @@ +# Copyright 2022 The BFE Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# apiVersion: apps/v1 kind: Deployment metadata: diff --git a/charts/bfe-ingress-controller/templates/hpa.yaml b/charts/bfe-ingress-controller/templates/hpa.yaml index 5448f0e2..1e67b73a 100644 --- a/charts/bfe-ingress-controller/templates/hpa.yaml +++ b/charts/bfe-ingress-controller/templates/hpa.yaml @@ -1,3 +1,17 @@ +# Copyright 2022 The BFE Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# {{- if .Values.autoscaling.enabled }} apiVersion: autoscaling/v2beta1 kind: HorizontalPodAutoscaler diff --git a/charts/bfe-ingress-controller/templates/rbac.yaml b/charts/bfe-ingress-controller/templates/rbac.yaml index cf61069b..4f06cc3c 100644 --- a/charts/bfe-ingress-controller/templates/rbac.yaml +++ b/charts/bfe-ingress-controller/templates/rbac.yaml @@ -1,3 +1,17 @@ +# Copyright 2022 The BFE Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# {{- if .Values.rbac.create }} apiVersion: rbac.authorization.k8s.io/v1 @@ -19,6 +33,13 @@ rules: - get - list - watch + - apiGroups: + - "" + resources: + - events + verbs: + - create + - patch - apiGroups: - extensions - networking.k8s.io diff --git a/charts/bfe-ingress-controller/templates/service.yaml b/charts/bfe-ingress-controller/templates/service.yaml index f0e9c969..d2a03f51 100644 --- a/charts/bfe-ingress-controller/templates/service.yaml +++ b/charts/bfe-ingress-controller/templates/service.yaml @@ -1,3 +1,17 @@ +# Copyright 2022 The BFE Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# apiVersion: v1 kind: Service metadata: diff --git a/charts/bfe-ingress-controller/templates/serviceaccount.yaml b/charts/bfe-ingress-controller/templates/serviceaccount.yaml index 5fe44224..9b761f07 100644 --- a/charts/bfe-ingress-controller/templates/serviceaccount.yaml +++ b/charts/bfe-ingress-controller/templates/serviceaccount.yaml @@ -1,3 +1,17 @@ +# Copyright 2022 The BFE Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# {{- if .Values.serviceAccount.create -}} apiVersion: v1 kind: ServiceAccount diff --git a/charts/bfe-ingress-controller/values.yaml b/charts/bfe-ingress-controller/values.yaml index 80626d11..4ef53622 100644 --- a/charts/bfe-ingress-controller/values.yaml +++ b/charts/bfe-ingress-controller/values.yaml @@ -1,3 +1,17 @@ +# Copyright 2022 The BFE Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# # Default values for bfe-ingress-controller. replicaCount: 1 diff --git a/cmd/ingress-controller/flags.go b/cmd/ingress-controller/flags.go index 10ef35b4..b2293100 100644 --- a/cmd/ingress-controller/flags.go +++ b/cmd/ingress-controller/flags.go @@ -17,21 +17,14 @@ package main import ( "flag" - corev1 "k8s.io/api/core/v1" - "github.com/bfenetworks/ingress-bfe/internal/option" ) var ( - help bool - showVersion bool - namespaces string - ingressClass string - configPath string - reloadAddr string - metricsAddr string - probeAddr string - defaultBackend string + help bool + showVersion bool + + opts *option.Options = option.NewOptions() ) func initFlags() { @@ -41,15 +34,19 @@ func initFlags() { flag.BoolVar(&showVersion, "version", false, "Show version of bfe-ingress-controller.") flag.BoolVar(&showVersion, "v", false, "Show version of bfe-ingress-controller.") - flag.StringVar(&namespaces, "namespace", corev1.NamespaceAll, "Namespaces to watch, delimited by ','.") - flag.StringVar(&namespaces, "n", corev1.NamespaceAll, "Namespaces to watch, delimited by ','.") + flag.StringVar(&opts.Namespaces, "namespace", opts.Namespaces, "Namespaces to watch, delimited by ','.") + flag.StringVar(&opts.Namespaces, "n", opts.Namespaces, "Namespaces to watch, delimited by ','.") + + flag.StringVar(&opts.MetricsAddr, "metrics-bind-address", opts.MetricsAddr, "The address the metric endpoint binds to.") + flag.StringVar(&opts.HealthProbeAddr, "health-probe-bind-address", opts.HealthProbeAddr, "The address the probe endpoint binds to.") + flag.StringVar(&opts.ClusterName, "k8s-cluster-name", opts.ClusterName, "k8s cluster name") - flag.StringVar(&configPath, "bfe-config-path", option.ConfigPath, "Root directory of bfe configuration files.") - flag.StringVar(&configPath, "c", option.ConfigPath, "Root directory of bfe configuration files.") + flag.StringVar(&opts.Ingress.ConfigPath, "bfe-config-path", opts.Ingress.ConfigPath, "Root directory of bfe configuration files.") + flag.StringVar(&opts.Ingress.ConfigPath, "c", opts.Ingress.ConfigPath, "Root directory of bfe configuration files.") + flag.StringVar(&opts.Ingress.BfeBinary, "bfe-binary", opts.Ingress.BfeBinary, "Absolute path of BFE binary. If set, is overwritten by /../conf") + flag.StringVar(&opts.Ingress.BfeBinary, "b", opts.Ingress.BfeBinary, "Absolute path of BFE binary. If set, is overwritten by /../conf,") + flag.StringVar(&opts.Ingress.ReloadAddr, "bfe-reload-address", opts.Ingress.ReloadAddr, "Address of bfe config reloading.") + flag.StringVar(&opts.Ingress.IngressClass, "ingress-class", opts.Ingress.IngressClass, "Class name of bfe ingress controller.") + flag.StringVar(&opts.Ingress.DefaultBackend, "default-backend", opts.Ingress.DefaultBackend, "set default backend name, default backend is used if no any ingress rule matched, format namespace/name.") - flag.StringVar(&reloadAddr, "bfe-reload-address", option.ReloadAddr, "Address of bfe config reloading.") - flag.StringVar(&ingressClass, "ingress-class", option.IngressClassName, "Class name of bfe ingress controller.") - flag.StringVar(&metricsAddr, "metrics-bind-address", option.MetricsBindAddress, "The address the metric endpoint binds to.") - flag.StringVar(&probeAddr, "health-probe-bind-address", option.HealthProbeBindAddress, "The address the probe endpoint binds to.") - flag.StringVar(&defaultBackend, "default-backend", option.DefaultBackend, "set default backend name, default backend is used if no any ingress rule matched, format namespace/name.") } diff --git a/cmd/ingress-controller/main.go b/cmd/ingress-controller/main.go index e20174de..04081888 100644 --- a/cmd/ingress-controller/main.go +++ b/cmd/ingress-controller/main.go @@ -25,7 +25,6 @@ import ( ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/log/zap" - "github.com/bfenetworks/ingress-bfe/internal/bfeConfig" "github.com/bfenetworks/ingress-bfe/internal/controllers" "github.com/bfenetworks/ingress-bfe/internal/option" ) @@ -46,12 +45,12 @@ var ( ) func main() { - opts := zap.Options{ + zapOpts := zap.Options{ Development: true, } - opts.BindFlags(flag.CommandLine) + zapOpts.BindFlags(flag.CommandLine) flag.Parse() - ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts))) + ctrl.SetLogger(zap.New(zap.UseFlagOptions(&zapOpts))) if help { flag.PrintDefaults() @@ -64,9 +63,7 @@ func main() { return } - err := option.SetOptions( - namespaces, ingressClass, configPath, reloadAddr, - metricsAddr, probeAddr, defaultBackend) + err := option.SetOptions(opts) if err != nil { setupLog.Error(err, "fail to start controllers") return @@ -74,10 +71,7 @@ func main() { setupLog.Info("starting bfe-ingress-controller") - configBuilder := bfeConfig.NewConfigBuilder() - configBuilder.InitReload() - - if err := controllers.Start(scheme, configBuilder); err != nil { + if err := controllers.Start(scheme); err != nil { setupLog.Error(err, "fail to start controllers") } diff --git a/docs/en_us/contribute/contribute-codes.md b/docs/en_us/contribute/contribute-codes.md index 0c403388..2120526e 100644 --- a/docs/en_us/contribute/contribute-codes.md +++ b/docs/en_us/contribute/contribute-codes.md @@ -48,15 +48,28 @@ Below tutorial will guide you to submit code $ pre-commit install ``` 2. use `gofmt` to format code. + + +5. Use `license-eye` tool + + [license-eye](http://github.com/apache/skywalking-eyes) helps us check and fix file's license header declaration. All files' license header should be done before committing. + + The `license-eye` check is part of the Github-Action. A PR that check failed cannot be submitted to BFE. Install `license-eye` and do check or fix: + + ```bash + $ make license-eye-install + $ make license-check + $ make license-fix + ``` -5. Coding +6. Coding -6. Build and test +7. Build and test Compile source code, build BFE Ingress Controller docker and then test it. See more instruction in [Deploy Guide](../deployment.md) -7. Commit +8. Commit run `git commit` . @@ -72,7 +85,7 @@ See more instruction in [Deploy Guide](../deployment.md) $ git commit -m "test=release/1.1" ``` -8. Keep local repository up-to-date +9. Keep local repository up-to-date An experienced Git user always pulls from the official repo before pushing. They even pull daily or hourly, so they notice conflicts earlier, and it's easier to resolve smaller conflicts. @@ -82,9 +95,9 @@ They even pull daily or hourly, so they notice conflicts earlier, and it's easie git pull upstream develop ``` -9. Push to remote repository +10. Push to remote repository - Push local to your repository on GitHub `https://github.com/${USERNAME}/ingress-bfe` + Push local to your repository on GitHub `https://github.com/${USERNAME}/ingress-bfe` ```bash # Example: push to remote repository `origin` branch `my-cool-stuff` diff --git a/docs/en_us/deployment.md b/docs/en_us/deployment.md index b63015db..42629270 100644 --- a/docs/en_us/deployment.md +++ b/docs/en_us/deployment.md @@ -35,5 +35,11 @@ kubectl apply -f https://raw.githubusercontent.com/bfenetworks/ingress-bfe/devel ``` shell script kubectl apply -f https://raw.githubusercontent.com/bfenetworks/ingress-bfe/develop/examples/ingress.yaml + ``` +* Create ingress resource for testing service to verify the installation if your kubernetes version >= 1.19 +``` shell script +kubectl apply -f https://raw.githubusercontent.com/bfenetworks/ingress-bfe/develop/examples/ingress-v1.19.yaml + +``` diff --git a/docs/en_us/ingress/priority.md b/docs/en_us/ingress/priority.md index 7527a6f7..3c89c3fd 100644 --- a/docs/en_us/ingress/priority.md +++ b/docs/en_us/ingress/priority.md @@ -108,7 +108,7 @@ spec: kind: Ingress apiVersion: networking.k8s.io/v1beta1 metadata: - name: "cond_priority1" + name: "cond_priority2" namespace: production annotations: kubernetes.io/ingress.class: bfe diff --git a/docs/images/arch.jpg b/docs/images/arch.jpg new file mode 100644 index 00000000..285f8085 Binary files /dev/null and b/docs/images/arch.jpg differ diff --git a/docs/zh_cn/SUMMARY.md b/docs/zh_cn/SUMMARY.md index 486484de..65e757ef 100644 --- a/docs/zh_cn/SUMMARY.md +++ b/docs/zh_cn/SUMMARY.md @@ -3,7 +3,7 @@ [comment]: <> "For user" * 部署 * [部署指南](deployment.md) - * [基于角色的控制访问（RBAC）](rbac.md) + * [基于角色的访问控制（RBAC）](rbac.md) * 配置 * [配置指南](ingress/basic.md) * [生效状态](ingress/validate-state.md) @@ -20,6 +20,9 @@ [comment]: <> "For developer" * [参与贡献](contribute/how-to-contribute.md) * [如何贡献代码](contribute/contribute-codes.md) + * [代码结构说明](development/source-code-layout.md) + * [核心工作原理](development/core-logic.md) + * [Annotation 开发指南](development/annotation-implement-guide.md) * [如何贡献文档](contribute/contribute-documents.md) * [版本发布说明](https://www.bfe-networks.net/zh_cn/development/release_regulation/) diff --git a/docs/zh_cn/contribute/contribute-codes.md b/docs/zh_cn/contribute/contribute-codes.md index 5133f560..ed7fb925 100644 --- a/docs/zh_cn/contribute/contribute-codes.md +++ b/docs/zh_cn/contribute/contribute-codes.md @@ -50,8 +50,20 @@ $ pip install pre-commit $ pre-commit install ``` - 1. 使用 `gofmt` 来调整 golang源代码格式。 - + 2. 使用 `gofmt` 来调整 golang源代码格式。 + +1. 使用 `license-eye` 工具 + + [license-eye](http://github.com/apache/skywalking-eyes) 工具可以帮助我们检查和修复所有文件的证书声明，在提交 (commit) 前证书声明都应该先完成。 + + `license-eye` 检查是 Github-Action 中检测的一部分，检测不通过的 PR 不能被提交到代码库，安装使用它： + + ```bash + $ make license-eye-install + $ make license-check + $ make license-fix + ``` + 1. 编写代码 在本例中，删除了 README.md 中的一行，并创建了一个新文件。 diff --git a/docs/zh_cn/deployment.md b/docs/zh_cn/deployment.md index 98af5909..32a51173 100644 --- a/docs/zh_cn/deployment.md +++ b/docs/zh_cn/deployment.md @@ -33,4 +33,9 @@ kubectl apply -f https://raw.githubusercontent.com/bfenetworks/ingress-bfe/devel kubectl apply -f https://raw.githubusercontent.com/bfenetworks/ingress-bfe/develop/examples/ingress.yaml ``` +* 创建k8s Ingress资源，验证消息路由 (kubernetes version >=1.19) +``` shell script +kubectl apply -f https://raw.githubusercontent.com/bfenetworks/ingress-bfe/develop/examples/ingress-v1.19.yaml + +``` diff --git a/docs/zh_cn/development/annotation-implement-guide.md b/docs/zh_cn/development/annotation-implement-guide.md new file mode 100644 index 00000000..ecd89e56 --- /dev/null +++ b/docs/zh_cn/development/annotation-implement-guide.md @@ -0,0 +1,356 @@ +# Annotation 开发指南 + +## 概述 + +在为 BFE Ingress Controller 中开发 Annotation 时，可参考本开发设计指南。 + +核心关注以下方面： +- Annotation 的定义 +- Annotation 的解析 +- BFE 配置的生成 +- BFE 配置的热加载 + +下面的讲述中，将结合 `bfe.ingress.kubernetes.io/balance.weight` 的实现作为例子。 +该Annotation用于支持多个Service之间的[负载均衡][]。 +> [balance.go][] + +## 1. Annotation 的定义 +根据需求不同，您可能需要定义并实现一个BFE Ingress Controller 的 Annotation，或者对 k8s Ingress 已定义的 Annotation 进行实现。 + +### 命名规范 +- Key: + - BFE Ingress Controller 的 Annotation + - `bfe.ingress.kubernetes.io/{module}.{key}` + - `bfe.ingress.kubernetes.io/{key}` + - k8s Ingress 约定的 Annotation + - `kubernetes.io/{key}` + - `ingressclass.kubernetes.io/{key}` +- Value: 根据需求设计定义 + +### 案例分析 + +- Key: `bfe.ingress.kubernetes.io/balance.weight` +- Value: 详见[负载均衡][] + - Demo: `{"service": {"service1":80, "service2":20}}` +- 源码 + - /internal/bfeConfig/annotations/[balance.go][] + + ```go + const ( + WeightKey = "balance.weight" + WeightAnnotation = BfeAnnotationPrefix + WeightKey + ) + + // ServicesWeight define struct of annotation "balance.weight" + // example: {"service": {"service1":80, "service2":20}} + type ServicesWeight map[string]int + type Balance map[string]ServicesWeight + ``` + +### 已有的 Annotation + +| Annotation 名 | 作用 | +| :--- | :---: | +| bfe.ingress.kubernetes.io/balance.weight | [负载均衡][] | +| bfe.ingress.kubernetes.io/router.cookie | 路由匹配条件：[匹配 Cookie](../ingress/basic.md#cookie) | +| bfe.ingress.kubernetes.io/router.header | 路由匹配条件：[匹配 Header](../ingress/basic.md#header) | +| bfe.ingress.kubernetes.io/bfe-ingress-status | [生效状态](../ingress/validate-state.md) | +| kubernetes.io/ingress.class | [申明 Ingress 类](https://kubernetes.io/zh/docs/concepts/services-networking/ingress/#deprecated-annotation) | +| ingressclass.kubernetes.io/is-default-class | [申明默认 Ingress 类](https://kubernetes.io/docs/concepts/services-networking/ingress/#default-ingress-class) | + +### 注意事项 + +新 BFE Ingress Controller Annotation 的定义需兼容已有的 Annotation，在实现指定功能的基础上，尽量做到： + +- 设计简洁 +- 避免与已有 Annotation 功能重复 + +更多细节建议在 [Issue][] 中讨论 + +## 2. Annotation 的解析 + +### 触发时机 + +1. [Kubernetes controller-runtime][] 监听事件后，触发 Reconcile +2. Reconciler 在[回调函数][ingress_controller.go]中，触发 configBuilder 的更新 +3. configBuilder 更新过程中，根据输入的 Ingress 资源 [解析][balance.go] 指定 Annotation，用于后续生成 BFE 配置 + +### 案例分析 + +- 指定 Annotation： `bfe.ingress.kubernetes.io/balance.weight` +- 源码 + - /internal/controllers/ingress/netv1/[ingress_controller.go][] + + ```go + func ReconcileV1Ingress(ctx context.Context, r client.Client, configBuilder *bfeConfig.ConfigBuilder, ingress *netv1.Ingress) error { + // ... + + if err = configBuilder.UpdateIngress(ingress, service, endpoints, secrets); err != nil { + configBuilder.DeleteIngress(ingress.Namespace, ingress.Name) + return err + } + + return nil + } + ``` + + - /internal/bfeConfig/configs/[clusterConfig.go][] + + ```go + func (c *ConfigBuilder) UpdateIngress(ingress *netv1.Ingress, services map[string]*corev1.Service, endpoints map[string]*corev1.Endpoints, secrets []*corev1.Secret) error { + // ... + + // update cluster conf + if err := c.clusterConf.UpdateIngress(ingress, services, endpoints); err != nil { + c.serverDataConf.DeleteIngress(ingress.Namespace, ingress.Name) + return err + } + + // ... + } + ``` + + - /internal/bfeConfig/configs/[clusterConfig.go][] + + ```go + func (c *ClusterConfig) UpdateIngress(ingress *netv1.Ingress, services map[string]*corev1.Service, endpoints map[string]*corev1.Endpoints) error { + // ... + + balance, _ := annotations.GetBalance(ingress.Annotations) + + // ... + } + ``` + + - /internal/bfeConfig/annotations/[balance.go][] + + ```go + // GetBalance parse annotation "balance.weight" + func GetBalance(annotations map[string]string) (Balance, error) { + value, ok := annotations[WeightAnnotation] + if !ok { + return nil, nil + } + + var lb = make(Balance) + err := json.Unmarshal([]byte(value), &lb) + if err != nil { + return nil, fmt.Errorf("annotation %s is illegal, error: %s", WeightAnnotation, err) + } + + // check whether weight sum > 0 + for _, services := range lb { + sum := 0 + for _, weight := range services { + if weight < 0 { + return nil, fmt.Errorf("weight of load balance service should >= 0") + } + sum += weight + } + if sum == 0 { + return nil, fmt.Errorf("sum of all load balance service weight should > 0") + } + } + return lb, nil + } + ``` + +## 3. BFE 配置的生成 + +### 触发时机 + +1. [Kubernetes controller-runtime][] 监听事件后，触发 Reconcile +2. Reconciler 在[回调函数][ingress_controller.go]中，触发 configBuilder 的更新 +3. configBuilder 更新过程中，根据输入的 Ingress 资源 [生成][clusterConfig.go] 多种 BFE 配置对象 + +### 案例分析 + +- 更新的配置对象：`configBuilder.clusterConf` +- 生成的负载均衡配置： [字段与格式说明](https://www.bfe-networks.net/zh_cn/configuration/cluster_conf/gslb.data/) +- 源码 + - /internal/controllers/ingress/netv1/[ingress_controller.go][] + + ```go + func ReconcileV1Ingress(ctx context.Context, r client.Client, configBuilder *bfeConfig.ConfigBuilder, ingress *netv1.Ingress) error { + // ... + + if err = configBuilder.UpdateIngress(ingress, service, endpoints, secrets); err != nil { + configBuilder.DeleteIngress(ingress.Namespace, ingress.Name) + return err + } + + return nil + } + ``` + + - /internal/bfeConfig/configs/[clusterConfig.go][] + + ```go + func (c *ConfigBuilder) UpdateIngress(ingress *netv1.Ingress, services map[string]*corev1.Service, endpoints map[string]*corev1.Endpoints, secrets []*corev1.Secret) error { + // ... + + // update cluster conf + if err := c.clusterConf.UpdateIngress(ingress, services, endpoints); err != nil { + c.serverDataConf.DeleteIngress(ingress.Namespace, ingress.Name) + return err + } + + // ... + } + ``` + + - /internal/bfeConfig/configs/[clusterConfig.go][] + + ```go + func (c *ClusterConfig) UpdateIngress(ingress *netv1.Ingress, services map[string]*corev1.Service, endpoints map[string]*corev1.Endpoints) error { + if len(ingress.Spec.Rules) == 0 { + return nil + } + + balance, _ := annotations.GetBalance(ingress.Annotations) + + ingressName := util.NamespacedName(ingress.Namespace, ingress.Name) + for _, rule := range ingress.Spec.Rules { + for _, path := range rule.HTTP.Paths { + // create cluster && subcluster for each Service + clusterName := util.ClusterName(ingressName, path.Backend.Service) + + // cluster config + (*c.clusterTableConf.Config)[clusterName] = c.newClusterBackend(ingress.Namespace, path.Backend.Service, balance, services, endpoints) + + // gslb config + (*c.gslbConf.Clusters)[clusterName] = c.newGslbClusterConf(ingress.Namespace, path.Backend.Service.Name, balance) + + // put into map + c.ingress2Cluster.Put(ingressName, clusterName) + for service := range (*c.gslbConf.Clusters)[clusterName] { + c.service2Cluster.Put(service, clusterName) + } + } + } + + if len(option.Opts.Ingress.DefaultBackend) > 0 { + c.addDefautBackend(endpoints[option.Opts.Ingress.DefaultBackend]) + } + + if err := cluster_table_conf.ClusterTableConfCheck(c.clusterTableConf); err != nil { + c.DeleteIngress(ingress.Namespace, ingress.Name) + return err + } + + c.setVersion() + return nil + } + ``` + +### 注意事项 + +- Ingress 资源的新增、更新、删除需分别适当处理 +- BFE 配置对象`configBuilder.*`可能存在缓存，注意对缓存内容的更新 + +## 4. BFE 配置的热加载 + +### 触发时机 + +根据配置时间间隔，定时触发 + +### 实现逻辑 + +对于多种 BFE 配置对象`configBuilder.*`，分别执行以下逻辑： + +1. 将配置对象以指定格式 [持久化为文件][clusterConfig.go]，存放在 BFE 指定路径 +2. 调用 BFE 进程 [reload 指令][clusterConfig.go]，完成相关配置的热加载 + +### 案例分析 + +- 负载均衡配置的热加载方式：[热加载接口说明](https://www.bfe-networks.net/zh_cn/operation/reload/#_5) +- 源码 + - /internal/bfeConfig/[configBuilder.go][] + + ```go + func (c *ConfigBuilder) InitReload(ctx context.Context) { + tick := time.NewTicker(option.Opts.Ingress.ReloadInterval) + + go func() { + defer tick.Stop() + for { + select { + case <-tick.C: + if err := c.reload(); err != nil { + log.Error(err, "fail to reload config") + } + case <-ctx.Done(): + log.Info("exit bfe reload") + return + } + } + }() + + } + + func (c *ConfigBuilder) reload() error { + // ... + + if err := c.clusterConf.Reload(); err != nil { + log.Error(err, "Fail to reload config", + "clusterConf", + c.clusterConf) + return err + } + + // ... + } + ``` + + - /internal/bfeConfig/configs/[clusterConfig.go][] + + ```go + func (c *ClusterConfig) Reload() error { + reload := false + if *c.gslbConf.Ts != c.gslbVersion { + err := util.DumpBfeConf(GslbData, c.gslbConf) + if err != nil { + return fmt.Errorf("dump gslb.data error: %v", err) + } + + reload = true + } + if *c.clusterTableConf.Version != c.clusterTableVersion { + err := util.DumpBfeConf(ClusterTableData, c.clusterTableConf) + if err != nil { + return fmt.Errorf("dump cluster_table.data error: %v", err) + } + reload = true + } + + if reload { + if err := util.ReloadBfe(ConfigNameclusterConf); err != nil { + return err + } + c.gslbVersion = *c.gslbConf.Ts + c.clusterTableVersion = *c.clusterTableConf.Version + } + + return nil + } + ``` + +## FAQ + +- [如何查询特定 BFE 配置的格式？](core-logic.md#BFE配置如何定义) +- [如何查询指定 BFE 配置的文件路径和热加载方式？][配置热加载] + +[Issue]: https://github.com/bfenetworks/ingress-bfe/labels/enhancement + +[Kubernetes controller-runtime]: https://github.com/kubernetes-sigs/controller-runtime + +[负载均衡]: ../ingress/load-balance.md +[配置热加载]: https://www.bfe-networks.net/zh_cn/operation/reload/ + +[balance.go]: ../../../internal/bfeConfig/annotations/balance.go + +[clusterConfig.go]: ../../../internal/bfeConfig/configs/clusterConfig.go + +[configBuilder.go]: ../../../internal/bfeConfig/configBuilder.go + +[ingress_controller.go]: ../../../internal/controllers/netv1/ingress_controller.go diff --git a/docs/zh_cn/development/core-logic.md b/docs/zh_cn/development/core-logic.md new file mode 100644 index 00000000..f12c99d1 --- /dev/null +++ b/docs/zh_cn/development/core-logic.md @@ -0,0 +1,51 @@ +# BFE Ingress Controller 工作原理 + +## 核心处理逻辑 + +> 前置知识 +> - 什么是 [Ingress](https://kubernetes.io/docs/concepts/services-networking/ingress/) +> - 什么是 [Ingress Controller](https://kubernetes.io/docs/concepts/services-networking/ingress-controllers/) + +![arch](../../images/arch.jpg) +BFE Ingress Controller 的核心处理逻辑是： +1. 监听获取 k8s 集群中的 Ingress 资源 +2. 解析 Ingress 资源中定义的配置逻辑，生成对应的 BFE 配置 +3. 使新生成的 BFE 配置在 BFE进程中生效 + +## 如何实现 Ingress 资源的监听 + +通过 [Kubernetes controller-runtime][] 框架实现： +1. 创建 Manager，Manager 内部维护了 Client、Cache 和 Schema 等 +2. 创建 Reconciler，并实现回调逻辑 +3. 使用 Builder 模式创建 Controller，并指定监听 Ingress 资源和对应的 Reconcile +4. 启动 Manager + +## BFE配置如何定义 + +BFE 的配置定义可以通过以下方式获得： +1. 官网文档：[配置概述][]（主要关注动态配置） +2. BFE 源码： bfe/[bfe_modules][]/mod_\*/\*_load.go + +官网更新可能存在延迟，以源代码为准。 + +> 常见动态配置： +> - 流量路由配置 +> - [域名规则配置](https://www.bfe-networks.net/zh_cn/configuration/server_data_conf/host_rule.data/) +> - [分流规则配置](https://www.bfe-networks.net/zh_cn/configuration/server_data_conf/route_rule.data/) +> - 负载均衡配置 +> - [子集群负载均衡配置](https://www.bfe-networks.net/zh_cn/configuration/cluster_conf/gslb.data/) +> - 接入协议配置 +> - [TLS协议配置](https://www.bfe-networks.net/zh_cn/configuration/tls_conf/tls_rule_conf.data/) +> - 扩展模块配置 +> - [mod_header 配置](https://www.bfe-networks.net/zh_cn/modules/mod_header/mod_header/) +> - [mod_redirect 配置](https://www.bfe-networks.net/zh_cn/modules/mod_header/mod_redirect/) +> - [mod_rewrite 配置](https://www.bfe-networks.net/zh_cn/modules/mod_header/mod_rewrite/) + +## BFE配置如何生效 + +通过 BFE [配置热加载][]，完成在 BFE 不停机的情况下更新动态配置。 + +[Kubernetes controller-runtime]: https://github.com/kubernetes-sigs/controller-runtime +[配置概述]: https://www.bfe-networks.net/zh_cn/configuration/config/ +[bfe_modules]: https://github.com/bfenetworks/bfe/tree/develop/bfe_modules +[配置热加载]: https://www.bfe-networks.net/zh_cn/operation/reload/ \ No newline at end of file diff --git a/docs/zh_cn/development/source-code-layout.md b/docs/zh_cn/development/source-code-layout.md new file mode 100644 index 00000000..d3bfe915 --- /dev/null +++ b/docs/zh_cn/development/source-code-layout.md @@ -0,0 +1,78 @@ +# BFE Ingress 源代码框架 + +```shell +. +├── .github : Github 工作流目录 +├── build : 编译脚本目录 +├── charts : BFE Ingress Controller 的 Helm Charts目录 +├── cmd +│   └── ingress-controller : 主程序目录 +├── docs : 中英文用户文档及其素材目录 +├── examples : k8s 资源描述文件示例目录 +├── internal : 核心源码目录 +│   ├── bfeConfig : BFE Ingress Controller 配置选项定义代码 +│   ├── controllers : k8s 集群交互相关代码，主要包含各资源的controller的实现以及reconcile逻辑 +│   └── option : BFE 配置相关代码，主要包含各 BFE 配置的生成和热加载逻辑 +├── scripts : 镜像依赖的脚本目录 +├── CHANGELOG.md : 版本修改日志 +├── Dockerfile : 构建镜像的指令文件 +├── LICENSE : License 协议 +├── Makefile : 程序编译与镜像制作的指令文件 +├── SECURITY.md : 安全策略 +├── VERSION : 程序版本 +├── go.mod : Go 语言依赖管理文件 +└── go.sum : Go 语言依赖管理文件 +``` + +## 核心代码 +- /[internal][]: 核心源码目录 + - /[option][]: BFE Ingress Controller 配置选项定义代码 + - /[controllers][]: k8s 集群交互相关代码，主要包含各资源的controller的实现以及reconcile逻辑 + - /[bfeConfig][]: BFE 配置相关代码，主要包含各 BFE 配置的生成和热加载逻辑 + +## 持续集成 + +### 工作流 +- /[.github][]: Github 工作流目录 +- /[Makefile][]: 程序编译与镜像制作的指令文件 + +### 程序编译 +- /cmd/[ingress-controller][]: 主程序目录 +- /[VERSION][]: 程序版本 +- /[build][]: 编译脚本目录 +- /[go.mod][]: Go 语言依赖管理文件 +- /[go.sum][]: Go 语言依赖管理文件 + +### 镜像制作 +- /[Dockerfile][]: 构建镜像的指令文件 +- /[scripts][]: 镜像依赖的脚本目录（如镜像启动脚本） + +## Helm 支持 +- /[charts][]: BFE Ingress Controller 的 Helm Charts + +## 项目文档 +- /[docs][]: 中英文用户文档及其素材目录 +- /[examples][]: k8s 资源描述文件示例目录 +- /[CHANGELOG.md][]: 版本修改日志 +- /[LICENSE][]: License 协议 +- /[SECURITY.md][]: 安全策略 + +[.github]: ../../../.github +[build]: ../../../build +[charts]: ../../../charts +[ingress-controller]: ../../../cmd/ingress-controller +[docs]: ../../../docs +[examples]: ../../../examples +[internal]: ../../../internal +[bfeConfig]: ../../../internal/bfeConfig +[controllers]: ../../../internal/controllers +[go.mod]: ../../../go.mod +[go.sum]: ../../../go.sum +[option]: ../../../internal/option +[scripts]: ../../../scripts +[CHANGELOG.md]: ../../../CHANGELOG.md +[Dockerfile]: ../../../Dockerfile +[LICENSE]: ../../../LICENSE +[Makefile]: ../../../Makefile +[SECURITY.md]: ../../../SECURITY.md +[VERSION]: ../../../VERSION \ No newline at end of file diff --git a/docs/zh_cn/ingress/priority.md b/docs/zh_cn/ingress/priority.md index 68d96e2a..541f143c 100644 --- a/docs/zh_cn/ingress/priority.md +++ b/docs/zh_cn/ingress/priority.md @@ -108,7 +108,7 @@ spec: kind: Ingress apiVersion: networking.k8s.io/v1beta1 metadata: - name: "cond_priority1" + name: "cond_priority2" namespace: production annotations: kubernetes.io/ingress.class: bfe diff --git a/docs/zh_cn/rbac.md b/docs/zh_cn/rbac.md index 061281e8..30c13084 100644 --- a/docs/zh_cn/rbac.md +++ b/docs/zh_cn/rbac.md @@ -1,4 +1,4 @@ -# 基于角色的控制访问（RBAC） +# 基于角色的访问控制（RBAC） ## 说明 diff --git a/examples/controller-all.yaml b/examples/controller-all.yaml index ab6bad72..cf5bd13a 100644 --- a/examples/controller-all.yaml +++ b/examples/controller-all.yaml @@ -36,6 +36,13 @@ rules: - get - list - watch +- apiGroups: + - "" + resources: + - events + verbs: + - create + - patch - apiGroups: - extensions resources: @@ -85,7 +92,6 @@ metadata: labels: app.kubernetes.io/name: bfe-ingress-controller app.kubernetes.io/instance: bfe-ingress-controller - spec: replicas: 1 selector: @@ -95,6 +101,7 @@ spec: metadata: labels: app.kubernetes.io/name: bfe-ingress-controller + app.kubernetes.io/instance: bfe-ingress-controller spec: serviceAccountName: bfe-ingress-controller containers: diff --git a/examples/ingress-v1.19.yaml b/examples/ingress-v1.19.yaml new file mode 100644 index 00000000..0a106fa1 --- /dev/null +++ b/examples/ingress-v1.19.yaml @@ -0,0 +1,20 @@ +kind: Ingress +apiVersion: networking.k8s.io/v1 +metadata: + name: ingress-test + namespace: ingress-bfe + annotations: + kubernetes.io/ingress.class: bfe + +spec: + rules: + - host: "foo.com" + http: + paths: + - path: /whoami + pathType: Prefix + backend: + service: + name: whoami + port: + number: 80 diff --git a/examples/ingress.yaml b/examples/ingress.yaml index 85974b5c..49f20c7e 100644 --- a/examples/ingress.yaml +++ b/examples/ingress.yaml @@ -1,5 +1,5 @@ kind: Ingress -apiVersion: networking.k8s.io/v1 +apiVersion: networking.k8s.io/v1beta1 metadata: name: ingress-test namespace: ingress-bfe diff --git a/examples/rbac.yaml b/examples/rbac.yaml index 40afc81f..5872549d 100644 --- a/examples/rbac.yaml +++ b/examples/rbac.yaml @@ -18,6 +18,13 @@ rules: - get - list - watch +- apiGroups: + - "" + resources: + - events + verbs: + - create + - patch - apiGroups: - extensions resources: diff --git a/examples/whoami.yaml b/examples/whoami.yaml index f8062af8..a525025d 100644 --- a/examples/whoami.yaml +++ b/examples/whoami.yaml @@ -3,7 +3,6 @@ apiVersion: apps/v1 metadata: name: whoami namespace: ingress-bfe - labels: labels: app.kubernetes.io/name: whoami app.kubernetes.io/instance: whoami diff --git a/internal/bfeConfig/annotations/balance.go b/internal/bfeConfig/annotations/balance.go index f553cced..a5e45702 100644 --- a/internal/bfeConfig/annotations/balance.go +++ b/internal/bfeConfig/annotations/balance.go @@ -23,7 +23,7 @@ const ( WeightAnnotation = BfeAnnotationPrefix + WeightKey ) -// Balance define struct of annotation "balance.weight" +// ServicesWeight define struct of annotation "balance.weight" // example: {"service": {"service1":80, "service2":20}} type ServicesWeight map[string]int type Balance map[string]ServicesWeight diff --git a/internal/bfeConfig/configBuilder.go b/internal/bfeConfig/configBuilder.go index d4310198..9b0e9854 100644 --- a/internal/bfeConfig/configBuilder.go +++ b/internal/bfeConfig/configBuilder.go @@ -15,6 +15,7 @@ package bfeConfig import ( + "context" "sync" "time" @@ -110,13 +111,20 @@ func (c *ConfigBuilder) DeleteSecret(namespace, name string) { c.tlsConf.DeleteSecret(namespace, name) } -func (c *ConfigBuilder) InitReload() { - tick := time.NewTicker(option.Opts.ReloadInterval) +func (c *ConfigBuilder) InitReload(ctx context.Context) { + tick := time.NewTicker(option.Opts.Ingress.ReloadInterval) go func() { - for range tick.C { - if err := c.reload(); err != nil { - log.Error(err, "fail to reload config") + defer tick.Stop() + for { + select { + case <-tick.C: + if err := c.reload(); err != nil { + log.Error(err, "fail to reload config") + } + case <-ctx.Done(): + log.Info("exit bfe reload") + return } } }() diff --git a/internal/bfeConfig/configs/TLSConfig.go b/internal/bfeConfig/configs/TLSConfig.go index e71e3dd3..fee5efa6 100644 --- a/internal/bfeConfig/configs/TLSConfig.go +++ b/internal/bfeConfig/configs/TLSConfig.go @@ -58,12 +58,10 @@ type TLSConfig struct { func NewTLSConfig(version string) *TLSConfig { tlsConf := &TLSConfig{ - serverCertVersion: version, - tlsRuleVersion: version, - ingress2secret: setmultimap.New(), - serverCertConf: newServerCertConf(version), - tlsRuleConf: newTlsRuleConf(version), - certs: make(map[string]certConf), + ingress2secret: setmultimap.New(), + serverCertConf: newServerCertConf(version), + tlsRuleConf: newTlsRuleConf(version), + certs: make(map[string]certConf), } return tlsConf diff --git a/internal/bfeConfig/configs/clusterConfig.go b/internal/bfeConfig/configs/clusterConfig.go index 951da361..d31ea2cc 100644 --- a/internal/bfeConfig/configs/clusterConfig.go +++ b/internal/bfeConfig/configs/clusterConfig.go @@ -62,10 +62,8 @@ func NewClusterConfig(version string) *ClusterConfig { clusterBackend := make(cluster_table_conf.AllClusterBackend) return &ClusterConfig{ - gslbVersion: version, - clusterTableVersion: version, - ingress2Cluster: setmultimap.New(), - service2Cluster: setmultimap.New(), + ingress2Cluster: setmultimap.New(), + service2Cluster: setmultimap.New(), gslbConf: gslb_conf.GslbConf{ Clusters: &gslbCluster, Hostname: &hostname, @@ -112,8 +110,8 @@ func (c *ClusterConfig) UpdateIngress(ingress *netv1.Ingress, services map[strin } } - if len(option.Opts.DefaultBackend) > 0 { - c.addDefautBackend(endpoints[option.Opts.DefaultBackend]) + if len(option.Opts.Ingress.DefaultBackend) > 0 { + c.addDefautBackend(endpoints[option.Opts.Ingress.DefaultBackend]) } if err := cluster_table_conf.ClusterTableConfCheck(c.clusterTableConf); err != nil { @@ -140,7 +138,7 @@ func (c *ClusterConfig) addDefautBackend(ep *corev1.Endpoints) { return } - serviceName := option.Opts.DefaultBackend + serviceName := option.Opts.Ingress.DefaultBackend subCluster := make(cluster_table_conf.ClusterBackend) subCluster[serviceName] = instanceList @@ -173,7 +171,7 @@ func (c *ClusterConfig) DeleteIngress(namespace, name string) { c.ingress2Cluster.RemoveAll(ingressName) // if no ingress exist, remove default backend config - if len(option.Opts.DefaultBackend) > 0 && c.ingress2Cluster.Empty() { + if len(option.Opts.Ingress.DefaultBackend) > 0 && c.ingress2Cluster.Empty() { c.delDefautBackend() } @@ -181,7 +179,7 @@ func (c *ClusterConfig) DeleteIngress(namespace, name string) { } func (c *ClusterConfig) delDefautBackend() { - c.service2Cluster.Remove(option.Opts.DefaultBackend, util.DefaultClusterName()) + c.service2Cluster.Remove(option.Opts.Ingress.DefaultBackend, util.DefaultClusterName()) delete(*c.clusterTableConf.Config, util.DefaultClusterName()) delete(*c.gslbConf.Clusters, util.DefaultClusterName()) } @@ -313,7 +311,7 @@ func (c *ClusterConfig) UpdateService(service *corev1.Service, endpoint *corev1. targetPort := getTargetPort(util.ParsePort(name), service) // targetPort not found, which is not allowed for normal backend beside default backend - if targetPort.IntVal == 0 && len(targetPort.StrVal) == 0 && serviceName != option.Opts.DefaultBackend { + if targetPort.IntVal == 0 && len(targetPort.StrVal) == 0 && serviceName != option.Opts.Ingress.DefaultBackend { c.DeleteService(service.Namespace, service.Name) log.V(0).Info("ingress backend port not found in service", "namespace", service.Namespace, "name", service.Name, "port", util.ParsePort(name)) return fmt.Errorf("cluster [%s] error, port can not found in service", name) diff --git a/internal/bfeConfig/configs/routeRuleCache.go b/internal/bfeConfig/configs/routeRuleCache.go index 74a0d485..507fa40c 100644 --- a/internal/bfeConfig/configs/routeRuleCache.go +++ b/internal/bfeConfig/configs/routeRuleCache.go @@ -171,7 +171,7 @@ func (c *HttpRouteRuleCache) put(rule *httpRule) error { } else if rule.createTime.Equal(r.createTime) { return nil } else { - return fmt.Errorf("conflict with %s, rule [host: %s, path: %s]", r.ingress, rule.host, rule.path) + return fmt.Errorf("ingress [%s] conflict with existing %s, rule [host: %s, path: %s]", rule.ingress, r.ingress, rule.host, rule.path) } } } diff --git a/internal/bfeConfig/configs/serverDataConfig.go b/internal/bfeConfig/configs/serverDataConfig.go index 11fddb89..5fe6eebc 100644 --- a/internal/bfeConfig/configs/serverDataConfig.go +++ b/internal/bfeConfig/configs/serverDataConfig.go @@ -52,13 +52,10 @@ type ServerDataConfig struct { func NewServerDataConfig(version string) *ServerDataConfig { return &ServerDataConfig{ - hostTableVersion: version, - routeTableVersion: version, - bfeClusterConfVersion: version, - routeRuleCache: NewRouteRuleCache(), - hostTableConf: newHostTableConf(util.NewVersion()), - routeTableFile: newRouteTableConfFile(version), - bfeClusterConf: newBfeClusterConf(version), + routeRuleCache: NewRouteRuleCache(), + hostTableConf: newHostTableConf(version), + routeTableFile: newRouteTableConfFile(version), + bfeClusterConf: newBfeClusterConf(version), } } @@ -252,7 +249,7 @@ func (c *ServerDataConfig) updateRouteTable() error { (*routeTableFile.ProductRule)[DefaultProduct] = append((*routeTableFile.ProductRule)[DefaultProduct], ruleFile) } - if len(option.Opts.DefaultBackend) > 0 && (len(basicRules) > 0 || len(advancedRules) > 0) { + if len(option.Opts.Ingress.DefaultBackend) > 0 && (len(basicRules) > 0 || len(advancedRules) > 0) { condition := "default_t()" cluster := util.DefaultClusterName() ruleFile := route_rule_conf.AdvancedRouteRuleFile{ @@ -293,7 +290,7 @@ func (c *ServerDataConfig) updateBfeClusterConf() { GslbBasic: newGslbBasicConf(), } } - if len(option.Opts.DefaultBackend) > 0 && (len(basicRules) > 0 || len(advancedRules) > 0) { + if len(option.Opts.Ingress.DefaultBackend) > 0 && (len(basicRules) > 0 || len(advancedRules) > 0) { (*clusterConf.Config)[util.DefaultClusterName()] = cluster_conf.ClusterConf{ CheckConf: newCheckConf(), GslbBasic: newGslbBasicConf(), diff --git a/internal/bfeConfig/util/io.go b/internal/bfeConfig/util/io.go index 2aaeb6b5..3f5c923b 100644 --- a/internal/bfeConfig/util/io.go +++ b/internal/bfeConfig/util/io.go @@ -34,23 +34,23 @@ func DumpBfeConf(configFile string, object interface{}) error { } func DumpFile(filename string, data []byte) error { - name := option.Opts.ConfigPath + filename + name := option.Opts.Ingress.ConfigPath + filename filePath := filepath.Dir(name) if _, err := os.Stat(filePath); os.IsNotExist(err) { - os.MkdirAll(filePath, option.FilePerm) + os.MkdirAll(filePath, option.Opts.Ingress.FilePerm) } - return ioutil.WriteFile(name, data, option.FilePerm) + return ioutil.WriteFile(name, data, option.Opts.Ingress.FilePerm) } func DeleteFile(filename string) { - name := option.Opts.ConfigPath + filename + name := option.Opts.Ingress.ConfigPath + filename os.Remove(name) } // ReloadBfe triggers bfe process to reload new config file through bfe monitor port func ReloadBfe(configName string) error { - url := option.Opts.ReloadUrl + configName + url := option.Opts.Ingress.ReloadUrl + configName res, err := http.Get(url) if err != nil { return err diff --git a/internal/bfeConfig/util/name.go b/internal/bfeConfig/util/name.go index 1ee9cd4a..f0746704 100644 --- a/internal/bfeConfig/util/name.go +++ b/internal/bfeConfig/util/name.go @@ -34,7 +34,7 @@ func ClusterName(ingressName string, backend *netv1.IngressServiceBackend) strin // DefaultClusterName returns a default cluster for default backend func DefaultClusterName() string { ingress := "__defaultCluster__" - return fmt.Sprintf("%s_%s_%d", ingress, option.Opts.DefaultBackend, 0) + return fmt.Sprintf("%s_%s_%d", ingress, option.Opts.Ingress.DefaultBackend, 0) } func ParsePort(clusterName string) netv1.ServiceBackendPort { diff --git a/internal/controllers/event/events.go b/internal/controllers/event/events.go new file mode 100644 index 00000000..1ad2add7 --- /dev/null +++ b/internal/controllers/event/events.go @@ -0,0 +1,19 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package event + +// event reason +const ( + SyncFailed = "SyncFailed" + SyncSucceed = "SyncSucceed" +) diff --git a/internal/controllers/filter/func.go b/internal/controllers/filter/ingressClass.go similarity index 66% rename from internal/controllers/filter/func.go rename to internal/controllers/filter/ingressClass.go index 45c23e96..ad0ba1c5 100644 --- a/internal/controllers/filter/func.go +++ b/internal/controllers/filter/ingressClass.go @@ -18,42 +18,24 @@ import ( "context" "strings" - corev1 "k8s.io/api/core/v1" netv1 "k8s.io/api/networking/v1" netv1beta1 "k8s.io/api/networking/v1beta1" "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/predicate" "github.com/bfenetworks/ingress-bfe/internal/bfeConfig/annotations" "github.com/bfenetworks/ingress-bfe/internal/option" ) -func NamespaceFilter() predicate.Funcs { - funcs := predicate.NewPredicateFuncs(func(obj client.Object) bool { - if len(option.Opts.Namespaces) == 1 && option.Opts.Namespaces[0] == corev1.NamespaceAll { - return true - } - for _, ns := range option.Opts.Namespaces { - if ns == obj.GetNamespace() { - return true - } - } - return false - }) - - return funcs -} - -func MatchIngressClass(ctx context.Context, r client.Reader, annots map[string]string, ingressClassName *string) bool { - if annots[annotations.IngressClassKey] == option.Opts.IngressClass { +func IngressClassFilter(ctx context.Context, r client.Reader, annots map[string]string, ingressClassName *string) bool { + if annots[annotations.IngressClassKey] == option.Opts.Ingress.IngressClass { return true } classListV1 := &netv1.IngressClassList{} - err := r.List(ctx, classListV1, client.MatchingLabels{".spec.controller": option.Opts.ControllerName}) + err := r.List(ctx, classListV1) if err == nil { for _, class := range classListV1.Items { - if class.Spec.Controller != option.Opts.ControllerName { + if class.Spec.Controller != option.Opts.Ingress.ControllerName { continue } if (ingressClassName != nil && *ingressClassName == class.Name) || @@ -69,7 +51,7 @@ func MatchIngressClass(ctx context.Context, r client.Reader, annots map[string]s return false } for _, classV1Beta1 := range classListV1Beta1.Items { - if classV1Beta1.Spec.Controller != option.Opts.ControllerName { + if classV1Beta1.Spec.Controller != option.Opts.Ingress.ControllerName { continue } if (ingressClassName != nil && *ingressClassName == classV1Beta1.Name) || diff --git a/internal/controllers/filter/namespace.go b/internal/controllers/filter/namespace.go new file mode 100644 index 00000000..5ee92c69 --- /dev/null +++ b/internal/controllers/filter/namespace.go @@ -0,0 +1,39 @@ +// Copyright (c) 2021 The BFE Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package filter + +import ( + corev1 "k8s.io/api/core/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/predicate" + + "github.com/bfenetworks/ingress-bfe/internal/option" +) + +func NamespaceFilter() predicate.Funcs { + funcs := predicate.NewPredicateFuncs(func(obj client.Object) bool { + if len(option.Opts.NamespaceList) == 1 && option.Opts.NamespaceList[0] == corev1.NamespaceAll { + return true + } + for _, ns := range option.Opts.NamespaceList { + if ns == obj.GetNamespace() { + return true + } + } + return false + }) + + return funcs +} diff --git a/internal/controllers/extv1beta1/ingress_controller.go b/internal/controllers/ingress/extv1beta1/ingress_controller.go similarity index 76% rename from internal/controllers/extv1beta1/ingress_controller.go rename to internal/controllers/ingress/extv1beta1/ingress_controller.go index b9f8e801..74a35de2 100644 --- a/internal/controllers/extv1beta1/ingress_controller.go +++ b/internal/controllers/ingress/extv1beta1/ingress_controller.go @@ -10,32 +10,57 @@ // See the License for the specific language governing permissions and // limitations under the License. -package extensionsv1beta1 +package extv1beta1 import ( "context" + "fmt" + corev1 "k8s.io/api/core/v1" extv1beta1 "k8s.io/api/extensions/v1beta1" netv1 "k8s.io/api/networking/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/tools/record" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/builder" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/reconcile" "github.com/bfenetworks/ingress-bfe/internal/bfeConfig" "github.com/bfenetworks/ingress-bfe/internal/bfeConfig/annotations" + "github.com/bfenetworks/ingress-bfe/internal/controllers/event" "github.com/bfenetworks/ingress-bfe/internal/controllers/filter" - controllerV1 "github.com/bfenetworks/ingress-bfe/internal/controllers/netv1" + controllerV1 "github.com/bfenetworks/ingress-bfe/internal/controllers/ingress/netv1" ) +func AddIngressController(mgr manager.Manager, cb *bfeConfig.ConfigBuilder) error { + reconciler := newIngressReconciler(mgr, cb) + if err := reconciler.setupWithManager(mgr); err != nil { + return fmt.Errorf("unable to create service controller") + } + + return nil +} + // IngressReconciler reconciles a extv1beta1 Ingress object type IngressReconciler struct { BfeConfigBuilder *bfeConfig.ConfigBuilder client.Client - Scheme *runtime.Scheme + Scheme *runtime.Scheme + recorder record.EventRecorder +} + +func newIngressReconciler(mgr manager.Manager, cb *bfeConfig.ConfigBuilder) *IngressReconciler { + return &IngressReconciler{ + BfeConfigBuilder: cb, + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + recorder: mgr.GetEventRecorderFor("bfe-ingress-controller"), + } } func (r *IngressReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { @@ -48,10 +73,10 @@ func (r *IngressReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct if err != nil { r.BfeConfigBuilder.DeleteIngress(req.Namespace, req.Name) log.V(1).Info("reconcile: ingress delete") - return reconcile.Result{}, err + return reconcile.Result{}, nil } - if !filter.MatchIngressClass(ctx, r, ingressExtV1beta1.Annotations, ingressExtV1beta1.Spec.IngressClassName) { + if !filter.IngressClassFilter(ctx, r, ingressExtV1beta1.Annotations, ingressExtV1beta1.Spec.IngressClassName) { return reconcile.Result{}, nil } @@ -62,11 +87,18 @@ func (r *IngressReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct err = controllerV1.ReconcileV1Ingress(ctx, r.Client, r.BfeConfigBuilder, ingressV1) setStatus(ctx, r.Client, err, ingressExtV1beta1) + + if err != nil { + r.recorder.Event(ingressExtV1beta1, corev1.EventTypeWarning, event.SyncFailed, err.Error()) + } else { + r.recorder.Event(ingressExtV1beta1, corev1.EventTypeNormal, event.SyncSucceed, "Synced") + } + return reconcile.Result{}, err } -// SetupWithManager sets up the controller with the Manager. -func (r *IngressReconciler) SetupWithManager(mgr ctrl.Manager) error { +// setupWithManager sets up the controller with the Manager. +func (r *IngressReconciler) setupWithManager(mgr ctrl.Manager) error { return ctrl.NewControllerManagedBy(mgr). For(&extv1beta1.Ingress{}, builder.WithPredicates(filter.NamespaceFilter())). Complete(r) diff --git a/internal/controllers/netv1/ingress_controller.go b/internal/controllers/ingress/netv1/ingress_controller.go similarity index 79% rename from internal/controllers/netv1/ingress_controller.go rename to internal/controllers/ingress/netv1/ingress_controller.go index 667fb0b8..86098b9c 100644 --- a/internal/controllers/netv1/ingress_controller.go +++ b/internal/controllers/ingress/netv1/ingress_controller.go @@ -21,26 +21,47 @@ import ( netv1 "k8s.io/api/networking/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/tools/record" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/builder" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/log" - "sigs.k8s.io/controller-runtime/pkg/predicate" + "sigs.k8s.io/controller-runtime/pkg/manager" "sigs.k8s.io/controller-runtime/pkg/reconcile" "github.com/bfenetworks/ingress-bfe/internal/bfeConfig" "github.com/bfenetworks/ingress-bfe/internal/bfeConfig/annotations" "github.com/bfenetworks/ingress-bfe/internal/bfeConfig/util" + "github.com/bfenetworks/ingress-bfe/internal/controllers/event" "github.com/bfenetworks/ingress-bfe/internal/controllers/filter" "github.com/bfenetworks/ingress-bfe/internal/option" ) +func AddIngressController(mgr manager.Manager, cb *bfeConfig.ConfigBuilder) error { + reconciler := newIngressReconciler(mgr, cb) + if err := reconciler.setupWithManager(mgr); err != nil { + return fmt.Errorf("unable to create ingress controller") + } + + return nil +} + // IngressReconciler reconciles a netv1 Ingress object type IngressReconciler struct { BfeConfigBuilder *bfeConfig.ConfigBuilder client.Client - Scheme *runtime.Scheme + Scheme *runtime.Scheme + recorder record.EventRecorder +} + +func newIngressReconciler(mgr manager.Manager, cb *bfeConfig.ConfigBuilder) *IngressReconciler { + return &IngressReconciler{ + BfeConfigBuilder: cb, + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + recorder: mgr.GetEventRecorderFor("bfe-ingress-controller"), + } } func (r *IngressReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { @@ -53,10 +74,10 @@ func (r *IngressReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct if err != nil { r.BfeConfigBuilder.DeleteIngress(req.Namespace, req.Name) log.V(1).Info("reconcile: ingress delete") - return reconcile.Result{}, err + return reconcile.Result{}, nil } - if !filter.MatchIngressClass(ctx, r, ingress.Annotations, ingress.Spec.IngressClassName) { + if !filter.IngressClassFilter(ctx, r, ingress.Annotations, ingress.Spec.IngressClassName) { return reconcile.Result{}, nil } @@ -64,11 +85,18 @@ func (r *IngressReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct err = ReconcileV1Ingress(ctx, r.Client, r.BfeConfigBuilder, ingress) setStatus(ctx, r.Client, err, ingress) + + if err != nil { + r.recorder.Event(ingress, corev1.EventTypeWarning, event.SyncFailed, err.Error()) + } else { + r.recorder.Event(ingress, corev1.EventTypeNormal, event.SyncSucceed, "Synced") + } + return reconcile.Result{}, err } -// SetupWithManager sets up the controller with the Manager. -func (r *IngressReconciler) SetupWithManager(mgr ctrl.Manager) error { +// setupWithManager sets up the controller with the Manager. +func (r *IngressReconciler) setupWithManager(mgr ctrl.Manager) error { return ctrl.NewControllerManagedBy(mgr). For(&netv1.Ingress{}, builder.WithPredicates(filter.NamespaceFilter())). Complete(r) @@ -83,6 +111,9 @@ func setStatus(ctx context.Context, r client.Client, err error, ingress *netv1.I } patch := client.MergeFrom(ingress.DeepCopy()) + if ingress.Annotations == nil { + ingress.Annotations = make(map[string]string) + } ingress.Annotations[annotations.StatusAnnotationKey] = annotations.GenErrorMsg(err) if err := r.Patch(ctx, ingress, patch); err != nil { log.Error(err, "fail to update annotation") @@ -96,9 +127,9 @@ func ReconcileV1Ingress(ctx context.Context, r client.Client, configBuilder *bfe return err } - if len(option.Opts.DefaultBackend) > 0 { + if len(option.Opts.Ingress.DefaultBackend) > 0 { // use default backend from controller command line argument - setDefautBackend(ingress, service[option.Opts.DefaultBackend]) + setDefautBackend(ingress, service[option.Opts.Ingress.DefaultBackend]) } secrets, err := getIngressSecret(ctx, r, ingress) @@ -115,30 +146,14 @@ func ReconcileV1Ingress(ctx context.Context, r client.Client, configBuilder *bfe return nil } -func NamespaceFilter() predicate.Funcs { - funcs := predicate.NewPredicateFuncs(func(obj client.Object) bool { - if len(option.Opts.Namespaces) == 0 { - return true - } - for _, ns := range option.Opts.Namespaces { - if ns == obj.GetNamespace() { - return true - } - } - return false - }) - - return funcs -} - func getIngressBackends(ctx context.Context, r client.Reader, ingress *netv1.Ingress) (map[string]*corev1.Service, map[string]*corev1.Endpoints, error) { services := make(map[string]*corev1.Service) endpoints := make(map[string]*corev1.Endpoints) - if len(option.Opts.DefaultBackend) > 0 { - if svc, ep, err := getDefaultBackends(ctx, r, option.Opts.DefaultBackend); err == nil { - services[option.Opts.DefaultBackend] = svc - endpoints[option.Opts.DefaultBackend] = ep + if len(option.Opts.Ingress.DefaultBackend) > 0 { + if svc, ep, err := getDefaultBackends(ctx, r, option.Opts.Ingress.DefaultBackend); err == nil { + services[option.Opts.Ingress.DefaultBackend] = svc + endpoints[option.Opts.Ingress.DefaultBackend] = ep } } @@ -266,11 +281,11 @@ func getSecret(ctx context.Context, r client.Reader, namespace, name string) (*c // set defaultBackend in ingress func setDefautBackend(ingress *netv1.Ingress, service *corev1.Service) { - if len(option.Opts.DefaultBackend) == 0 || service == nil || len(service.Spec.Ports) == 0 { + if len(option.Opts.Ingress.DefaultBackend) == 0 || service == nil || len(service.Spec.Ports) == 0 { return } - names := strings.Split(option.Opts.DefaultBackend, "/") + names := strings.Split(option.Opts.Ingress.DefaultBackend, "/") backend := &netv1.IngressBackend{} backend.Service = &netv1.IngressServiceBackend{ diff --git a/internal/controllers/netv1beta1/ingress_controller.go b/internal/controllers/ingress/netv1beta1/ingress_controller.go similarity index 78% rename from internal/controllers/netv1beta1/ingress_controller.go rename to internal/controllers/ingress/netv1beta1/ingress_controller.go index 4e953cf1..b627cdbc 100644 --- a/internal/controllers/netv1beta1/ingress_controller.go +++ b/internal/controllers/ingress/netv1beta1/ingress_controller.go @@ -14,28 +14,52 @@ package netv1beta1 import ( "context" + "fmt" + corev1 "k8s.io/api/core/v1" netv1 "k8s.io/api/networking/v1" netv1beta1 "k8s.io/api/networking/v1beta1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/tools/record" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/builder" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/manager" "sigs.k8s.io/controller-runtime/pkg/reconcile" "github.com/bfenetworks/ingress-bfe/internal/bfeConfig" "github.com/bfenetworks/ingress-bfe/internal/bfeConfig/annotations" + "github.com/bfenetworks/ingress-bfe/internal/controllers/event" "github.com/bfenetworks/ingress-bfe/internal/controllers/filter" - controllerV1 "github.com/bfenetworks/ingress-bfe/internal/controllers/netv1" + controllerV1 "github.com/bfenetworks/ingress-bfe/internal/controllers/ingress/netv1" ) +func AddIngressController(mgr manager.Manager, cb *bfeConfig.ConfigBuilder) error { + reconciler := newIngressReconciler(mgr, cb) + if err := reconciler.setupWithManager(mgr); err != nil { + return fmt.Errorf("unable to create ingress controller") + } + + return nil +} + // IngressReconciler reconciles a netv1beta1 Ingress object type IngressReconciler struct { BfeConfigBuilder *bfeConfig.ConfigBuilder client.Client - Scheme *runtime.Scheme + Scheme *runtime.Scheme + recorder record.EventRecorder +} + +func newIngressReconciler(mgr manager.Manager, cb *bfeConfig.ConfigBuilder) *IngressReconciler { + return &IngressReconciler{ + BfeConfigBuilder: cb, + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + recorder: mgr.GetEventRecorderFor("bfe-ingress-controller"), + } } func (r *IngressReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { @@ -50,7 +74,7 @@ func (r *IngressReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct log.V(1).Info("reconcile: ingress delete") return reconcile.Result{}, nil } - if !filter.MatchIngressClass(ctx, r, ingressV1beta1.Annotations, ingressV1beta1.Spec.IngressClassName) { + if !filter.IngressClassFilter(ctx, r, ingressV1beta1.Annotations, ingressV1beta1.Spec.IngressClassName) { return reconcile.Result{}, nil } @@ -61,11 +85,18 @@ func (r *IngressReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct err = controllerV1.ReconcileV1Ingress(ctx, r.Client, r.BfeConfigBuilder, ingressV1) setStatus(ctx, r.Client, err, ingressV1beta1) + + if err != nil { + r.recorder.Event(ingressV1beta1, corev1.EventTypeWarning, event.SyncFailed, err.Error()) + } else { + r.recorder.Event(ingressV1beta1, corev1.EventTypeNormal, event.SyncSucceed, "Synced") + } + return reconcile.Result{}, err } -// SetupWithManager sets up the controller with the Manager. -func (r *IngressReconciler) SetupWithManager(mgr ctrl.Manager) error { +// setupWithManager sets up the controller with the Manager. +func (r *IngressReconciler) setupWithManager(mgr ctrl.Manager) error { return ctrl.NewControllerManagedBy(mgr). For(&netv1beta1.Ingress{}, builder.WithPredicates(filter.NamespaceFilter())). Complete(r) diff --git a/internal/controllers/corev1/secret_controller.go b/internal/controllers/ingress/secret_controller.go similarity index 71% rename from internal/controllers/corev1/secret_controller.go rename to internal/controllers/ingress/secret_controller.go index bc22f057..048cb260 100644 --- a/internal/controllers/corev1/secret_controller.go +++ b/internal/controllers/ingress/secret_controller.go @@ -12,10 +12,11 @@ // See the License for the specific language governing permissions and // limitations under the License. -package corev1 +package ingress import ( "context" + "fmt" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" @@ -23,11 +24,21 @@ import ( "sigs.k8s.io/controller-runtime/pkg/builder" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/manager" "github.com/bfenetworks/ingress-bfe/internal/bfeConfig" "github.com/bfenetworks/ingress-bfe/internal/controllers/filter" ) +func AddSecretController(mgr manager.Manager, cb *bfeConfig.ConfigBuilder) error { + reconciler := newSecretReconciler(mgr, cb) + if err := reconciler.setupWithManager(mgr); err != nil { + return fmt.Errorf("unable to create ingress controller") + } + + return nil +} + // SecretReconciler reconciles a Secret object type SecretReconciler struct { BfeConfigBuilder *bfeConfig.ConfigBuilder @@ -36,6 +47,14 @@ type SecretReconciler struct { Scheme *runtime.Scheme } +func newSecretReconciler(mgr manager.Manager, cb *bfeConfig.ConfigBuilder) *SecretReconciler { + return &SecretReconciler{ + BfeConfigBuilder: cb, + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + } +} + func (r *SecretReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { log := log.FromContext(ctx) log.V(1).Info("reconciling Secret", "api version", "corev1") @@ -54,8 +73,8 @@ func (r *SecretReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctr return ctrl.Result{}, nil } -// SetupWithManager sets up the controller with the Manager. -func (r *SecretReconciler) SetupWithManager(mgr ctrl.Manager) error { +// setupWithManager sets up the controller with the Manager. +func (r *SecretReconciler) setupWithManager(mgr ctrl.Manager) error { return ctrl.NewControllerManagedBy(mgr). For(&corev1.Secret{}, builder.WithPredicates(filter.NamespaceFilter())). Complete(r) diff --git a/internal/controllers/corev1/endpoints_controller.go b/internal/controllers/ingress/service_controller.go similarity index 50% rename from internal/controllers/corev1/endpoints_controller.go rename to internal/controllers/ingress/service_controller.go index 33081c24..94caf29b 100644 --- a/internal/controllers/corev1/endpoints_controller.go +++ b/internal/controllers/ingress/service_controller.go @@ -12,48 +12,71 @@ // See the License for the specific language governing permissions and // limitations under the License. -package corev1 +package ingress import ( "context" + "fmt" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/builder" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/handler" "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + "sigs.k8s.io/controller-runtime/pkg/source" "github.com/bfenetworks/ingress-bfe/internal/bfeConfig" "github.com/bfenetworks/ingress-bfe/internal/controllers/filter" ) -// EndpointsReconciler reconciles a Endpoints object -type EndpointsReconciler struct { +func AddServiceController(mgr manager.Manager, cb *bfeConfig.ConfigBuilder) error { + reconciler := newServiceReconciler(mgr, cb) + if err := reconciler.setupWithManager(mgr); err != nil { + return fmt.Errorf("unable to create ingress controller") + } + + return nil +} + +// ServiceReconciler reconciles a Service/Endpoints object +type ServiceReconciler struct { BfeConfigBuilder *bfeConfig.ConfigBuilder client.Client Scheme *runtime.Scheme } -func (r *EndpointsReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { +func newServiceReconciler(mgr manager.Manager, cb *bfeConfig.ConfigBuilder) *ServiceReconciler { + return &ServiceReconciler{ + BfeConfigBuilder: cb, + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + } +} + +func (r *ServiceReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { log := log.FromContext(ctx) - log.V(1).Info("reconciling endpoints", "api version", "corev1") + log.V(1).Info("reconciling service", "api version", "corev1") - ep := &corev1.Endpoints{} + svc := &corev1.Service{} err := r.Get(ctx, client.ObjectKey{ Namespace: req.Namespace, Name: req.Name, - }, ep) + }, svc) if err != nil { return ctrl.Result{}, nil } - svc := &corev1.Service{} + ep := &corev1.Endpoints{} err = r.Get(ctx, client.ObjectKey{ Namespace: req.Namespace, Name: req.Name, - }, svc) + }, ep) if err != nil { return ctrl.Result{}, nil } @@ -63,9 +86,21 @@ func (r *EndpointsReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( return ctrl.Result{}, nil } -// SetupWithManager sets up the controller with the Manager. -func (r *EndpointsReconciler) SetupWithManager(mgr ctrl.Manager) error { +// setupWithManager sets up the controller with the Manager. +func (r *ServiceReconciler) setupWithManager(mgr ctrl.Manager) error { return ctrl.NewControllerManagedBy(mgr). - For(&corev1.Endpoints{}, builder.WithPredicates(filter.NamespaceFilter())). + For(&corev1.Service{}, builder.WithPredicates(filter.NamespaceFilter())). + Watches( + &source.Kind{Type: &corev1.Endpoints{}}, + handler.EnqueueRequestsFromMapFunc(func(a client.Object) []reconcile.Request { + return []reconcile.Request{ + {NamespacedName: types.NamespacedName{ + Name: a.GetName(), + Namespace: a.GetNamespace(), + }}, + } + }), + builder.WithPredicates(filter.NamespaceFilter()), + ). Complete(r) } diff --git a/internal/controllers/start.go b/internal/controllers/start.go index e0d86954..f66ca9e3 100644 --- a/internal/controllers/start.go +++ b/internal/controllers/start.go @@ -15,26 +15,32 @@ package controllers import ( + "context" "fmt" + "os" + "os/exec" + "path/filepath" + "syscall" "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/discovery" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/healthz" + "sigs.k8s.io/controller-runtime/pkg/manager" "github.com/bfenetworks/ingress-bfe/internal/bfeConfig" - "github.com/bfenetworks/ingress-bfe/internal/controllers/corev1" - extensionsv1beta1 "github.com/bfenetworks/ingress-bfe/internal/controllers/extv1beta1" - "github.com/bfenetworks/ingress-bfe/internal/controllers/netv1" - "github.com/bfenetworks/ingress-bfe/internal/controllers/netv1beta1" - option "github.com/bfenetworks/ingress-bfe/internal/option" + "github.com/bfenetworks/ingress-bfe/internal/controllers/ingress" + "github.com/bfenetworks/ingress-bfe/internal/controllers/ingress/extv1beta1" + "github.com/bfenetworks/ingress-bfe/internal/controllers/ingress/netv1" + "github.com/bfenetworks/ingress-bfe/internal/controllers/ingress/netv1beta1" + "github.com/bfenetworks/ingress-bfe/internal/option" ) var ( log = ctrl.Log.WithName("controllers") ) -func Start(scheme *runtime.Scheme, configBuilder *bfeConfig.ConfigBuilder) error { +func Start(scheme *runtime.Scheme) error { config, err := ctrl.GetConfig() if err != nil { return fmt.Errorf("unable to get client config: %s", err) @@ -49,22 +55,42 @@ func Start(scheme *runtime.Scheme, configBuilder *bfeConfig.ConfigBuilder) error return fmt.Errorf("unable to start controller manager: %s", err) } - if err = (&corev1.EndpointsReconciler{ - BfeConfigBuilder: configBuilder, - Client: mgr.GetClient(), - Scheme: mgr.GetScheme(), - }).SetupWithManager(mgr); err != nil { - return fmt.Errorf("unable to create controller Endpoints: %s", err) + if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { + return fmt.Errorf("unable to set up health check: %s", err) + } + if err := mgr.AddReadyzCheck("readyz", healthz.Ping); err != nil { + return fmt.Errorf("unable to set up ready check: %s", err) } - if err = (&corev1.SecretReconciler{ - BfeConfigBuilder: configBuilder, - Client: mgr.GetClient(), - Scheme: mgr.GetScheme(), - }).SetupWithManager(mgr); err != nil { - return fmt.Errorf("unable to create controller secret: %s", err) + ctx := ctrl.SetupSignalHandler() + + // new bfe config builder + cb := bfeConfig.NewConfigBuilder() + cb.InitReload(ctx) + + // add controller to watch ingress resource + if err := addController(cb, mgr); err != nil { + return err } + // start bfe process + if err := startBFE(ctx); err != nil { + return err + } + + log.Info("starting manager") + + // start controller manager and blocking + if err := mgr.Start(ctx); err != nil { + return fmt.Errorf("fail to run manager: %s", err) + } + + log.Info("exit manager") + + return nil +} + +func addController(cb *bfeConfig.ConfigBuilder, mgr manager.Manager) error { client := discovery.NewDiscoveryClientForConfigOrDie(ctrl.GetConfigOrDie()) serverVersion, err := client.ServerVersion() if err != nil { @@ -72,43 +98,59 @@ func Start(scheme *runtime.Scheme, configBuilder *bfeConfig.ConfigBuilder) error } if serverVersion.Major >= "1" && serverVersion.Minor >= "19" { - if err = (&netv1.IngressReconciler{ - BfeConfigBuilder: configBuilder, - Client: mgr.GetClient(), - Scheme: mgr.GetScheme(), - }).SetupWithManager(mgr); err != nil { + if err = netv1.AddIngressController(mgr, cb); err != nil { return fmt.Errorf("unable to create controller Ingress(netwokingv1): %s", err) } } else if serverVersion.Major >= "1" && serverVersion.Minor >= "14" { - if err = (&netv1beta1.IngressReconciler{ - BfeConfigBuilder: configBuilder, - Client: mgr.GetClient(), - Scheme: mgr.GetScheme(), - }).SetupWithManager(mgr); err != nil { + if err = netv1beta1.AddIngressController(mgr, cb); err != nil { return fmt.Errorf("unable to create controller Ingress(netwokingv1beta1): %s", err) } } else { - if err = (&extensionsv1beta1.IngressReconciler{ - BfeConfigBuilder: configBuilder, - Client: mgr.GetClient(), - Scheme: mgr.GetScheme(), - }).SetupWithManager(mgr); err != nil { + if err = extv1beta1.AddIngressController(mgr, cb); err != nil { return fmt.Errorf("unable to create controller Ingress(extensionsv1beta1): %s", err) } } - if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { - return fmt.Errorf("unable to set up health check: %s", err) - } - if err := mgr.AddReadyzCheck("readyz", healthz.Ping); err != nil { - return fmt.Errorf("unable to set up ready check: %s", err) + if err := ingress.AddServiceController(mgr, cb); err != nil { + return fmt.Errorf("unable to create controller Service: %s", err) } - log.Info("starting manager") - if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil { - return fmt.Errorf("fail to run manager: %s", err) + if err := ingress.AddSecretController(mgr, cb); err != nil { + return fmt.Errorf("unable to create controller secret: %s", err) } - log.Info("existing manager") return nil } + +func startBFE(ctx context.Context) error { + cmd := exec.Command(option.Opts.Ingress.BfeBinary, "-c", "../conf", "-l", "../log", "-s") + cmd.Dir = filepath.Dir(option.Opts.Ingress.BfeBinary) + + log.Info("bfe is starting") + + err := cmd.Start() + if err != nil { + return fmt.Errorf("fail to start bfe: %s", err.Error()) + } + + go func() { + if err := cmd.Wait(); err != nil { + log.Error(err, "bfe exit") + } else { + log.Info("bfe exit") + } + + // bfe exit, signaling controller to exit + raise(syscall.SIGTERM) + }() + + return err +} + +func raise(sig os.Signal) error { + p, err := os.FindProcess(os.Getpid()) + if err != nil { + return err + } + return p.Signal(sig) +} diff --git a/internal/option/ingress/options.go b/internal/option/ingress/options.go new file mode 100644 index 00000000..5c43a48f --- /dev/null +++ b/internal/option/ingress/options.go @@ -0,0 +1,96 @@ +// Copyright (c) 2021 The BFE Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ingress + +import ( + "fmt" + "os" + "path/filepath" + "strings" + "time" + + "k8s.io/apimachinery/pkg/types" +) + +const ( + enableIngress = true + + configPath = "/bfe/conf/" + bfeBinary = "/bfe/bin/bfe" + reloadAddr = "localhost:8421" + reloadInterval = 3 * time.Second + reloadUrlPrefix = "http://%s/reload/" + + filePerm os.FileMode = 0744 + + // used in ingress annotation as value of key kubernetes.io/ingress.class + ingressClassName = "bfe" + + // used in IngressClass resource as value of controller + controllerName = "bfe-networks.com/ingress-controller" + + // default backend + defaultBackend = "" +) + +type Options struct { + EnableIngress bool + IngressClass string + ControllerName string + ReloadAddr string + ReloadUrl string + BfeBinary string + ConfigPath string + FilePerm os.FileMode + ReloadInterval time.Duration + DefaultBackend string +} + +func NewOptions() *Options { + return &Options{ + EnableIngress: enableIngress, + IngressClass: ingressClassName, + ControllerName: controllerName, + ReloadAddr: reloadAddr, + BfeBinary: bfeBinary, + ConfigPath: configPath, + FilePerm: filePerm, + ReloadInterval: reloadInterval, + DefaultBackend: defaultBackend, + } +} + +func (opts *Options) Check() error { + if !opts.EnableIngress { + return nil + } + + if len(opts.DefaultBackend) > 0 { + names := strings.Split(opts.DefaultBackend, string(types.Separator)) + if len(names) != 2 { + return fmt.Errorf("invalid command line argument default-backend: %s", opts.DefaultBackend) + } + } + if len(opts.BfeBinary) > 0 { + opts.ConfigPath = filepath.Dir(filepath.Dir(opts.BfeBinary)) + "/conf" + } + + if !strings.HasSuffix(opts.ConfigPath, "/") { + opts.ConfigPath = opts.ConfigPath + "/" + } + + opts.ReloadUrl = fmt.Sprintf(reloadUrlPrefix, opts.ReloadAddr) + return nil +} diff --git a/internal/option/options.go b/internal/option/options.go index faf4e737..bfe2f9e0 100644 --- a/internal/option/options.go +++ b/internal/option/options.go @@ -15,74 +15,51 @@ package option import ( - "fmt" - "os" "strings" - "time" - "k8s.io/apimachinery/pkg/types" + corev1 "k8s.io/api/core/v1" + + "github.com/bfenetworks/ingress-bfe/internal/option/ingress" ) const ( - ConfigPath = "/bfe/conf/" - ReloadAddr = "localhost:8421" - reloadInterval = 3 * time.Second - reloadUrlPrefix = "http://%s/reload/" - - FilePerm os.FileMode = 0744 - + ClusterName = "default" MetricsBindAddress = ":9080" HealthProbeBindAddress = ":9081" - - // used in ingress annotation as value of key kubernetes.io/ingress.class - IngressClassName = "bfe" - - // used in IngressClass resource as value of controller - ControllerName = "bfe-networks.com/ingress-controller" - - // default backend - DefaultBackend = "" ) type Options struct { - Namespaces []string - IngressClass string - ControllerName string - ReloadUrl string - ConfigPath string + ClusterName string + + Namespaces string + NamespaceList []string MetricsAddr string HealthProbeAddr string - ReloadInterval time.Duration - DefaultBackend string + + Ingress *ingress.Options } var ( Opts *Options ) -func SetOptions(namespaces, class, configPath, reloadAddr, metricsAddr, probeAddr, defaultBackend string) error { - if len(defaultBackend) > 0 { - names := strings.Split(defaultBackend, string(types.Separator)) - if len(names) != 2 { - return fmt.Errorf("invalid command line argument default-backend: %s", defaultBackend) - } +func NewOptions() *Options { + return &Options{ + ClusterName: ClusterName, + Namespaces: corev1.NamespaceAll, + MetricsAddr: MetricsBindAddress, + HealthProbeAddr: HealthProbeBindAddress, + Ingress: ingress.NewOptions(), } +} - if !strings.HasSuffix(configPath, "/") { - configPath = configPath + "/" +func SetOptions(option *Options) error { + if err := option.Ingress.Check(); err != nil { + return err } - Opts = &Options{ - Namespaces: strings.Split(namespaces, ","), - IngressClass: class, - ControllerName: ControllerName, - ReloadUrl: fmt.Sprintf(reloadUrlPrefix, reloadAddr), - ConfigPath: configPath, - MetricsAddr: metricsAddr, - HealthProbeAddr: probeAddr, - ReloadInterval: reloadInterval, - DefaultBackend: defaultBackend, - } + Opts = option + Opts.NamespaceList = strings.Split(Opts.Namespaces, ",") return nil } diff --git a/test/e2e/Dockerfile b/test/e2e/Dockerfile new file mode 100644 index 00000000..0da33e9f --- /dev/null +++ b/test/e2e/Dockerfile @@ -0,0 +1,43 @@ +# Copyright 2020 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Build +FROM golang:1.16-alpine3.14 as builder + +WORKDIR /go/src/github.com/bfenetworks/ingress-bfe +# Copy the Go Modules manifests +COPY go.mod go.mod +COPY go.sum go.sum +# cache deps before building and copying source so that we don't need to re-download as much +# and so that source changes don't invalidate our downloaded layer +RUN go mod download + +# Copy the go source +COPY . /go/src/github.com/bfenetworks/ingress-bfe + +# Build +RUN make e2e_test + +FROM alpine3.14 + +ENV RESULTS_DIR="/tmp/results" +ENV WAIT_FOR_STATUS_TIMEOUT="5m" +ENV TEST_TIMEOUT="5m" + +COPY --from=builder /go/src/github.com/bfenetworks/ingress-bfe/e2e_test / + +COPY features /features +COPY run.sh / + +CMD [ "/run.sh" ] diff --git a/test/e2e/Makefile b/test/e2e/Makefile new file mode 100644 index 00000000..93d88ced --- /dev/null +++ b/test/e2e/Makefile @@ -0,0 +1,66 @@ +# Copyright 2020 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +.DEFAULT_GOAL:=help + +MKDIR_P := mkdir -p +RM_F := rm -rf + + +PROGRAMS := \ + e2e_test + +TAG ?= 0.0.1 + +REGISTRY ?= local + +build: $(PROGRAMS) ## Build the e2e-test binary + +.PHONY: build-image +build-image: ## Build the e2e-test image + docker build -t $(REGISTRY)/e2e_test:$(TAG) . + +.PHONY: publish-image +publish-image: + docker push $(REGISTRY)/e2e_test:$(TAG) + +.PHONY: e2e_test +e2e_test: check-go-version + @CGO_ENABLED=0 go test -c -trimpath -ldflags="-buildid= -w" -o $@ . + +.PHONY: clean +clean: ## Remove build artifacts + $(RM_F) internal/pkg/assets/assets.go + $(RM_F) $(PROGRAMS) + +.PHONY: codegen +codegen: check-go-version ## Generate or update missing Go code defined in feature files + @go run hack/codegen.go -update -dest-path=steps/conformance features/conformance + +.PHONY: verify-codegen +verify-codegen: check-go-version ## Verify if generated Go code is in sync with feature files + @go run hack/codegen.go -dest-path=steps/conformance features + +.PHONY: verify-gherkin +verify-gherkin: check-go-version ## Verify format of gherkin feature files + @hack/verify-gherkin.sh + +.PHONY: help +help: ## Display this help + @echo Targets: + @awk 'BEGIN {FS = ":.*?## "} /^[a-zA-Z0-9._-]+:.*?## / {printf " %-20s %s\n", $$1, $$2}' $(MAKEFILE_LIST) | sort + +.PHONY: check-go-version +check-go-version: + @hack/check-go-version.sh diff --git a/test/e2e/README.md b/test/e2e/README.md new file mode 100644 index 00000000..849c23ec --- /dev/null +++ b/test/e2e/README.md @@ -0,0 +1,78 @@ +# BFE ingress controller e2e test + +This test follows K8s project [ingress-controller-conformance](https://github.com/kubernetes-sigs/ingress-controller-conformance), and add more test cases for ingress-bfe special features. + +## How to run + +To run all e2e test cases, execute following command in ingress-bfe project's top directory: + +``` +$ make e2e-test +``` +It would automatically start the whole testing with following procedures: + +- Build bfe-ingress-controller docker image +- Prepare test environment, including spining up a local k8s cluster with [Kind](https://kind.sigs.k8s.io/), loading docker images, etc. All scripts used to prepare environment are located in [test/script](../script). +- Execute test cases by running [run.sh](./run.sh), which actually build and execute program e2e_test. + +## Contributing + +We encourage contributors write e2e test case for new feature needed to be merged into ingress-bfe. + +The test code is based on BDD testing framework [godog](https://github.com/cucumber/godog), a testing framework of [cucumber](https://cucumber.io/). It uses [Gherkin Syntax]( https://cucumber.io/docs/gherkin/reference/) to describe test case. + +Steps to add new test case as below: + +### Step1: Create Gherkin feature + +* Create feature file to describe your test case. + +All feature files are under directory [test/e2e/features](./features). Please put your feature file into proper sub-directory. For example, features//.feature + > Try to reuse steps from existing feature files if possible. + +### Step2: Create steps definition + +* Generate steps.go for your case. Under directory test/e2e, run: + +```bash +$ go run hack/codegen.go -dest-path=steps/ features//.feature +``` + + +* Edit generated code, implement all generated functions. If you reuse step description from other feature file, you can also reuse corresponding function from that `step.go` file in this step. + +### Step3: Add Init function into e2e_test.go + +* In e2e_test.go, add generated feature file and InitializeScenario function into map `features`. + +```go +var ( + features = map[string]InitialFunc{ + "features/conformance/host_rules.feature": {hostrules.InitializeScenario, nil}, + ... + } +) + +``` + +### Step4: Build and run + + + +* Build e2e_test: + +```bash +$ make build +``` + +* Run your case, using `--feature` to specify the feature file. + +```bash +$ ./e2e_test --feature features//.feature +``` +> Before running, your testing environment must be ready. + +* Run all cases +```bash +$ ./run.sh +``` diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go new file mode 100644 index 00000000..2bb79269 --- /dev/null +++ b/test/e2e/e2e_test.go @@ -0,0 +1,235 @@ +/* +Copyright 2020 The BFE Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "bufio" + "flag" + "fmt" + "io" + "os" + "os/signal" + "path" + "path/filepath" + "strings" + "syscall" + "testing" + "time" + + "github.com/cucumber/godog" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/klog/v2" + + "github.com/bfenetworks/ingress-bfe/test/e2e/pkg/http" + "github.com/bfenetworks/ingress-bfe/test/e2e/pkg/kubernetes" + "github.com/bfenetworks/ingress-bfe/test/e2e/pkg/kubernetes/templates" + "github.com/bfenetworks/ingress-bfe/test/e2e/steps/annotations/balance/loadbalance" + "github.com/bfenetworks/ingress-bfe/test/e2e/steps/annotations/route/cookie" + "github.com/bfenetworks/ingress-bfe/test/e2e/steps/annotations/route/header" + "github.com/bfenetworks/ingress-bfe/test/e2e/steps/annotations/route/priority" + "github.com/bfenetworks/ingress-bfe/test/e2e/steps/conformance/hostrules" + "github.com/bfenetworks/ingress-bfe/test/e2e/steps/conformance/ingressclass" + "github.com/bfenetworks/ingress-bfe/test/e2e/steps/conformance/loadbalancing" + "github.com/bfenetworks/ingress-bfe/test/e2e/steps/conformance/pathrules" + "github.com/bfenetworks/ingress-bfe/test/e2e/steps/rules/host1" + "github.com/bfenetworks/ingress-bfe/test/e2e/steps/rules/host2" + "github.com/bfenetworks/ingress-bfe/test/e2e/steps/rules/multipleingress" + "github.com/bfenetworks/ingress-bfe/test/e2e/steps/rules/path1" + "github.com/bfenetworks/ingress-bfe/test/e2e/steps/rules/path2" + "github.com/bfenetworks/ingress-bfe/test/e2e/steps/rules/patherr" +) + +var ( + godogFormat string + godogTags string + godogStopOnFailure bool + godogNoColors bool + godogOutput string + godogTestFeature string + FeatureParallel int +) + +func TestMain(m *testing.M) { + // register flags from klog (client-go verbose logging) + klog.InitFlags(nil) + + flag.StringVar(&godogFormat, "format", "pretty", "Set godog format to use. Valid values are pretty and cucumber") + flag.StringVar(&godogTags, "tags", "", "Tags for e2e test") + flag.BoolVar(&godogStopOnFailure, "stop-on-failure", false, "Stop when failure is found") + flag.BoolVar(&godogNoColors, "no-colors", false, "Disable colors in godog output") + flag.StringVar(&godogOutput, "output-directory", ".", "Output directory for test reports") + flag.StringVar(&kubernetes.IngressClassValue, "ingress-class", "bfe", "Sets the value of the annotation kubernetes.io/ingress.class in Ingress definitions") + flag.DurationVar(&kubernetes.WaitForIngressAddressTimeout, "wait-time-for-ingress-status", 3*time.Minute, "Maximum wait time for valid ingress status value") + flag.DurationVar(&kubernetes.WaitForEndpointsTimeout, "wait-time-for-ready", 3*time.Minute, "Maximum wait time for ready endpoints") + flag.StringVar(&kubernetes.IngressControllerNameSpace, "ingress-controller-namespace", "ingress-bfe", "Sets the value of the namespace for ingress controller") + flag.StringVar(&kubernetes.IngressControllerServiceName, "ingress-controller-service-name", "bfe-controller-service", "Sets the name of the service for ingress controller") + flag.StringVar(&kubernetes.K8sNodeAddr, "k8s-node-addr", "127.0.0.1", "Sets the ip address of one k8s node") + flag.StringVar(&godogTestFeature, "feature", "", "Sets the file to test") + flag.IntVar(&FeatureParallel, "feature-parallel", 1, "Sets the file to test") + flag.BoolVar(&http.EnableDebug, "enable-http-debug", false, "Enable dump of requests and responses of HTTP requests (useful for debug)") + flag.BoolVar(&kubernetes.EnableOutputYamlDefinitions, "enable-output-yaml-definitions", false, "Dump yaml definitions of Kubernetes objects before creation") + + flag.Parse() + + validFormats := sets.NewString("cucumber", "pretty") + if !validFormats.Has(godogFormat) { + klog.Fatalf("the godog format '%v' is not supported", godogFormat) + } + + err := setup() + if err != nil { + klog.Fatal(err) + } + + if err := kubernetes.CleanupNamespaces(kubernetes.KubeClient); err != nil { + klog.Fatalf("error deleting temporal namespaces: %v", err) + } + + go handleSignals() + + os.Exit(m.Run()) +} + +func setup() error { + err := templates.Load() + if err != nil { + return fmt.Errorf("error loading templates: %v", err) + } + + kubernetes.KubeClient, err = kubernetes.LoadClientset() + if err != nil { + return fmt.Errorf("error loading client: %v", err) + } + + return nil +} + +type InitialFunc struct { + Scenario func(*godog.ScenarioContext) + Suite func(*godog.TestSuiteContext) +} + +var ( + features = map[string]InitialFunc{ + "features/conformance/host_rules.feature": {hostrules.InitializeScenario, nil}, + "features/conformance/ingress_class.feature": {ingressclass.InitializeScenario, nil}, + "features/conformance/load_balancing.feature": {loadbalancing.InitializeScenario, nil}, + "features/conformance/path_rules.feature": {pathrules.InitializeScenario, pathrules.InitializeSuite}, + "features/rules/host_rule1.feature": {host1.InitializeScenario, nil}, + "features/rules/host_rule2.feature": {host2.InitializeScenario, nil}, + "features/rules/multiple_ingress.feature": {multipleingress.InitializeScenario, nil}, + "features/rules/path_rule1.feature": {path1.InitializeScenario, path1.InitializeSuite}, + "features/rules/path_rule2.feature": {path2.InitializeScenario, path2.InitializeSuite}, + "features/rules/path_err.feature": {patherr.InitializeScenario, nil}, + "features/annotations/route/cookie.feature": {cookie.InitializeScenario, nil}, + "features/annotations/route/header.feature": {header.InitializeScenario, nil}, + "features/annotations/route/priority.feature": {priority.InitializeScenario, nil}, + "features/annotations/balance/load_balance.feature": {loadbalance.InitializeScenario, nil}, + } +) + +func TestSuite(t *testing.T) { + var failed bool + + activeFeatures := make(map[string]InitialFunc) + for file, init := range features { + if strings.HasPrefix(file, godogTestFeature) { + activeFeatures[file] = init + } + } + + queue := make(chan int, FeatureParallel) + + for i := 0; i < FeatureParallel; i++ { + queue <- 1 + } + + for feature, initFunc := range activeFeatures { + <-queue + go func(feature string, init InitialFunc) { + err := testFeature(feature, init) + if err != nil { + failed = true + } + queue <- 1 + }(feature, initFunc) + } + + for i := 0; i < FeatureParallel; i++ { + <-queue + } + + if failed { + t.Fatal("at least one step/scenario failed") + } + +} + +func testFeature(feature string, initFunc InitialFunc) error { + var testOutput io.Writer + // default output is stdout + testOutput = os.Stdout + + if godogFormat == "cucumber" { + rf := path.Join(godogOutput, fmt.Sprintf("%v-report.json", filepath.Base(feature))) + file, err := os.Create(rf) + if err != nil { + return fmt.Errorf("error creating report file %v: %w", rf, err) + } + + defer file.Close() + + writer := bufio.NewWriter(file) + defer writer.Flush() + + testOutput = writer + } + + opts := godog.Options{ + Format: godogFormat, + Paths: []string{feature}, + Tags: godogTags, + StopOnFailure: godogStopOnFailure, + NoColors: godogNoColors, + Output: testOutput, + Concurrency: 1, // do not run tests concurrently + } + + exitCode := godog.TestSuite{ + Name: "e2e-test", + TestSuiteInitializer: initFunc.Suite, + ScenarioInitializer: initFunc.Scenario, + Options: &opts, + }.Run() + if exitCode > 0 { + return fmt.Errorf("unexpected exit code testing %v: %v", feature, exitCode) + } + + return nil +} + +func handleSignals() { + signals := make(chan os.Signal, 1) + signal.Notify(signals, syscall.SIGINT, syscall.SIGTERM) + <-signals + + if err := kubernetes.CleanupNamespaces(kubernetes.KubeClient); err != nil { + klog.Fatalf("error deleting temporal namespaces: %v", err) + } + + os.Exit(1) +} diff --git a/test/e2e/features/annotations/balance/load_balance.feature b/test/e2e/features/annotations/balance/load_balance.feature new file mode 100644 index 00000000..56e14e68 --- /dev/null +++ b/test/e2e/features/annotations/balance/load_balance.feature @@ -0,0 +1,243 @@ +@annotations @balance.weight @release-1.22 +Feature: Load balance rules + + Scenario: An Ingress with load balance rule 50-50 should send traffic to the matching backend service + (exact / and cookie matches request /) + Given an Ingress with service info "whoami1:3000|whoami2:3000" resource in a new random namespace + """ + apiVersion: networking.k8s.io/v1 + kind: Ingress + metadata: + name: balance-rules + annotations: + bfe.ingress.kubernetes.io/balance.weight: "{\"service\":{\"whoami1\":50, \"whoami2\":50}}" + spec: + rules: + - host: "balance-weight-50-50" + http: + paths: + - path: / + pathType: Exact + backend: + service: + name: service + port: + number: 3000 + """ + And The Ingress status shows the IP address or FQDN where it is exposed + When I send 100 "GET" requests to "http://balance-weight-50-50/" + Then the response status-code must be 200 the response body should contain the IP address of 2 different Kubernetes pods + And the response must be served by one of "whoami1|whoami2" service + + Scenario: An Ingress with load balance rule 1-1 should send traffic to the matching backend service + (exact / and cookie matches request /) + Given an Ingress with service info "whoami1:3000|whoami2:3000" resource in a new random namespace + """ + apiVersion: networking.k8s.io/v1 + kind: Ingress + metadata: + name: balance-rules + annotations: + bfe.ingress.kubernetes.io/balance.weight: "{\"service\":{\"whoami1\":1, \"whoami2\":1}}" + spec: + rules: + - host: "balance-weight-1-1" + http: + paths: + - path: / + pathType: Exact + backend: + service: + name: service + port: + number: 3000 + """ + And The Ingress status shows the IP address or FQDN where it is exposed + When I send 100 "GET" requests to "http://balance-weight-1-1/" + Then the response status-code must be 200 the response body should contain the IP address of 2 different Kubernetes pods + And the response must be served by one of "whoami1|whoami2" service + + Scenario: An Ingress with load balance rule 3-1 should send traffic to the matching backend service + (exact / and cookie matches request /) + Given an Ingress with service info "whoami1:3000|whoami2:3000" resource in a new random namespace + """ + apiVersion: networking.k8s.io/v1 + kind: Ingress + metadata: + name: balance-rules + annotations: + bfe.ingress.kubernetes.io/balance.weight: "{\"service\":{\"whoami1\":3, \"whoami2\":1}}" + spec: + rules: + - host: "balance-weight-3-1" + http: + paths: + - path: / + pathType: Exact + backend: + service: + name: service + port: + number: 3000 + """ + And The Ingress status shows the IP address or FQDN where it is exposed + When I send 100 "GET" requests to "http://balance-weight-3-1/" + Then the response status-code must be 200 the response body should contain the IP address of 2 different Kubernetes pods + And the response must be served by one of "whoami1|whoami2" service + + Scenario: An Ingress with load balance rule 1-1-1 should send traffic to the matching backend service + (exact / and cookie matches request /) + Given an Ingress with service info "whoami1:3000|whoami2:3000|whoami3:3000" resource in a new random namespace + """ + apiVersion: networking.k8s.io/v1 + kind: Ingress + metadata: + name: balance-rules + annotations: + bfe.ingress.kubernetes.io/balance.weight: "{\"service\":{\"whoami1\":1, \"whoami2\":1, \"whoami3\":1}}" + spec: + rules: + - host: "balance-weight-1-1-1" + http: + paths: + - path: / + pathType: Exact + backend: + service: + name: service + port: + number: 3000 + """ + And The Ingress status shows the IP address or FQDN where it is exposed + When I send 100 "GET" requests to "http://balance-weight-1-1-1/" + Then the response status-code must be 200 the response body should contain the IP address of 3 different Kubernetes pods + And the response must be served by one of "whoami1|whoami2|whoami3" service + + Scenario: An Ingress with load balance rule 200-177-121 should send traffic to the matching backend service + (exact / and cookie matches request /) + Given an Ingress with service info "whoami1:3000|whoami2:3000|whoami3:3000" resource in a new random namespace + """ + apiVersion: networking.k8s.io/v1 + kind: Ingress + metadata: + name: balance-rules + annotations: + bfe.ingress.kubernetes.io/balance.weight: "{\"service\":{\"whoami1\":200, \"whoami2\":177, \"whoami3\":121}}" + spec: + rules: + - host: "balance-weight-200-177-121" + http: + paths: + - path: / + pathType: Exact + backend: + service: + name: service + port: + number: 3000 + """ + And The Ingress status shows the IP address or FQDN where it is exposed + When I send 100 "GET" requests to "http://balance-weight-200-177-121/" + Then the response status-code must be 200 the response body should contain the IP address of 3 different Kubernetes pods + And the response must be served by one of "whoami1|whoami2|whoami3" service + + Scenario: An Ingress with load balance rule format error should not send traffic to the matching backend service + (exact / and cookie matches request /) + Given an Ingress with service info "whoami1:3000|whoami2:3000" resource in a new random namespace + """ + apiVersion: networking.k8s.io/v1 + kind: Ingress + metadata: + name: balance-rules + annotations: + bfe.ingress.kubernetes.io/balance.weight: "{\"service\":{\"whoami1\":50,, \"whoami2\":50}}" + spec: + rules: + - host: "balance-weight-50-50" + http: + paths: + - path: / + pathType: Exact + backend: + service: + name: service + port: + number: 3000 + """ + And The Ingress status should not be success + + Scenario: An Ingress with load balance rule 0-0-0 should not send traffic to the matching backend service + (exact / and cookie matches request /) + Given an Ingress with service info "whoami1:3000|whoami2:3000|whoami3:3000" resource in a new random namespace + """ + apiVersion: networking.k8s.io/v1 + kind: Ingress + metadata: + name: balance-rules + annotations: + bfe.ingress.kubernetes.io/balance.weight: "{\"service\":{\"whoami1\":0, \"whoami2\":0, \"whoami3\":0}}" + spec: + rules: + - host: "balance-weight-0-0-0" + http: + paths: + - path: / + pathType: Exact + backend: + service: + name: service + port: + number: 3000 + """ + And The Ingress status should not be success + + Scenario: An Ingress with load balance rule 1-1-(-1) should not send traffic to the matching backend service + (exact / and cookie matches request /) + Given an Ingress with service info "whoami1:3000|whoami2:3000|whoami3:3000" resource in a new random namespace + """ + apiVersion: networking.k8s.io/v1 + kind: Ingress + metadata: + name: balance-rules + annotations: + bfe.ingress.kubernetes.io/balance.weight: "{\"service\":{\"whoami1\":1, \"whoami2\":1, \"whoami3\":-1}}" + spec: + rules: + - host: "balance-weight-1-1-mimus-1" + http: + paths: + - path: / + pathType: Exact + backend: + service: + name: service + port: + number: 3000 + """ + And The Ingress status should not be success + + Scenario: An Ingress with load balance rule 0.33-0.33-0.34 should not send traffic to the matching backend service + (exact / and cookie matches request /) + Given an Ingress with service info "whoami1:3000|whoami2:3000|whoami3:3000" resource in a new random namespace + """ + apiVersion: networking.k8s.io/v1 + kind: Ingress + metadata: + name: balance-rules + annotations: + bfe.ingress.kubernetes.io/balance.weight: "{\"service\":{\"whoami1\":0.33, \"whoami2\":0.33, \"whoami3\":0.34}}" + spec: + rules: + - host: "balance-weight-0.33-0.33-0.34" + http: + paths: + - path: / + pathType: Exact + backend: + service: + name: service + port: + number: 3000 + """ + And The Ingress status should not be success + diff --git a/test/e2e/features/annotations/route/cookie.feature b/test/e2e/features/annotations/route/cookie.feature new file mode 100644 index 00000000..bcfe9270 --- /dev/null +++ b/test/e2e/features/annotations/route/cookie.feature @@ -0,0 +1,243 @@ +@annotations @router.cookie @release-1.22 +Feature: Cookie rules + An Ingress may define routing rules based on the request path. + + If the HTTP request path matches the paths and cookie info in the + Ingress objects, the traffic is routed to its backend service. + + Scenario: An Ingress with path rules and cookie should send traffic to the matching backend service + (exact / and cookie matches request / with cookie) + Given an Ingress resource in a new random namespace + """ + apiVersion: networking.k8s.io/v1 + kind: Ingress + metadata: + name: cookie-rules + annotations: + bfe.ingress.kubernetes.io/router.cookie: "uid:abcd" + spec: + rules: + - host: "exact-path-rules-cookie-uid" + http: + paths: + - path: / + pathType: Exact + backend: + service: + name: foo-exact + port: + number: 3000 + """ + And The Ingress status shows the IP address or FQDN where it is exposed + When I send a "GET" request to "http://exact-path-rules-cookie-uid/" with header + """ + {"Cookie": ["uid=abcd"]} + """ + Then the response status-code must be 200 + And the response must be served by the "foo-exact" service + + Scenario: An Ingress with path rules and cookie should not send traffic to the matching backend service + (exact / and cookie dose not matches request / without cookie) + Given an Ingress resource in a new random namespace + """ + apiVersion: networking.k8s.io/v1 + kind: Ingress + metadata: + name: cookie-rules + annotations: + bfe.ingress.kubernetes.io/router.cookie: "uid:abcd" + spec: + rules: + - host: "exact-path-rules-cookie-uid-req-no-cookie" + http: + paths: + - path: / + pathType: Exact + backend: + service: + name: foo-exact + port: + number: 3000 + """ + And The Ingress status shows the IP address or FQDN where it is exposed + When I send a "GET" request to "http://exact-path-rules-cookie-uid-req-no-cookie/" + Then the response status-code must be 500 + + Scenario: An Ingress with path rules and cookie should send traffic to the matching backend service + (exact / and cookie config length 0 matches request / with cookie) + Given an Ingress resource in a new random namespace + """ + apiVersion: networking.k8s.io/v1 + kind: Ingress + metadata: + name: cookie-rules + annotations: + bfe.ingress.kubernetes.io/router.cookie: "" + spec: + rules: + - host: "exact-path-rules-cookie-length0" + http: + paths: + - path: / + pathType: Exact + backend: + service: + name: foo-exact-no-cookie + port: + number: 3000 + """ + And The Ingress status shows the IP address or FQDN where it is exposed + When I send a "GET" request to "http://exact-path-rules-cookie-length0/" with header + """ + {"Cookie": ["uid=abcd"]} + """ + Then the response status-code must be 200 + And the response must be served by the "foo-exact-no-cookie" service + + Scenario: An Ingress with path rules and cookie should send traffic to the matching backend service + (exact / and cookie matches request / with cookie) + Given an Ingress resource in a new random namespace + """ + apiVersion: networking.k8s.io/v1 + kind: Ingress + metadata: + name: cookie-rules + annotations: + bfe.ingress.kubernetes.io/router.cookie: "" + spec: + rules: + - host: "exact-path-rules-no-cookie" + http: + paths: + - path: / + pathType: Exact + backend: + service: + name: foo-exact-no-cookie + port: + number: 3000 + """ + And The Ingress status shows the IP address or FQDN where it is exposed + When I send a "GET" request to "http://exact-path-rules-no-cookie/" + Then the response status-code must be 200 + And the response must be served by the "foo-exact-no-cookie" service + + Scenario: An Ingress with path rules and cookie should send traffic to the matching backend service + (exact / and cookie "c_ref:https://www.baidu.com/link" matches request / with right cookie) + Given an Ingress resource in a new random namespace + """ + apiVersion: networking.k8s.io/v1 + kind: Ingress + metadata: + name: cookie-rules + annotations: + bfe.ingress.kubernetes.io/router.cookie: "c_ref:https://www.baidu.com/link" + spec: + rules: + - host: "exact-with-cookie-right" + http: + paths: + - path: / + pathType: Exact + backend: + service: + name: exact-cookie-right + port: + number: 3000 + """ + And The Ingress status shows the IP address or FQDN where it is exposed + When I send a "GET" request to "http://exact-with-cookie-right/" with header + """ + {"Cookie": ["c_ref=https://www.baidu.com/link"]} + """ + Then the response status-code must be 200 + And the response must be served by the "exact-cookie-right" service + + Scenario: An Ingress with path rules and error cookie should not send traffic to the matching backend service + (exact / and cookie "c_ref:https://www.baidu.com/" not matches request / with wrong cookie) + Given an Ingress resource in a new random namespace + """ + apiVersion: networking.k8s.io/v1 + kind: Ingress + metadata: + name: cookie-rules + annotations: + bfe.ingress.kubernetes.io/router.cookie: "c_ref:https://www.baidu.com/" + spec: + rules: + - host: "exact-with-cookie-wrong" + http: + paths: + - path: / + pathType: Exact + backend: + service: + name: exact-cookie-wrong + port: + number: 3000 + """ + And The Ingress status shows the IP address or FQDN where it is exposed + When I send a "GET" request to "http://exact-with-cookie-wrong/" with header + """ + {"Cookie": ["c_ref=https://www.baidu.com/link"]} + """ + Then the response status-code must be 500 + + Scenario: An Ingress with path rules and cookie should not send traffic to the matching backend service + (exact / and cookie "c_ref:https://www.baidu.com/" not matches request / with no cookie) + Given an Ingress resource in a new random namespace + """ + apiVersion: networking.k8s.io/v1 + kind: Ingress + metadata: + name: cookie-rules + annotations: + bfe.ingress.kubernetes.io/router.cookie: "c_ref:https://www.baidu.com/" + spec: + rules: + - host: "exact-with-no-cookie" + http: + paths: + - path: / + pathType: Exact + backend: + service: + name: exact-cookie-wrong + port: + number: 3000 + """ + And The Ingress status shows the IP address or FQDN where it is exposed + When I send a "GET" request to "http://exact-with-no-cookie/" + Then the response status-code must be 500 + + Scenario: An Ingress with path rules and multi cookie should send traffic to the matching backend service, use last config + (exact / and multi cookie "cookie_key:cookie_value" and "cookie_key1:cookie_value1" matches request / with cookie) + Given an Ingress resource in a new random namespace + """ + apiVersion: networking.k8s.io/v1 + kind: Ingress + metadata: + name: multicookie-rules + annotations: + bfe.ingress.kubernetes.io/router.cookie: "cookie_key:cookie_value" + bfe.ingress.kubernetes.io/router.cookie: "cookie_key1:cookie_value1" + spec: + rules: + - host: "multi-coookie-rule" + http: + paths: + - path: / + pathType: Exact + backend: + service: + name: multi-cookie + port: + number: 3000 + """ + And The Ingress status shows the IP address or FQDN where it is exposed + When I send a "GET" request to "http://multi-coookie-rule/" with header + """ + {"Cookie": ["cookie_key1=cookie_value1"]} + """ + Then the response status-code must be 200 + And the response must be served by the "multi-cookie" service \ No newline at end of file diff --git a/test/e2e/features/annotations/route/header.feature b/test/e2e/features/annotations/route/header.feature new file mode 100644 index 00000000..095ec7d8 --- /dev/null +++ b/test/e2e/features/annotations/route/header.feature @@ -0,0 +1,348 @@ +@annotations @router.header @release-1.22 +Feature: Header rules + An Ingress may define routing rules based on the request path and header. + + If the HTTP request path matches the paths and header info in the + Ingress objects, the traffic is routed to its backend service. + + Scenario: An Ingress with path rules and header should send traffic to the matching backend service + (exact / and header matches request / with header) + Given an Ingress resource in a new random namespace + """ + apiVersion: networking.k8s.io/v1 + kind: Ingress + metadata: + name: header-rules + annotations: + bfe.ingress.kubernetes.io/router.header: "h_key:h_value" + spec: + rules: + - host: "exact-path-header-key" + http: + paths: + - path: / + pathType: Exact + backend: + service: + name: exact-h-key + port: + number: 3000 + """ + And The Ingress status shows the IP address or FQDN where it is exposed + When I send a "GET" request to "http://exact-path-header-key/" with header + """ + {"h_key": ["h_value"]} + """ + Then the response status-code must be 200 + And the response must be served by the "exact-h-key" service + + Scenario: An Ingress with path rules and header should not send traffic to the matching backend service + (exact / and header matches request / without header) + Given an Ingress resource in a new random namespace + """ + apiVersion: networking.k8s.io/v1 + kind: Ingress + metadata: + name: header-rules + annotations: + bfe.ingress.kubernetes.io/router.header: "h_key:h_value" + spec: + rules: + - host: "exact-path-no-header" + http: + paths: + - path: / + pathType: Exact + backend: + service: + name: exact-h-key + port: + number: 3000 + """ + And The Ingress status shows the IP address or FQDN where it is exposed + When I send a "GET" request to "http://exact-path-no-header/" + Then the response status-code must be 500 + + Scenario: An Ingress with path rules and header should send traffic to the matching backend service + (exact / and header null matches request / with header) + Given an Ingress resource in a new random namespace + """ + apiVersion: networking.k8s.io/v1 + kind: Ingress + metadata: + name: header-rules + annotations: + bfe.ingress.kubernetes.io/router.header: "" + spec: + rules: + - host: "header-null" + http: + paths: + - path: / + pathType: Exact + backend: + service: + name: h-key-null + port: + number: 3000 + """ + And The Ingress status shows the IP address or FQDN where it is exposed + When I send a "GET" request to "http://header-null/" with header + """ + {"h_key": ["h_value"]} + """ + Then the response status-code must be 200 + And the response must be served by the "h-key-null" service + + Scenario: An Ingress with path rules and header should send traffic to the matching backend service + (exact / and header null matches request / without header) + Given an Ingress resource in a new random namespace + """ + apiVersion: networking.k8s.io/v1 + kind: Ingress + metadata: + name: header-rules + annotations: + bfe.ingress.kubernetes.io/router.header: "" + spec: + rules: + - host: "header-null" + http: + paths: + - path: / + pathType: Exact + backend: + service: + name: h-key-null-without-header + port: + number: 3000 + """ + And The Ingress status shows the IP address or FQDN where it is exposed + When I send a "GET" request to "http://header-null/" + Then the response status-code must be 200 + And the response must be served by the "h-key-null-without-header" service + + Scenario: An Ingress with path rules and header should send traffic to the matching backend service + (exact / and header only key matches request / with header only key) + Given an Ingress resource in a new random namespace + """ + apiVersion: networking.k8s.io/v1 + kind: Ingress + metadata: + name: header-rules + annotations: + bfe.ingress.kubernetes.io/router.header: "h-key" + spec: + rules: + - host: "header-only-key" + http: + paths: + - path: / + pathType: Exact + backend: + service: + name: header-only-key + port: + number: 3000 + """ + And The Ingress status should not be success + + + Scenario: An Ingress with path rules and header should send traffic to the matching backend service + (exact / and header only key matches request / without header) + Given an Ingress resource in a new random namespace + """ + apiVersion: networking.k8s.io/v1 + kind: Ingress + metadata: + name: header-rules + annotations: + bfe.ingress.kubernetes.io/router.header: "h-key" + spec: + rules: + - host: "header-only-key" + http: + paths: + - path: / + pathType: Exact + backend: + service: + name: header-only-key-no-header + port: + number: 3000 + """ + And The Ingress status should not be success + + Scenario: An Ingress with path rules and header should send traffic to the matching backend service + (exact / and header normal matches request / with header right) + Given an Ingress resource in a new random namespace + """ + apiVersion: networking.k8s.io/v1 + kind: Ingress + metadata: + name: header-rules + annotations: + bfe.ingress.kubernetes.io/router.header: "User-Agent:Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/85.0.4183.121 Safari/537.36" + spec: + rules: + - host: "header-normal-user-agent" + http: + paths: + - path: / + pathType: Exact + backend: + service: + name: header-normal-right + port: + number: 3000 + """ + And The Ingress status shows the IP address or FQDN where it is exposed + When I send a "GET" request to "http://header-normal-user-agent/" with header + """ + {"User-Agent": ["Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/85.0.4183.121 Safari/537.36"]} + """ + Then the response status-code must be 200 + And the response must be served by the "header-normal-right" service + + Scenario: An Ingress with path rules and header should send traffic to the matching backend service + (exact / and header normal dose not matches request / without header) + Given an Ingress resource in a new random namespace + """ + apiVersion: networking.k8s.io/v1 + kind: Ingress + metadata: + name: header-rules + annotations: + bfe.ingress.kubernetes.io/router.header: "user-agent:Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/85.0.4183.121 Safari/537.36" + spec: + rules: + - host: "header-normal" + http: + paths: + - path: / + pathType: Exact + backend: + service: + name: header-normal-right + port: + number: 3000 + """ + And The Ingress status shows the IP address or FQDN where it is exposed + When I send a "GET" request to "http://header-normal/" + Then the response status-code must be 500 + + Scenario: An Ingress with path rules and header should send traffic to the matching backend service + (exact / and header a:b:c matches request / with header right) + Given an Ingress resource in a new random namespace + """ + apiVersion: networking.k8s.io/v1 + kind: Ingress + metadata: + name: header-rules + annotations: + bfe.ingress.kubernetes.io/router.header: "a:b:c" + spec: + rules: + - host: "header-a-b-c" + http: + paths: + - path: / + pathType: Exact + backend: + service: + name: header-a-b-c-right + port: + number: 3000 + """ + And The Ingress status shows the IP address or FQDN where it is exposed + When I send a "GET" request to "http://header-a-b-c/" with header + """ + {"a": ["b:c"]} + """ + Then the response status-code must be 200 + And the response must be served by the "header-a-b-c-right" service + + Scenario: An Ingress with path rules and header should send traffic to the matching backend service + (exact / and header a:b:c dose not matches request / without header) + Given an Ingress resource in a new random namespace + """ + apiVersion: networking.k8s.io/v1 + kind: Ingress + metadata: + name: header-rules + annotations: + bfe.ingress.kubernetes.io/router.header: "a:b:c" + spec: + rules: + - host: "header-a-b-c-without" + http: + paths: + - path: / + pathType: Exact + backend: + service: + name: without-header-a-b-c + port: + number: 3000 + """ + And The Ingress status shows the IP address or FQDN where it is exposed + When I send a "GET" request to "http://header-a-b-c-without/" + Then the response status-code must be 500 + + Scenario: An Ingress with path rules and header should send traffic to the matching backend service + (exact / and header a:+/?%#& matches request / with header right) + Given an Ingress resource in a new random namespace + """ + apiVersion: networking.k8s.io/v1 + kind: Ingress + metadata: + name: header-rules + annotations: + bfe.ingress.kubernetes.io/router.header: "a:+/?%#&" + spec: + rules: + - host: "header-special-character" + http: + paths: + - path: / + pathType: Exact + backend: + service: + name: header-special-character-right + port: + number: 3000 + """ + And The Ingress status shows the IP address or FQDN where it is exposed + When I send a "GET" request to "http://header-special-character/" with header + """ + {"a": ["+/?%#&"]} + """ + Then the response status-code must be 200 + And the response must be served by the "header-special-character-right" service + + Scenario: An Ingress with path rules and header should send traffic to the matching backend service + (exact / and header a:+/?%#& matches request / without header) + Given an Ingress resource in a new random namespace + """ + apiVersion: networking.k8s.io/v1 + kind: Ingress + metadata: + name: header-rules + annotations: + bfe.ingress.kubernetes.io/router.header: "a:+/?%#&" + spec: + rules: + - host: "header-special-character-no" + http: + paths: + - path: / + pathType: Exact + backend: + service: + name: header-special-character-no + port: + number: 3000 + """ + And The Ingress status shows the IP address or FQDN where it is exposed + When I send a "GET" request to "http://header-special-character-no/" + Then the response status-code must be 500 \ No newline at end of file diff --git a/test/e2e/features/annotations/route/priority.feature b/test/e2e/features/annotations/route/priority.feature new file mode 100644 index 00000000..4248bd47 --- /dev/null +++ b/test/e2e/features/annotations/route/priority.feature @@ -0,0 +1,217 @@ +@annotations @route.header @router.cookie @release-1.22 +Feature: Priority rules + An Ingress may define routing rules based on the request path. + + If the HTTP request path matches the paths and cookie info in the + Ingress objects, the traffic is routed to its backend service. + + Scenario: An Ingress with path rules and cookie should send traffic to the matching backend service + (exact /, cookie, header matches request / with cookie and header) + Given an Ingress resource in a new random namespace + """ + apiVersion: networking.k8s.io/v1 + kind: Ingress + metadata: + name: same-host-same-path-num-diff + annotations: + bfe.ingress.kubernetes.io/router.cookie: "uid:abcd" + bfe.ingress.kubernetes.io/router.header: "h_key:h_value" + spec: + rules: + - host: "same-host-same-path-num-diff" + http: + paths: + - path: / + pathType: Exact + backend: + service: + name: service-match-2 + port: + number: 3000 + """ + And an Ingress resource in a new random namespace + """ + apiVersion: networking.k8s.io/v1 + kind: Ingress + metadata: + name: same-host-same-path-num-diff + annotations: + bfe.ingress.kubernetes.io/router.cookie: "uid:abcd" + spec: + rules: + - host: "same-host-same-path-num-diff" + http: + paths: + - path: / + pathType: Exact + backend: + service: + name: service-match-1 + port: + number: 3000 + """ + And The Ingress status shows the IP address or FQDN where it is exposed + When I send a "GET" request to "http://same-host-same-path-num-diff/" with header + """ + {"Cookie": ["uid=abcd"],"h_key": ["h_value"]} + """ + Then the response status-code must be 200 + And the response must be served by the "service-match-2" service + + Scenario: An Ingress with path rules and cookie should send traffic to the matching backend service + (exact /, cookie matches request / with cookie) + Given an Ingress resource in a new random namespace + """ + apiVersion: networking.k8s.io/v1 + kind: Ingress + metadata: + name: same-host-same-path-num-diff + annotations: + bfe.ingress.kubernetes.io/router.cookie: "uid:abcd" + bfe.ingress.kubernetes.io/router.header: "h_key:h_value" + spec: + rules: + - host: "same-host-same-path-num-diff" + http: + paths: + - path: / + pathType: ImplementationSpecific + backend: + service: + name: service-match-2 + port: + number: 3000 + """ + And an Ingress resource in a new random namespace + """ + apiVersion: networking.k8s.io/v1 + kind: Ingress + metadata: + name: same-host-same-path-num-diff + annotations: + bfe.ingress.kubernetes.io/router.cookie: "uid:abcd" + spec: + rules: + - host: "same-host-same-path-num-diff" + http: + paths: + - path: / + pathType: ImplementationSpecific + backend: + service: + name: service-match-1 + port: + number: 3000 + """ + And The Ingress status shows the IP address or FQDN where it is exposed + When I send a "GET" request to "http://same-host-same-path-num-diff/" with header + """ + {"Cookie": ["uid=abcd"]} + """ + Then the response status-code must be 200 + And the response must be served by the "service-match-1" service + + Scenario: An Ingress with path rules and cookie should send traffic to the matching backend service + (exact /, cookie matches request / with cookie) + Given an Ingress resource in a new random namespace + """ + apiVersion: networking.k8s.io/v1 + kind: Ingress + metadata: + name: same-host-same-path-num-diff + annotations: + bfe.ingress.kubernetes.io/router.cookie: "uid:abcd" + bfe.ingress.kubernetes.io/router.header: "h_key:h_value" + spec: + rules: + - host: "same-host-same-path-num-diff" + http: + paths: + - path: / + pathType: ImplementationSpecific + backend: + service: + name: service-match-2 + port: + number: 3000 + """ + And an Ingress resource in a new random namespace + """ + apiVersion: networking.k8s.io/v1 + kind: Ingress + metadata: + name: same-host-same-path-num-diff + annotations: + bfe.ingress.kubernetes.io/router.header: "h_key:h_value" + spec: + rules: + - host: "same-host-same-path-num-diff" + http: + paths: + - path: / + pathType: ImplementationSpecific + backend: + service: + name: service-match-1 + port: + number: 3000 + """ + And The Ingress status shows the IP address or FQDN where it is exposed + When I send a "GET" request to "http://same-host-same-path-num-diff/" with header + """ + {"h_key": ["h_value"]} + """ + Then the response status-code must be 200 + And the response must be served by the "service-match-1" service + + Scenario: An Ingress with path rules and cookie should send traffic to the matching backend service + (exact /, cookie matches request / with cookie) + Given an Ingress resource in a new random namespace + """ + apiVersion: networking.k8s.io/v1 + kind: Ingress + metadata: + name: same-host-same-path-num-same + annotations: + bfe.ingress.kubernetes.io/router.cookie: "uid:abcd" + spec: + rules: + - host: "same-host-same-path-num-same" + http: + paths: + - path: / + pathType: ImplementationSpecific + backend: + service: + name: service-match-2 + port: + number: 3000 + """ + And an Ingress resource in a new random namespace + """ + apiVersion: networking.k8s.io/v1 + kind: Ingress + metadata: + name: same-host-same-path-num-same1 + annotations: + bfe.ingress.kubernetes.io/router.header: "h_key:h_value" + spec: + rules: + - host: "same-host-same-path-num-same" + http: + paths: + - path: / + pathType: ImplementationSpecific + backend: + service: + name: service-match-1 + port: + number: 3000 + """ + And The Ingress status shows the IP address or FQDN where it is exposed + When I send a "GET" request to "http://same-host-same-path-num-same/" with header + """ + {"Cookie": ["uid=abcd"],"h_key": ["h_value"]} + """ + Then the response status-code must be 200 + And the response must be served by the "service-match-2" service \ No newline at end of file diff --git a/test/e2e/features/conformance/default_backend.feature b/test/e2e/features/conformance/default_backend.feature new file mode 100644 index 00000000..f2c40d1d --- /dev/null +++ b/test/e2e/features/conformance/default_backend.feature @@ -0,0 +1,46 @@ +@sig-network @conformance @release-1.22 +Feature: Default backend + An Ingress with no rules sends all traffic to the single default backend. + The default backend is part of the Ingress resource spec field `defaultBackend`. + + If none of the hosts or paths match the HTTP request in the + Ingress objects, the traffic is routed to your default backend. + + Background: + Given a new random namespace + Given an Ingress resource named "default-backend" with this spec: + """ + defaultBackend: + service: + name: echo-service + port: + number: 3000 + """ + Then The Ingress status shows the IP address or FQDN where it is exposed + + Scenario Outline: An Ingress with no rules should send all requests to the default backend + When I send a "" request to http://""/"" + Then the response status-code must be 200 + And the response must be served by the "echo-service" service + And the response proto must be "HTTP/1.1" + And the response headers must contain with matching + | key | value | + | Content-Length | * | + | Content-Type | * | + | Date | * | + | Server | * | + And the request method must be "" + And the request path must be "" + And the request proto must be "HTTP/1.1" + And the request headers must contain with matching + | key | value | + | User-Agent | Go-http-client/1.1 | + + Examples: + | method | host | path | + | GET | my-host | | + | GET | my-host | sub-path | + | POST | some-host | | + | PUT | | resource | + | DELETE | some-host | resource | + | PATCH | my-host | resource | diff --git a/test/e2e/features/conformance/host_rules.feature b/test/e2e/features/conformance/host_rules.feature new file mode 100644 index 00000000..98066134 --- /dev/null +++ b/test/e2e/features/conformance/host_rules.feature @@ -0,0 +1,90 @@ +@sig-network @conformance @release-1.22 +Feature: Host rules + An Ingress may define routing rules based on the request host. + + If the HTTP request host matches one of the hosts in the + Ingress objects, the traffic is routed to its backend service. + + Background: + Given a new random namespace + Given a self-signed TLS secret named "conformance-tls" for the "foo.bar.com" hostname + Given an Ingress resource + """ + apiVersion: networking.k8s.io/v1 + kind: Ingress + metadata: + name: host-rules + spec: + tls: + - hosts: + - foo.bar.com + secretName: conformance-tls + rules: + - host: "*.foo.com" + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: wildcard-foo-com + port: + number: 8080 + + - host: foo.bar.com + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: foo-bar-com + port: + number: 80 + + """ + Then The Ingress status shows the IP address or FQDN where it is exposed + + + Scenario: An Ingress with a host rule should send TLS traffic to the matching backend service + (host foo.bar.com matches request foo.bar.com) + + When I send a "GET" request to "https://foo.bar.com" + Then the secure connection must verify the "foo.bar.com" hostname + And the response status-code must be 200 + And the response must be served by the "foo-bar-com" service + And the request host must be "foo.bar.com" + + Scenario: An Ingress with a host rule should send traffic to the matching backend service + (host foo.bar.com matches request foo.bar.com) + + When I send a "GET" request to "http://foo.bar.com" + And the response status-code must be 200 + And the response must be served by the "foo-bar-com" service + And the request host must be "foo.bar.com" + + Scenario: An Ingress with a host rule should not route traffic when hostname does not match + (host foo.bar.com does not match request subdomain.bar.com) + + When I send a "GET" request to "http://subdomain.bar.com" + Then the response status-code must be 500 + + Scenario: An Ingress with a wildcard host rule should send traffic to the matching backend service + (Matches based on shared suffix) + + When I send a "GET" request to "http://bar.foo.com" + Then the response status-code must be 200 + And the response must be served by the "wildcard-foo-com" service + And the request host must be "bar.foo.com" + + Scenario: An Ingress with a wildcard host rule should not route traffic matching on more than a single dns label + (No match, wildcard only covers a single DNS label) + + When I send a "GET" request to "http://baz.bar.foo.com" + Then the response status-code must be 500 + + Scenario: An Ingress with a wildcard host rule should not route traffic matching no dns label + (No match, wildcard only covers a single DNS label) + + When I send a "GET" request to "http://foo.com" + Then the response status-code must be 500 diff --git a/test/e2e/features/conformance/ingress_class.feature b/test/e2e/features/conformance/ingress_class.feature new file mode 100644 index 00000000..f9246a17 --- /dev/null +++ b/test/e2e/features/conformance/ingress_class.feature @@ -0,0 +1,31 @@ +@sig-network @conformance @release-1.22 +Feature: Ingress class + Ingresses can be implemented by different controllers, often with different configuration. + Each Ingress definition could specify a class, a reference to an IngressClass resource that contains + additional configuration including the name of the controller that should implement the class. + + https://kubernetes.io/docs/concepts/services-networking/ingress/#ingress-class + + Scenario: An Ingress with an invalid ingress class should not send traffic to the matching backend service + Given an Ingress resource in a new random namespace + """ + apiVersion: networking.k8s.io/v1 + kind: Ingress + metadata: + name: test-ingress-class + spec: + ingressClassName: some-invalid-class-name + rules: + - host: "ingress-class" + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: ingress-class-prefix + port: + number: 8080 + + """ + Then The Ingress status should not contain the IP address or FQDN diff --git a/test/e2e/features/conformance/load_balancing.feature b/test/e2e/features/conformance/load_balancing.feature new file mode 100644 index 00000000..56073ffc --- /dev/null +++ b/test/e2e/features/conformance/load_balancing.feature @@ -0,0 +1,32 @@ +@sig-network @conformance @release-1.22 +Feature: Load Balancing + An Ingress exposing a backend service with multiple replicas should use all the pods available + The feature sessionAffinity is not configured in the backend service https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.19/#service-v1-core + + Background: + Given an Ingress resource in a new random namespace + """ + apiVersion: networking.k8s.io/v1 + kind: Ingress + metadata: + name: path-rules + spec: + rules: + - host: "load-balancing" + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: echo-service + port: + number: 8080 + + """ + Then The Ingress status shows the IP address or FQDN where it is exposed + Then The backend deployment "echo-service" for the ingress resource is scaled to 10 + + Scenario Outline: An Ingress should send all requests to the backend + When I send 100 requests to "http://load-balancing" + Then all the responses status-code must be 200 and the response body should contain the IP address of 10 different Kubernetes pods diff --git a/test/e2e/features/conformance/path_rules.feature b/test/e2e/features/conformance/path_rules.feature new file mode 100644 index 00000000..628cf8ab --- /dev/null +++ b/test/e2e/features/conformance/path_rules.feature @@ -0,0 +1,200 @@ +@sig-network @conformance @release-1.22 +Feature: Path rules + An Ingress may define routing rules based on the request path. + + If the HTTP request path matches one of the paths in the + Ingress objects, the traffic is routed to its backend service. + + Background: + Given an Ingress resource in a new random namespace + """ + apiVersion: networking.k8s.io/v1 + kind: Ingress + metadata: + name: path-rules + spec: + rules: + - host: "exact-path-rules" + http: + paths: + - path: /foo + pathType: Exact + backend: + service: + name: foo-exact + port: + number: 8080 + + - host: "prefix-path-rules" + http: + paths: + - path: /foo + pathType: Prefix + backend: + service: + name: foo-prefix + port: + number: 8080 + + - path: /aaa/bbb + pathType: Prefix + backend: + service: + name: aaa-slash-bbb-prefix + port: + number: 8080 + + - path: /aaa + pathType: Prefix + backend: + service: + name: aaa-prefix + port: + number: 8080 + + - host: "mixed-path-rules" + http: + paths: + - path: /foo + pathType: Prefix + backend: + service: + name: foo-prefix + port: + number: 8080 + + - path: /foo + pathType: Exact + backend: + service: + name: foo-exact + port: + number: 8080 + + + - host: "trailing-slash-path-rules" + http: + paths: + - path: /aaa/bbb/ + pathType: Prefix + backend: + service: + name: aaa-slash-bbb-slash-prefix + port: + number: 8080 + - path: /foo/ + pathType: Exact + backend: + service: + name: foo-slash-exact + port: + number: 8080 + + """ + Then The Ingress status shows the IP address or FQDN where it is exposed + + Scenario: An Ingress with exact path rules should send traffic to the matching backend service + (exact /foo matches request /foo) + + When I send a "GET" request to "http://exact-path-rules/foo" + Then the response status-code must be 200 + And the response must be served by the "foo-exact" service + + Scenario: An Ingress with exact path rules should not match requests with trailing slash + (exact /foo does not match request /foo/) + + When I send a "GET" request to "http://exact-path-rules/foo/" + Then the response status-code must be 500 + + Scenario: An Ingress with exact path rules should be case sensitive + (exact /foo does not match request /FOO) + + When I send a "GET" request to "http://exact-path-rules/FOO" + Then the response status-code must be 500 + + Scenario: An Ingress with exact path rules should not match any other label + (exact /foo does not match request /bar) + + When I send a "GET" request to "http://exact-path-rules/bar" + Then the response status-code must be 500 + + Scenario: An Ingress with prefix path rules should send traffic to the matching backend service + (prefix /foo matches request /foo) + + When I send a "GET" request to "http://prefix-path-rules/foo" + Then the response status-code must be 200 + And the response must be served by the "foo-prefix" service + + Scenario: An Ingress with prefix path rules should ignore the request trailing slash and send traffic to the matching backend service + (prefix /foo matches request /foo/) + + When I send a "GET" request to "http://prefix-path-rules/foo/" + Then the response status-code must be 200 + And the response must be served by the "foo-prefix" service + + Scenario: An Ingress with prefix path rules should be case sensitive + (prefix /foo does not match request /FOO) + + When I send a "GET" request to "http://prefix-path-rules/FOO" + Then the response status-code must be 500 + + Scenario: An Ingress with prefix path rules should match multiple labels, match the longest path, and send traffic to the matching backend service + (prefix /aaa/bbb matches request /aaa/bbb) + + When I send a "GET" request to "http://prefix-path-rules/aaa/bbb" + Then the response status-code must be 200 + And the response must be served by the "aaa-slash-bbb-prefix" service + + Scenario: An Ingress with prefix path rules should match multiple labels, match the longest path, and subpaths and send traffic to the matching backend service + (prefix /aaa/bbb matches request /aaa/bbb/ccc) + + When I send a "GET" request to "http://prefix-path-rules/aaa/bbb/ccc" + Then the response status-code must be 200 + And the response must be served by the "aaa-slash-bbb-prefix" service + + Scenario: An Ingress with prefix path rules should and send traffic to the matching backend service + (prefix /aaa matches request /aaa/ccc) + + When I send a "GET" request to "http://prefix-path-rules/aaa/ccc" + Then the response status-code must be 200 + And the response must be served by the "aaa-prefix" service + + Scenario: An Ingress with prefix path rules should match each labels string prefix + (prefix /aaa does not match request /aaaccc) + + When I send a "GET" request to "http://prefix-path-rules/aaaccc" + Then the response status-code must be 500 + + Scenario: An Ingress with prefix path rules should ignore the request trailing slash and send traffic to the matching backend service + (prefix /foo matches request /foo/) + + When I send a "GET" request to "http://prefix-path-rules/foo/" + Then the response status-code must be 200 + And the response must be served by the "foo-prefix" service + + Scenario: An Ingress with mixed path rules should send traffic to the matching backend service where Exact is preferred + (exact /foo matches request /foo) + + When I send a "GET" request to "http://mixed-path-rules/foo" + Then the response status-code must be 200 + And the response must be served by the "foo-exact" service + + Scenario: An Ingress with a trailing slashes in a prefix path rule should ignore the trailing slash and send traffic to the matching backend service + (prefix /aaa/bbb/ matches request /aaa/bbb) + + When I send a "GET" request to "http://trailing-slash-path-rules/aaa/bbb" + Then the response status-code must be 200 + And the response must be served by the "aaa-slash-bbb-slash-prefix" service + + Scenario: An Ingress with a trailing slashes in a prefix path rule should ignore the trailing slash and send traffic to the matching backend service + (prefix /aaa/bbb/ matches request /aaa/bbb/) + + When I send a "GET" request to "http://trailing-slash-path-rules/aaa/bbb/" + Then the response status-code must be 200 + And the response must be served by the "aaa-slash-bbb-slash-prefix" service + + Scenario: An Ingress with a trailing slashes in an exact path rule should not match requests without a trailing slash + (exact /foo/ does not match request /foo) + + When I send a "GET" request to "http://trailing-slash-path-rules/foo" + Then the response status-code must be 500 diff --git a/test/e2e/features/rules/host_rule1.feature b/test/e2e/features/rules/host_rule1.feature new file mode 100644 index 00000000..b9a04c34 --- /dev/null +++ b/test/e2e/features/rules/host_rule1.feature @@ -0,0 +1,82 @@ +@ingress.rule @release-1.22 +Feature: host rule test + An Ingress may define routing rules based on the request path and host. + + If the HTTP request path matches one of the paths in the + Ingress objects, the traffic is routed to its backend service. + + Background: + Given an Ingress resource in a new random namespace + """ + apiVersion: networking.k8s.io/v1 + kind: Ingress + metadata: + name: diff-host-same-path + spec: + rules: + - host: "diff-host-same-path" + http: + paths: + - path: /whoami + pathType: ImplementationSpecific + backend: + service: + name: service-diff-host + port: + number: 8080 + + - host: "diff-host-same-path-slash" + http: + paths: + - path: /whoami + pathType: ImplementationSpecific + backend: + service: + name: service-diff-host-slash + port: + number: 8080 + """ + Then The Ingress status shows the IP address or FQDN where it is exposed + + Scenario: An Ingress with path rules slash and host should not send traffic to the matching backend service + (path / and host diff-host-same-path dose not matches request / and host diff-host-same-path) + + When I send a "GET" request to "http://diff-host-same-path/" + Then the response status-code must be 500 + + Scenario: An Ingress with path rules slash and host should send traffic to the matching backend service + (path / and host diff-host-same-path matches request /whoami and host diff-host-same-path) + + When I send a "GET" request to "http://diff-host-same-path/whoami" + Then the response status-code must be 200 + And the response must be served by the "service-diff-host" service + + Scenario: An Ingress with path rules slash and host should send traffic to the matching backend service + (path / and host diff-host-same-path matches request /whoami/a/b/c/d/e/f/g and host diff-host-same-path) + + When I send a "GET" request to "http://diff-host-same-path/whoami/a/b/c/d/e/f/g" + Then the response status-code must be 200 + And the response must be served by the "service-diff-host" service + + + Scenario: An Ingress with path rules slash and host should not send traffic to the matching backend service + (path / and host diff-host-same-path-slash dose not matches request / and diff-host-same-path-slash) + + When I send a "GET" request to "http://diff-host-same-path-slash/" + Then the response status-code must be 500 + + Scenario: An Ingress with path rules slash and host should send traffic to the matching backend service + (path / and host diff-host-same-path-slash matches request /whoami and host diff-host-same-path-slash) + + When I send a "GET" request to "http://diff-host-same-path-slash/whoami" + Then the response status-code must be 200 + And the response must be served by the "service-diff-host-slash" service + + Scenario: An Ingress with path rules slash and host should send traffic to the matching backend service + (path / and host diff-host-same-path-slash matches request /whoami/a/b/c/d/e/f/g and host diff-host-same-path-slash) + + When I send a "GET" request to "http://diff-host-same-path-slash/whoami/a/b/c/d/e/f/g" + Then the response status-code must be 200 + And the response must be served by the "service-diff-host-slash" service + + diff --git a/test/e2e/features/rules/host_rule2.feature b/test/e2e/features/rules/host_rule2.feature new file mode 100644 index 00000000..eacbfbf3 --- /dev/null +++ b/test/e2e/features/rules/host_rule2.feature @@ -0,0 +1,80 @@ +@ingress.rule @release-1.22 +Feature: host rule test + An Ingress may define routing rules based on the request path and host. + + If the HTTP request path matches one of the paths in the + Ingress objects, the traffic is routed to its backend service. + + Background: + Given an Ingress resource in a new random namespace + """ + apiVersion: networking.k8s.io/v1 + kind: Ingress + metadata: + name: diff-host-rules + spec: + rules: + - host: "diff-host-diff-path-slash" + http: + paths: + - path: / + pathType: ImplementationSpecific + backend: + service: + name: service-diff-host-slash + port: + number: 8080 + + - host: "diff-host-diff-path-slash-whoami" + http: + paths: + - path: /whoami + pathType: ImplementationSpecific + backend: + service: + name: service-diff-host-slash-whoami + port: + number: 8080 + """ + Then The Ingress status shows the IP address or FQDN where it is exposed + + Scenario: An Ingress with path rules slash and host should send traffic to the matching backend service + (path / and host diff-host-diff-path-slash matches request / and host diff-host-diff-path-slash) + + When I send a "GET" request to "http://diff-host-diff-path-slash/" + Then the response status-code must be 200 + And the response must be served by the "service-diff-host-slash" service + + Scenario: An Ingress with path rules slash and host should send traffic to the matching backend service + (path / and host diff-host-diff-path-slash matches request /whoami and host diff-host-diff-path-slash) + + When I send a "GET" request to "http://diff-host-diff-path-slash/whoami" + Then the response status-code must be 200 + And the response must be served by the "service-diff-host-slash" service + + Scenario: An Ingress with path rules slash and host should send traffic to the matching backend service + (path / and host diff-host-diff-path-slash matches request /whoami/a/b/c/d/e/f/g and host diff-host-diff-path-slash) + + When I send a "GET" request to "http://diff-host-diff-path-slash/whoami/a/b/c/d/e/f/g" + Then the response status-code must be 200 + And the response must be served by the "service-diff-host-slash" service + + Scenario: An Ingress with path rules slash whoami and host should not send traffic to the matching backend service + (path /whoami and diff-host-diff-path-slash-whoami dose not matches request / and host diff-host-diff-path-slash-whoami) + + When I send a "GET" request to "http://diff-host-diff-path-slash-whoami/" + Then the response status-code must be 500 + + Scenario: An Ingress with path rules slash whoami and host should send traffic to the matching backend service + (path /whoami and diff-host-diff-path-slash-whoami matches request /whoami and host diff-host-diff-path-slash-whoami) + + When I send a "GET" request to "http://diff-host-diff-path-slash-whoami/whoami" + Then the response status-code must be 200 + And the response must be served by the "service-diff-host-slash-whoami" service + + Scenario: An Ingress with path rules slash whoami and host should send traffic to the matching backend service + (path /whoami and diff-host-diff-path-slash-whoami matches request /whoami/a/b/c/d/e/f/g and host diff-host-diff-path-slash-whoami) + + When I send a "GET" request to "http://diff-host-diff-path-slash-whoami/whoami/a/b/c/d/e/f/g" + Then the response status-code must be 200 + And the response must be served by the "service-diff-host-slash-whoami" service diff --git a/test/e2e/features/rules/multiple_ingress.feature b/test/e2e/features/rules/multiple_ingress.feature new file mode 100644 index 00000000..d30f4226 --- /dev/null +++ b/test/e2e/features/rules/multiple_ingress.feature @@ -0,0 +1,97 @@ +@ingress.rule @release-1.22 +Feature: Same host in multiple ingresses + An Ingress may define routing rules based on the request path and host. + + If the HTTP request path matches one of the paths in the + Ingress objects, the traffic is routed to its backend service. + + Background: + Given an Ingress resource in a new random namespace + """ + apiVersion: networking.k8s.io/v1 + kind: Ingress + metadata: + name: same-host-diff-path-1 + spec: + rules: + - host: "same-host" + http: + paths: + - path: /test/foo + pathType: Prefix + backend: + service: + name: service-same-host-1 + port: + number: 8080 + """ + Then The Ingress status shows the IP address or FQDN where it is exposed + + And an Ingress resource + """ + apiVersion: networking.k8s.io/v1 + kind: Ingress + metadata: + name: same-host-diff-path-2 + spec: + rules: + - host: "same-host" + http: + paths: + - path: /test + pathType: Prefix + backend: + service: + name: service-same-host-2 + port: + number: 8080 + """ + Then The Ingress status shows the IP address or FQDN where it is exposed + + Scenario: An Ingress with path rules slash and host should not send traffic to the matching backend service + (path /test and host same-host dose not matches request / and host same-host) + + When I send a "GET" request to "http://same-host/" + Then the response status-code must be 500 + + Scenario: An Ingress with path rules slash and host should send traffic to the matching backend service + (path /test and host same-host matches request /test and host same-host) + + When I send a "GET" request to "http://same-host/test" + Then the response status-code must be 200 + And the response must be served by the "service-same-host-2" service + + Scenario: An Ingress with path rules slash and host should send traffic to the matching backend service + (path /test and host same-host matches request /test/ and host same-host) + + When I send a "GET" request to "http://same-host/test/" + Then the response status-code must be 200 + And the response must be served by the "service-same-host-2" service + + Scenario: An Ingress with path rules slash and host should send traffic to the matching backend service + (path /test and host same-host matches request /test/fo and host same-host) + + When I send a "GET" request to "http://same-host/test/fo" + Then the response status-code must be 200 + And the response must be served by the "service-same-host-2" service + + Scenario: An Ingress with path rules slash and host should send traffic to the matching backend service + (path /test/foo and host same-host matches request /test/foo and host same-host) + + When I send a "GET" request to "http://same-host/test/foo" + Then the response status-code must be 200 + And the response must be served by the "service-same-host-1" service + + Scenario: An Ingress with path rules slash and host should send traffic to the matching backend service + (path /test/foo and host same-host matches request /test/foo/ and host same-host) + + When I send a "GET" request to "http://same-host/test/foo/" + Then the response status-code must be 200 + And the response must be served by the "service-same-host-1" service + + Scenario: An Ingress with path rules slash and host should send traffic to the matching backend service + (path /test/foo and host same-host matches request /test/foo/a/b/c/e/f/g and host same-host) + + When I send a "GET" request to "http://same-host/test/foo/a/b/c/e/f/g" + Then the response status-code must be 200 + And the response must be served by the "service-same-host-1" service diff --git a/test/e2e/features/rules/path_err.feature b/test/e2e/features/rules/path_err.feature new file mode 100644 index 00000000..db744c72 --- /dev/null +++ b/test/e2e/features/rules/path_err.feature @@ -0,0 +1,47 @@ +@ingress.rule @release-1.22 +Feature: Path error rules + An Ingress status error when given error ingress. + + Scenario: An Ingress with path rules a status should be error and not send traffic to the matching backend service + (path rule a) + Given an Ingress resource in a new random namespace should not create + """ + apiVersion: networking.k8s.io/v1 + kind: Ingress + metadata: + name: path-err-rules + spec: + rules: + - host: "exact-path-rules-cookie-uid" + http: + paths: + - path: a + pathType: Exact + backend: + service: + name: foo-exact + port: + number: 3000 + """ + + Scenario: An Ingress with path rules null status should be error and not send traffic to the matching backend service + (path rule null) + Given an Ingress resource in a new random namespace should not create + """ + apiVersion: networking.k8s.io/v1 + kind: Ingress + metadata: + name: path-err-rules + spec: + rules: + - host: "exact-path-rules-cookie-uid" + http: + paths: + - path: + pathType: Exact + backend: + service: + name: foo-exact + port: + number: 3000 + """ diff --git a/test/e2e/features/rules/path_rule1.feature b/test/e2e/features/rules/path_rule1.feature new file mode 100644 index 00000000..fc893021 --- /dev/null +++ b/test/e2e/features/rules/path_rule1.feature @@ -0,0 +1,648 @@ +@ingress.rule @release-1.22 +Feature: Path test + An Ingress may define routing rules based on the request path. + + If the HTTP request path matches one of the paths in the + Ingress objects, the traffic is routed to its backend service. + + Background: + Given an Ingress resource in a new random namespace + """ + apiVersion: networking.k8s.io/v1 + kind: Ingress + metadata: + name: path-rules + spec: + rules: + - host: "path-rules-single-slash" + http: + paths: + - path: / + pathType: ImplementationSpecific + backend: + service: + name: single-slash + port: + number: 3000 + + - host: "path-rules-single-slash-exact" + http: + paths: + - path: / + pathType: Exact + backend: + service: + name: single-slash-exact + port: + number: 3000 + + - host: "path-rules-single-slash-prefix" + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: single-slash-prefix + port: + number: 3000 + + - host: "path-rules-slash-foo" + http: + paths: + - path: /foo + pathType: ImplementationSpecific + backend: + service: + name: slash-foo + port: + number: 3000 + + - host: "path-rules-slash-foo-exact" + http: + paths: + - path: /foo + pathType: Exact + backend: + service: + name: slash-foo-exact + port: + number: 3000 + + - host: "path-rules-slash-foo-prefix" + http: + paths: + - path: /foo + pathType: Prefix + backend: + service: + name: slash-foo-prefix + port: + number: 3000 + + - host: "path-rules-slash-foo-slash" + http: + paths: + - path: /foo/ + pathType: ImplementationSpecific + backend: + service: + name: slash-foo-slash + port: + number: 3000 + + - host: "path-rules-slash-foo-slash-exact" + http: + paths: + - path: /foo/ + pathType: Exact + backend: + service: + name: slash-foo-slash-exact + port: + number: 3000 + + - host: "path-rules-slash-foo-slash-prefix" + http: + paths: + - path: /foo/ + pathType: Prefix + backend: + service: + name: slash-foo-slash-prefix + port: + number: 3000 + + - host: "path-rules-slash-aaa-slash-bb" + http: + paths: + - path: /aaa/bb + pathType: ImplementationSpecific + backend: + service: + name: slash-aaa-slash-bb + port: + number: 3000 + + - host: "path-rules-slash-aaa-slash-bb-exact" + http: + paths: + - path: /aaa/bb + pathType: Exact + backend: + service: + name: slash-aaa-slash-bb-exact + port: + number: 3000 + + - host: "path-rules-slash-aaa-slash-bb-prefix" + http: + paths: + - path: /aaa/bb + pathType: Prefix + backend: + service: + name: slash-aaa-slash-bb-prefix + port: + number: 3000 + + - host: "path-rules-slash-aaa-slash-bbb" + http: + paths: + - path: /aaa/bbb + pathType: ImplementationSpecific + backend: + service: + name: slash-aaa-slash-bbb + port: + number: 3000 + + - host: "path-rules-slash-aaa-slash-bbb-exact" + http: + paths: + - path: /aaa/bbb + pathType: Exact + backend: + service: + name: slash-aaa-slash-bbb-exact + port: + number: 3000 + + - host: "path-rules-slash-aaa-slash-bbb-prefix" + http: + paths: + - path: /aaa/bbb + pathType: Prefix + backend: + service: + name: slash-aaa-slash-bbb-prefix + port: + number: 3000 + + - host: "path-rules-slash-aaa-slash-bbb-slash" + http: + paths: + - path: /aaa/bbb/ + pathType: ImplementationSpecific + backend: + service: + name: slash-aaa-slash-bbb-slash + port: + number: 3000 + + - host: "path-rules-slash-aaa-slash-bbb-slash-exact" + http: + paths: + - path: /aaa/bbb/ + pathType: Exact + backend: + service: + name: slash-aaa-slash-bbb-slash-exact + port: + number: 3000 + + - host: "path-rules-slash-aaa-slash-bbb-slash-prefix" + http: + paths: + - path: /aaa/bbb/ + pathType: Prefix + backend: + service: + name: slash-aaa-slash-bbb-slash-prefix + port: + number: 3000 + """ + Then The Ingress status shows the IP address or FQDN where it is exposed + + Scenario: An Ingress with default path rules / should send traffic to the matching backend service + (default / matches request /) + + When I send a "GET" request to "http://path-rules-single-slash/" + Then the response status-code must be 200 + And the response must be served by the "single-slash" service + + Scenario: An Ingress with default path rules / should send traffic to the matching backend service + (default / matches request /a) + + When I send a "GET" request to "http://path-rules-single-slash/a" + Then the response status-code must be 200 + And the response must be served by the "single-slash" service + + Scenario: An Ingress with default path rules / should send traffic to the matching backend service + (default / matches request /a/b/c/d/e/f/g) + + When I send a "GET" request to "http://path-rules-single-slash/a/b/c/d/e/f/g" + Then the response status-code must be 200 + And the response must be served by the "single-slash" service + + Scenario: An Ingress with Exact path rules / should send traffic to the matching backend service + (Exact / matches request /) + + When I send a "GET" request to "http://path-rules-single-slash-exact/" + Then the response status-code must be 200 + And the response must be served by the "single-slash-exact" service + + Scenario: An Ingress with Exact path rules / should not send traffic to the matching backend service + (Exact / dose not matches request /a) + When I send a "GET" request to "http://path-rules-single-slash-exact/a" + Then the response status-code must be 500 + + Scenario: An Ingress with Exact path rules / should not send traffic to the matching backend service + (Exact / dose not matches request /a/b/c/d/e/f/g) + When I send a "GET" request to "http://path-rules-single-slash-exact/a/b/c/d/e/f/g" + Then the response status-code must be 500 + + Scenario: An Ingress with Exact path rules / should send traffic to the matching backend service + (Prefix / matches request /) + + When I send a "GET" request to "http://path-rules-single-slash-prefix/" + Then the response status-code must be 200 + And the response must be served by the "single-slash-prefix" service + + Scenario: An Ingress with Exact path rules / should send traffic to the matching backend service + (Prefix / dose not matches request /a) + When I send a "GET" request to "http://path-rules-single-slash-prefix/a" + Then the response status-code must be 200 + And the response must be served by the "single-slash-prefix" service + + Scenario: An Ingress with Exact path rules / should send traffic to the matching backend service + (Prefix / dose not matches request /a/b/c/d/e/f/g) + When I send a "GET" request to "http://path-rules-single-slash-prefix/a/b/c/d/e/f/g" + Then the response status-code must be 200 + And the response must be served by the "single-slash-prefix" service + + Scenario: An Ingress with path rules /foo should not send traffic to the matching backend service + (default /foo dose not matches request /) + When I send a "GET" request to "http://path-rules-slash-foo/" + Then the response status-code must be 500 + + Scenario: An Ingress with path rules /foo should send traffic to the matching backend service + (default /foo matches request /foo) + When I send a "GET" request to "http://path-rules-slash-foo/foo" + Then the response status-code must be 200 + And the response must be served by the "slash-foo" service + + Scenario: An Ingress with path rules /foo should send traffic to the matching backend service + (default /foo matches request /foo/) + When I send a "GET" request to "http://path-rules-slash-foo/foo/" + Then the response status-code must be 200 + And the response must be served by the "slash-foo" service + + Scenario: An Ingress with path rules /foo should send traffic to the matching backend service + (default /foo matches request /foo/a/b/c/d/e/f/g) + When I send a "GET" request to "http://path-rules-slash-foo/foo/a/b/c/d/e/f/g" + Then the response status-code must be 200 + And the response must be served by the "slash-foo" service + + Scenario: An Ingress with path rules /foo should not send traffic to the matching backend service + (Exact /foo dose not matches request /) + When I send a "GET" request to "http://path-rules-slash-foo-exact/" + Then the response status-code must be 500 + + Scenario: An Ingress with path rules /foo should send traffic to the matching backend service + (Exact /foo matches request /foo) + When I send a "GET" request to "http://path-rules-slash-foo-exact/foo" + Then the response status-code must be 200 + And the response must be served by the "slash-foo-exact" service + + Scenario: An Ingress with path rules /foo should send traffic to the matching backend service + (Exact /foo dose not matches request /foo/) + When I send a "GET" request to "http://path-rules-slash-foo-exact/foo/" + Then the response status-code must be 500 + + Scenario: An Ingress with path rules /foo should send traffic to the matching backend service + (Exact /foo dose not matches request /foo/a/b/c/d/e/f/g) + When I send a "GET" request to "http://path-rules-slash-foo-exact/foo/a/b/c/d/e/f/g" + Then the response status-code must be 500 + + Scenario: An Ingress with path rules /foo should not send traffic to the matching backend service + (Prefix /foo dose not matches request /) + When I send a "GET" request to "http://path-rules-slash-foo-prefix/" + Then the response status-code must be 500 + + Scenario: An Ingress with path rules /foo should send traffic to the matching backend service + (Prefix /foo matches request /foo) + When I send a "GET" request to "http://path-rules-slash-foo-prefix/foo" + Then the response status-code must be 200 + And the response must be served by the "slash-foo-prefix" service + + Scenario: An Ingress with path rules /foo should send traffic to the matching backend service + (Prefix /foo matches request /foo/) + When I send a "GET" request to "http://path-rules-slash-foo-prefix/foo/" + Then the response status-code must be 200 + And the response must be served by the "slash-foo-prefix" service + + Scenario: An Ingress with path rules /foo should send traffic to the matching backend service + (Prefix /foo matches request /foo/a/b/c/d/e/f/g) + When I send a "GET" request to "http://path-rules-slash-foo-prefix/foo/" + Then the response status-code must be 200 + And the response must be served by the "slash-foo-prefix" service + + Scenario: An Ingress with path rules /foo/ should not send traffic to the matching backend service + (default /foo/ dose not matches request /) + When I send a "GET" request to "http://path-rules-slash-foo-slash/" + Then the response status-code must be 500 + + Scenario: An Ingress with path rules /foo/ should send traffic to the matching backend service + (default /foo/ matches request /foo) + When I send a "GET" request to "http://path-rules-slash-foo-slash/foo" + Then the response status-code must be 200 + And the response must be served by the "slash-foo-slash" service + + Scenario: An Ingress with path rules /foo/ should send traffic to the matching backend service + (default /foo/ matches request /foo/) + When I send a "GET" request to "http://path-rules-slash-foo-slash/foo/" + Then the response status-code must be 200 + And the response must be served by the "slash-foo-slash" service + + Scenario: An Ingress with path rules /foo/ should send traffic to the matching backend service + (default /foo/ matches request /foo/a/b/c/d/e/f/g) + When I send a "GET" request to "http://path-rules-slash-foo-slash/foo/a/b/c/d/e/f/g" + Then the response status-code must be 200 + And the response must be served by the "slash-foo-slash" service + + Scenario: An Ingress with path rules /foo/ should not send traffic to the matching backend service + (Exact /foo/ dose not matches request /) + When I send a "GET" request to "http://path-rules-slash-foo-slash-exact/" + Then the response status-code must be 500 + + Scenario: An Ingress with path rules /foo/ should not send traffic to the matching backend service + (Exact /foo/ dose not matches request /foo) + When I send a "GET" request to "http://path-rules-slash-foo-slash-exact/foo" + Then the response status-code must be 500 + + Scenario: An Ingress with path rules /foo/ should send traffic to the matching backend service + (Exact /foo/ matches request /foo/) + When I send a "GET" request to "http://path-rules-slash-foo-slash-exact/foo/" + Then the response status-code must be 200 + And the response must be served by the "slash-foo-slash-exact" service + + Scenario: An Ingress with path rules /foo/ should not send traffic to the matching backend service + (Exact /foo/ dose not matches request /foo/a/b/c/d/e/f/g) + When I send a "GET" request to "http://path-rules-slash-foo-slash-exact/foo/a/b/c/d/e/f/g" + Then the response status-code must be 500 + + Scenario: An Ingress with path rules /foo/ should not send traffic to the matching backend service + (Prefix /foo/ dose not matches request /) + When I send a "GET" request to "http://path-rules-slash-foo-slash-prefix/" + Then the response status-code must be 500 + + Scenario: An Ingress with path rules /foo/ should send traffic to the matching backend service + (Prefix /foo/ matches request /foo) + When I send a "GET" request to "http://path-rules-slash-foo-slash-prefix/foo" + Then the response status-code must be 200 + And the response must be served by the "slash-foo-slash-prefix" service + + Scenario: An Ingress with path rules /foo/ should send traffic to the matching backend service + (Prefix /foo/ matches request /foo/) + When I send a "GET" request to "http://path-rules-slash-foo-slash-prefix/foo/" + Then the response status-code must be 200 + And the response must be served by the "slash-foo-slash-prefix" service + + Scenario: An Ingress with path rules /foo/ should send traffic to the matching backend service + (Prefix /foo/ matches request /foo/a/b/c/d/e/f/g) + When I send a "GET" request to "http://path-rules-slash-foo-slash-prefix/foo/a/b/c/d/e/f/g" + Then the response status-code must be 200 + And the response must be served by the "slash-foo-slash-prefix" service + + Scenario: An Ingress with path rules /foo/ should send traffic to the matching backend service + (Prefix /foo/ matches request /foo/a/b/c/d/e/f/g) + When I send a "GET" request to "http://path-rules-slash-foo-slash-prefix/foo/a/b/c/d/e/f/g" + Then the response status-code must be 200 + And the response must be served by the "slash-foo-slash-prefix" service + + Scenario: An Ingress with path rules /aaa/bb should not send traffic to the matching backend service + (default /aaa/bb dose not matches request /aaa/b) + When I send a "GET" request to "http://path-rules-slash-aaa-slash-bb/aaa/b" + Then the response status-code must be 500 + + Scenario: An Ingress with path rules /aaa/bb should send traffic to the matching backend service + (default /aaa/bb matches request /aaa/bb) + When I send a "GET" request to "http://path-rules-slash-aaa-slash-bb/aaa/bb" + Then the response status-code must be 200 + And the response must be served by the "slash-aaa-slash-bb" service + + Scenario: An Ingress with path rules /aaa/bb should send traffic to the matching backend service + (default /aaa/bb matches request /aaa/bb/) + When I send a "GET" request to "http://path-rules-slash-aaa-slash-bb/aaa/bb/" + Then the response status-code must be 200 + And the response must be served by the "slash-aaa-slash-bb" service + + Scenario: An Ingress with path rules /aaa/bb should send traffic to the matching backend service + (default /aaa/bb matches request /aaa/bb/a/b/c/d/e/f/g) + When I send a "GET" request to "http://path-rules-slash-aaa-slash-bb/aaa/bb/a/b/c/d/e/f/g" + Then the response status-code must be 200 + And the response must be served by the "slash-aaa-slash-bb" service + + Scenario: An Ingress with path rules /aaa/bb should not send traffic to the matching backend service + (default /aaa/bb dose not matches request /aaa/bbb) + When I send a "GET" request to "http://path-rules-slash-aaa-slash-bb/aaa/bbb" + Then the response status-code must be 500 + + Scenario: An Ingress with path rules exact /aaa/bb should not send traffic to the matching backend service + (exact /aaa/bb dose not matches request /aaa/b) + When I send a "GET" request to "http://path-rules-slash-aaa-slash-bb-exact/aaa/b" + Then the response status-code must be 500 + + Scenario: An Ingress with path rules exact /aaa/bb should send traffic to the matching backend service + (exact /aaa/bb matches request /aaa/bb) + When I send a "GET" request to "http://path-rules-slash-aaa-slash-bb-exact/aaa/bb" + Then the response status-code must be 200 + And the response must be served by the "slash-aaa-slash-bb-exact" service + + Scenario: An Ingress with path rules exact /aaa/bb should not send traffic to the matching backend service + (exact /aaa/bb dose not matches request /aaa/bb/) + When I send a "GET" request to "http://path-rules-slash-aaa-slash-bb-exact/aaa/bb/" + Then the response status-code must be 500 + + Scenario: An Ingress with path rules exact /aaa/bb should not send traffic to the matching backend service + (exact /aaa/bb dose not matches request /aaa/bb/a/b/c/e/f/g) + When I send a "GET" request to "http://path-rules-slash-aaa-slash-bb-exact/aaa/bb/a/b/c/e/f/g" + Then the response status-code must be 500 + + Scenario: An Ingress with path rules exact /aaa/bb should not send traffic to the matching backend service + (exact /aaa/bb dose not matches request /aaa/bbb) + When I send a "GET" request to "http://path-rules-slash-aaa-slash-bb-exact/aaa/bbb" + Then the response status-code must be 500 + + Scenario: An Ingress with path rules prefix /aaa/bb should not send traffic to the matching backend service + (prefix /aaa/bb dose not matches request /aaa/b) + When I send a "GET" request to "http://path-rules-slash-aaa-slash-bb-prefix/aaa/b" + Then the response status-code must be 500 + + Scenario: An Ingress with path rules prefix /aaa/bb should send traffic to the matching backend service + (prefix /aaa/bb matches request /aaa/bb) + When I send a "GET" request to "http://path-rules-slash-aaa-slash-bb-prefix/aaa/bb" + Then the response status-code must be 200 + And the response must be served by the "slash-aaa-slash-bb-prefix" service + + Scenario: An Ingress with path rules prefix /aaa/bb should send traffic to the matching backend service + (prefix /aaa/bb matches request /aaa/bb/) + When I send a "GET" request to "http://path-rules-slash-aaa-slash-bb-prefix/aaa/bb/" + Then the response status-code must be 200 + And the response must be served by the "slash-aaa-slash-bb-prefix" service + + Scenario: An Ingress with path rules prefix /aaa/bb should send traffic to the matching backend service + (prefix /aaa/bb matches request /aaa/bb/a/b/c/d/e/f/g) + When I send a "GET" request to "http://path-rules-slash-aaa-slash-bb-prefix/aaa/bb/a/b/c/d/e/f/g" + Then the response status-code must be 200 + And the response must be served by the "slash-aaa-slash-bb-prefix" service + + Scenario: An Ingress with path rules prefix /aaa/bb should send traffic to the matching backend service + (prefix /aaa/bb matches request /aaa/bbb) + When I send a "GET" request to "http://path-rules-slash-aaa-slash-bb-prefix/aaa/bbb" + Then the response status-code must be 500 + + Scenario: An Ingress with path rules default /aaa/bbb should not send traffic to the matching backend service + (default /aaa/bbb dose not matches request /aaa/bb) + When I send a "GET" request to "http://path-rules-slash-aaa-slash-bbb/aaa/bb" + Then the response status-code must be 500 + + Scenario: An Ingress with path rules default /aaa/bbb should send traffic to the matching backend service + (default /aaa/bbb matches request /aaa/bbb) + When I send a "GET" request to "http://path-rules-slash-aaa-slash-bbb/aaa/bbb" + Then the response status-code must be 200 + And the response must be served by the "slash-aaa-slash-bbb" service + + Scenario: An Ingress with path rules default /aaa/bbb should not send traffic to the matching backend service + (default /aaa/bbb dose not matches request /aaa/bbbxyz) + When I send a "GET" request to "http://path-rules-slash-aaa-slash-bbb/aaa/bbbxyz" + Then the response status-code must be 500 + + Scenario: An Ingress with path rules default /aaa/bbb should send traffic to the matching backend service + (default /aaa/bbb matches request /aaa/bbb/) + When I send a "GET" request to "http://path-rules-slash-aaa-slash-bbb/aaa/bbb/" + Then the response status-code must be 200 + And the response must be served by the "slash-aaa-slash-bbb" service + + Scenario: An Ingress with path rules default /aaa/bbb should send traffic to the matching backend service + (default /aaa/bbb matches request /aaa/bbb/ccc) + When I send a "GET" request to "http://path-rules-slash-aaa-slash-bbb/aaa/bbb/ccc" + Then the response status-code must be 200 + And the response must be served by the "slash-aaa-slash-bbb" service + + Scenario: An Ingress with path rules exact /aaa/bbb should not send traffic to the matching backend service + (exact /aaa/bbb dose not matches request /aaa/bb) + When I send a "GET" request to "http://path-rules-slash-aaa-slash-bbb-exact/aaa/bb" + Then the response status-code must be 500 + + Scenario: An Ingress with path rules exact /aaa/bbb should send traffic to the matching backend service + (exact /aaa/bbb matches request /aaa/bbb) + When I send a "GET" request to "http://path-rules-slash-aaa-slash-bbb-exact/aaa/bbb" + Then the response status-code must be 200 + And the response must be served by the "slash-aaa-slash-bbb-exact" service + + Scenario: An Ingress with path rules exact /aaa/bbb should not send traffic to the matching backend service + (exact /aaa/bbb dose not matches request /aaa/bbb/) + When I send a "GET" request to "http://path-rules-slash-aaa-slash-bbb-exact/aaa/bbb/" + Then the response status-code must be 500 + + Scenario: An Ingress with path rules exact /aaa/bbb should not send traffic to the matching backend service + (exact /aaa/bbb dose not matches request /aaa/bbb/ccc) + When I send a "GET" request to "http://path-rules-slash-aaa-slash-bbb-exact/aaa/bbb/ccc" + Then the response status-code must be 500 + + Scenario: An Ingress with path rules prefix /aaa/bbb should not send traffic to the matching backend service + (prefix /aaa/bbb dose not matches request /aaa/bb) + When I send a "GET" request to "http://path-rules-slash-aaa-slash-bbb-prefix/aaa/bb" + Then the response status-code must be 500 + + Scenario: An Ingress with path rules prefix /aaa/bbb should send traffic to the matching backend service + (prefix /aaa/bbb matches request /aaa/bbb) + When I send a "GET" request to "http://path-rules-slash-aaa-slash-bbb-prefix/aaa/bbb" + Then the response status-code must be 200 + And the response must be served by the "slash-aaa-slash-bbb-prefix" service + + Scenario: An Ingress with path rules prefix /aaa/bbb should not send traffic to the matching backend service + (prefix /aaa/bbb dose not matches request /aaa/bbbxyz) + When I send a "GET" request to "http://path-rules-slash-aaa-slash-bbb-prefix/aaa/bbbxyz" + Then the response status-code must be 500 + + Scenario: An Ingress with path rules prefix /aaa/bbb should not send traffic to the matching backend service + (prefix /aaa/bbb dose not matches request /aaa/bbb/) + When I send a "GET" request to "http://path-rules-slash-aaa-slash-bbb-prefix/aaa/bbb/" + Then the response status-code must be 200 + And the response must be served by the "slash-aaa-slash-bbb-prefix" service + + Scenario: An Ingress with path rules default /aaa/bbb/ should not send traffic to the matching backend service + (default /aaa/bbb/ dose not matches request /aaa/bb) + When I send a "GET" request to "http://path-rules-slash-aaa-slash-bbb-slash/aaa/bb" + Then the response status-code must be 500 + + Scenario: An Ingress with path rules default /aaa/bbb/ should send traffic to the matching backend service + (default /aaa/bbb/ matches request /aaa/bbb) + When I send a "GET" request to "http://path-rules-slash-aaa-slash-bbb-slash/aaa/bbb" + Then the response status-code must be 200 + And the response must be served by the "slash-aaa-slash-bbb-slash" service + + Scenario: An Ingress with path rules default /aaa/bbb/ should send traffic to the matching backend service + (default /aaa/bbb/ matches request /aaa/bbbxyz) + When I send a "GET" request to "http://path-rules-slash-aaa-slash-bbb-slash/aaa/bbbxyz" + Then the response status-code must be 500 + + Scenario: An Ingress with path rules default /aaa/bbb/ should send traffic to the matching backend service + (default /aaa/bbb/ matches request /aaa/bbb/) + When I send a "GET" request to "http://path-rules-slash-aaa-slash-bbb-slash/aaa/bbb/" + Then the response status-code must be 200 + And the response must be served by the "slash-aaa-slash-bbb-slash" service + + Scenario: An Ingress with path rules default /aaa/bbb/ should send traffic to the matching backend service + (default /aaa/bbb/ matches request /aaa/bbb/ccc) + When I send a "GET" request to "http://path-rules-slash-aaa-slash-bbb-slash/aaa/bbb/ccc" + Then the response status-code must be 200 + And the response must be served by the "slash-aaa-slash-bbb-slash" service + + Scenario: An Ingress with path rules exact /aaa/bbb/ should send not traffic to the matching backend service + (exact /aaa/bbb/ dose not matches request /aaa/bb) + When I send a "GET" request to "http://path-rules-slash-aaa-slash-bbb-slash-exact/aaa/bb" + Then the response status-code must be 500 + + Scenario: An Ingress with path rules exact /aaa/bbb/ should send not traffic to the matching backend service + (exact /aaa/bbb/ dose not matches request /aaa/bbb) + When I send a "GET" request to "http://path-rules-slash-aaa-slash-bbb-slash-exact/aaa/bbb" + Then the response status-code must be 500 + #And the response must be served by the "slash-aaa-slash-bbb-slash-exact" service + + Scenario: An Ingress with path rules exact /aaa/bbb/ should send not traffic to the matching backend service + (exact /aaa/bbb/ dose not matches request /aaa/bbbxyz) + When I send a "GET" request to "http://path-rules-slash-aaa-slash-bbb-slash-exact/aaa/bbbxyz" + Then the response status-code must be 500 + + Scenario: An Ingress with path rules exact /aaa/bbb/ should send not traffic to the matching backend service + (exact /aaa/bbb/ dose not matches request /aaa/bbb/) + When I send a "GET" request to "http://path-rules-slash-aaa-slash-bbb-slash-exact/aaa/bbb/" + Then the response status-code must be 200 + And the response must be served by the "slash-aaa-slash-bbb-slash-exact" service + + Scenario: An Ingress with path rules exact /aaa/bbb/ should send not traffic to the matching backend service + (exact /aaa/bbb/ dose not matches request /aaa/bbb/ccc) + When I send a "GET" request to "http://path-rules-slash-aaa-slash-bbb-slash-exact/aaa/bbb/ccc" + Then the response status-code must be 500 + + Scenario: An Ingress with path rules prefix /aaa/bbb/ should send not traffic to the matching backend service + (prefix /aaa/bbb/ dose not matches request /aaa/bb) + When I send a "GET" request to "http://path-rules-slash-aaa-slash-bbb-slash-prefix/aaa/bb" + Then the response status-code must be 500 + + Scenario: An Ingress with path rules prefix /aaa/bbb/ should send traffic to the matching backend service + (prefix /aaa/bbb/ matches request /aaa/bbb) + When I send a "GET" request to "http://path-rules-slash-aaa-slash-bbb-slash-prefix/aaa/bbb" + Then the response status-code must be 200 + And the response must be served by the "slash-aaa-slash-bbb-slash-prefix" service + + Scenario: An Ingress with path rules prefix /aaa/bbb/ should send traffic to the matching backend service + (prefix /aaa/bbb/ matches request /aaa/bbbxyz) + When I send a "GET" request to "http://path-rules-slash-aaa-slash-bbb-slash-prefix/aaa/bbbxyz" + Then the response status-code must be 500 + + Scenario: An Ingress with path rules prefix /aaa/bbb/ should send traffic to the matching backend service + (prefix /aaa/bbb/ matches request /aaa/bbb/) + When I send a "GET" request to "http://path-rules-slash-aaa-slash-bbb-slash-prefix/aaa/bbb/" + Then the response status-code must be 200 + And the response must be served by the "slash-aaa-slash-bbb-slash-prefix" service + + Scenario: An Ingress with path rules prefix /aaa/bbb/ should send traffic to the matching backend service + (prefix /aaa/bbb/ matches request /aaa/bbb/ccc) + When I send a "GET" request to "http://path-rules-slash-aaa-slash-bbb-slash-prefix/aaa/bbb/ccc" + Then the response status-code must be 200 + And the response must be served by the "slash-aaa-slash-bbb-slash-prefix" service diff --git a/test/e2e/features/rules/path_rule2.feature b/test/e2e/features/rules/path_rule2.feature new file mode 100644 index 00000000..fd47b9a5 --- /dev/null +++ b/test/e2e/features/rules/path_rule2.feature @@ -0,0 +1,374 @@ +@ingress.rule @release-1.22 +Feature: Path test + An Ingress may define routing rules based on the request path. + + If the HTTP request path matches one of the paths in the + Ingress objects, the traffic is routed to its backend service. + + Background: + Given an Ingress resource in a new random namespace + """ + apiVersion: networking.k8s.io/v1 + kind: Ingress + metadata: + name: multi-path + spec: + rules: + - host: "all-prefix" + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: single-slash-type-prefix + port: + number: 3000 + - path: /aaa + pathType: Prefix + backend: + service: + name: single-slash-aaa-type-prefix + port: + number: 3000 + + - host: "prefix-exact" + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: single-slash-type-prefix-mix + port: + number: 3000 + - path: /aaa + pathType: Exact + backend: + service: + name: single-slash-aaa-type-exact-mix + port: + number: 3000 + + - host: "exact-prefix" + http: + paths: + - path: / + pathType: Exact + backend: + service: + name: single-slash-type-exact-mix + port: + number: 3000 + - path: /aaa + pathType: Prefix + backend: + service: + name: single-slash-aaa-type-prefix-mix + port: + number: 3000 + + - host: "all-exact" + http: + paths: + - path: / + pathType: Exact + backend: + service: + name: single-slash-type-exact + port: + number: 3000 + - path: /aaa + pathType: Exact + backend: + service: + name: single-slash-aaa-type-exact + port: + number: 3000 + + - host: "multi-path-route" + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: single-slash + port: + number: 3000 + - path: /aaa + pathType: Prefix + backend: + service: + name: single-slash-aaa + port: + number: 3000 + - path: /aaa/bbb + pathType: Prefix + backend: + service: + name: single-slash-aaa-bbb + port: + number: 3000 + + """ + Then The Ingress status shows the IP address or FQDN where it is exposed + + Scenario: An Ingress with prefix path rules / and /aaa should send traffic to the matching backend service + (prefix / and /aaa matches request / and route to single-slash-type-prefix) + + When I send a "GET" request to "http://all-prefix/" + Then the response status-code must be 200 + And the response must be served by the "single-slash-type-prefix" service + + Scenario: An Ingress with prefix path rules / and /aaa should send traffic to the matching backend service + (prefix / and /aaa matches request /a and route to single-slash-type-prefix) + + When I send a "GET" request to "http://all-prefix/a" + Then the response status-code must be 200 + And the response must be served by the "single-slash-type-prefix" service + + Scenario: An Ingress with prefix path rules / and /aaa should send traffic to the matching backend service + (prefix / and /aaa matches request /aaa and route to single-slash-aaa-type-prefix) + + When I send a "GET" request to "http://all-prefix/aaa" + Then the response status-code must be 200 + And the response must be served by the "single-slash-aaa-type-prefix" service + + Scenario: An Ingress with prefix path rules / and /aaa should send traffic to the matching backend service + (prefix / and /aaa matches request /aaa/ and route to single-slash-aaa-type-prefix) + + When I send a "GET" request to "http://all-prefix/aaa/" + Then the response status-code must be 200 + And the response must be served by the "single-slash-aaa-type-prefix" service + + Scenario: An Ingress with prefix path rules / and /aaa should send traffic to the matching backend service + (prefix / and /aaa matches request /aaa/ and route to single-slash-type-prefix) + + When I send a "GET" request to "http://all-prefix/aaaxyz" + Then the response status-code must be 200 + And the response must be served by the "single-slash-type-prefix" service + + Scenario: An Ingress with prefix path rules / and /aaa should send traffic to the matching backend service + (prefix / and /aaa matches request /aaa/ccc and route to single-slash-aaa-type-prefix) + + When I send a "GET" request to "http://all-prefix/aaa/ccc" + Then the response status-code must be 200 + And the response must be served by the "single-slash-aaa-type-prefix" service + + Scenario: An Ingress with prefix path rules / and /aaa should send traffic to the matching backend service + (prefix / and /aaa matches request /aaa/ccc/a/b/c/d/e/f/g and route to single-slash-aaa-type-prefix) + + When I send a "GET" request to "http://all-prefix/aaa/ccc/a/b/c/d/e/f/g" + Then the response status-code must be 200 + And the response must be served by the "single-slash-aaa-type-prefix" service + + Scenario: An Ingress with prefix path rules / and exact path rules /aaa should send traffic to the matching backend service + (prefix / and /aaa matches request / and route to single-slash-type-prefix-mix) + + When I send a "GET" request to "http://prefix-exact/" + Then the response status-code must be 200 + And the response must be served by the "single-slash-type-prefix-mix" service + + Scenario: An Ingress with prefix path rules / and exact path rules /aaa should send traffic to the matching backend service + (prefix / and /aaa matches request /a and route to single-slash-type-prefix-mix) + + When I send a "GET" request to "http://prefix-exact/a" + Then the response status-code must be 200 + And the response must be served by the "single-slash-type-prefix-mix" service + + Scenario: An Ingress with prefix path rules / and exact path rules /aaa should send traffic to the matching backend service + (prefix / and /aaa matches request /aaa and route to single-slash-type-prefix) + + When I send a "GET" request to "http://prefix-exact/aaa" + Then the response status-code must be 200 + And the response must be served by the "single-slash-aaa-type-exact-mix" service + + Scenario: An Ingress with prefix path rules / and exact path rules /aaa should send traffic to the matching backend service + (prefix / and /aaa matches request /aaa and route to single-slash-type-prefix-mix) + + When I send a "GET" request to "http://prefix-exact/aaa/" + Then the response status-code must be 200 + And the response must be served by the "single-slash-type-prefix-mix" service + + Scenario: An Ingress with prefix path rules / and exact path rules /aaa should send traffic to the matching backend service + (prefix / and /aaa matches request /aaaxyz and route to single-slash-type-prefix-mix) + + When I send a "GET" request to "http://prefix-exact/aaaxyz" + Then the response status-code must be 200 + And the response must be served by the "single-slash-type-prefix-mix" service + + Scenario: An Ingress with prefix path rules / and exact path rules /aaa should send traffic to the matching backend service + (prefix / and /aaa matches request /aaa/ccc and route to single-slash-type-prefix-mix) + + When I send a "GET" request to "http://prefix-exact/aaa/ccc" + Then the response status-code must be 200 + And the response must be served by the "single-slash-type-prefix-mix" service + + Scenario: An Ingress with prefix path rules / and exact path rules /aaa should send traffic to the matching backend service + (prefix / and /aaa matches request /aaa/ccc/a/b/c/d/e/f/g and route to single-slash-type-prefix-mix) + + When I send a "GET" request to "http://prefix-exact/aaa/ccc/a/b/c/d/e/f/g" + Then the response status-code must be 200 + And the response must be served by the "single-slash-type-prefix-mix" service + + Scenario: An Ingress with exact path rules / and prefix path rules /aaa should send traffic to the matching backend service + (exact / and prefix /aaa matches request / and route to single-slash-type-exact-mix) + + When I send a "GET" request to "http://exact-prefix/" + Then the response status-code must be 200 + And the response must be served by the "single-slash-type-exact-mix" service + + Scenario: An Ingress with exact path rules / and prefix path rules /aaa should send traffic to the matching backend service + (exact / and prefix /aaa matches request /a and not route to backend) + + When I send a "GET" request to "http://exact-prefix/a" + Then the response status-code must be 500 + + Scenario: An Ingress with exact path rules / and prefix path rules /aaa should send traffic to the matching backend service + (exact / and prefix /aaa matches request /aaa and route to single-slash-aaa-type-prefix-mix) + + When I send a "GET" request to "http://exact-prefix/aaa" + Then the response status-code must be 200 + And the response must be served by the "single-slash-aaa-type-prefix-mix" service + + Scenario: An Ingress with exact path rules / and prefix path rules /aaa should send traffic to the matching backend service + (exact / and prefix /aaa matches request /aaa and route to single-slash-aaa-type-prefix-mix) + + When I send a "GET" request to "http://exact-prefix/aaa" + Then the response status-code must be 200 + And the response must be served by the "single-slash-aaa-type-prefix-mix" service + + Scenario: An Ingress with exact path rules / and prefix path rules /aaa should send traffic to the matching backend service + (exact / and prefix /aaa matches request /aaaxyz and not route to backend) + + When I send a "GET" request to "http://exact-prefix/aaaxyz" + Then the response status-code must be 500 + + Scenario: An Ingress with exact path rules / and prefix path rules /aaa should send traffic to the matching backend service + (exact / and prefix /aaa matches request /aaa/ccc and route to single-slash-aaa-type-prefix-mix) + + When I send a "GET" request to "http://exact-prefix/aaa/ccc" + Then the response status-code must be 200 + And the response must be served by the "single-slash-aaa-type-prefix-mix" service + + Scenario: An Ingress with exact path rules / and prefix path rules /aaa should send traffic to the matching backend service + (exact / and prefix /aaa matches request /aaa/ccc/a/b/c/d/e/f/g and route to single-slash-aaa-type-prefix-mix) + + When I send a "GET" request to "http://exact-prefix/aaa/ccc/a/b/c/d/e/f/g" + Then the response status-code must be 200 + And the response must be served by the "single-slash-aaa-type-prefix-mix" service + + Scenario: An Ingress with exact path rules / and path rules /aaa should send traffic to the matching backend service + (exact / and exact /aaa matches request / and route to single-slash-type-exact) + + When I send a "GET" request to "http://all-exact/" + Then the response status-code must be 200 + And the response must be served by the "single-slash-type-exact" service + + Scenario: An Ingress with exact path rules / and path rules /aaa should send traffic to the matching backend service + (exact / and exact /aaa matches request /a and not route to backend) + + When I send a "GET" request to "http://all-exact/a" + Then the response status-code must be 500 + + Scenario: An Ingress with exact path rules / and path rules /aaa should send traffic to the matching backend service + (exact / and exact /aaa matches request /aaa and route to single-slash-aaa-type-exact) + + When I send a "GET" request to "http://all-exact/aaa" + Then the response status-code must be 200 + And the response must be served by the "single-slash-aaa-type-exact" service + + Scenario: An Ingress with exact path rules / and path rules /aaa should send traffic to the matching backend service + (exact / and exact /aaa matches request /aaa/ and not route to backend) + + When I send a "GET" request to "http://all-exact/aaa/" + Then the response status-code must be 500 + + Scenario: An Ingress with exact path rules / and path rules /aaa should send traffic to the matching backend service + (exact / and exact /aaa matches request /aaaxyz and not route to backend) + + When I send a "GET" request to "http://all-exact/aaazyx" + Then the response status-code must be 500 + + Scenario: An Ingress with exact path rules / and path rules /aaa should send traffic to the matching backend service + (exact / and exact /aaa matches request /aaa/ccc and not route to backend) + + When I send a "GET" request to "http://all-exact/aaa/ccc" + Then the response status-code must be 500 + + Scenario: An Ingress with exact path rules / and path rules /aaa should send traffic to the matching backend service + (exact / and exact /aaa matches request /aaa/ccc/a/b/c/d/e/f/g and not route to backend) + + When I send a "GET" request to "http://all-exact/aaa/ccc/a/b/c/d/e/f/g" + Then the response status-code must be 500 + + Scenario: An Ingress with path rules / and path rules /aaa and /aaa/bbb should send traffic to the matching backend service + (prefix /, /aaa, /aaa/bbb matches request / and route to single-slash) + + When I send a "GET" request to "http://multi-path-route/" + Then the response status-code must be 200 + And the response must be served by the "single-slash" service + + Scenario: An Ingress with path rules / and path rules /aaa and /aaa/bbb should send traffic to the matching backend service + (prefix /, /aaa, /aaa/bbb matches request /a and route to single-slash) + + When I send a "GET" request to "http://multi-path-route/a" + Then the response status-code must be 200 + And the response must be served by the "single-slash" service + + Scenario: An Ingress with path rules / and path rules /aaa and /aaa/bbb should send traffic to the matching backend service + (prefix /, /aaa, /aaa/bbb matches request /aaa and route to single-slash-aaa) + + When I send a "GET" request to "http://multi-path-route/aaa" + Then the response status-code must be 200 + And the response must be served by the "single-slash-aaa" service + + Scenario: An Ingress with path rules / and path rules /aaa and /aaa/bbb should send traffic to the matching backend service + (prefix /, /aaa, /aaa/bbb matches request /aaa/ and route to single-slash-aaa) + + When I send a "GET" request to "http://multi-path-route/aaa/" + Then the response status-code must be 200 + And the response must be served by the "single-slash-aaa" service + + Scenario: An Ingress with path rules / and path rules /aaa and /aaa/bbb should send traffic to the matching backend service + (prefix /, /aaa, /aaa/bbb matches request /aaa/bb and route to single-slash-aaa) + + When I send a "GET" request to "http://multi-path-route/aaa/bb" + Then the response status-code must be 200 + And the response must be served by the "single-slash-aaa" service + + Scenario: An Ingress with path rules / and path rules /aaa and /aaa/bbb should send traffic to the matching backend service + (prefix /, /aaa, /aaa/bbb matches request /aaa/bbb and route to single-slash-aaa-bbb) + + When I send a "GET" request to "http://multi-path-route/aaa/bbb" + Then the response status-code must be 200 + And the response must be served by the "single-slash-aaa-bbb" service + + Scenario: An Ingress with path rules / and path rules /aaa and /aaa/bbb should send traffic to the matching backend service + (prefix /, /aaa, /aaa/bbb matches request /aaa/bbb/ and route to single-slash-aaa-bbb) + + When I send a "GET" request to "http://multi-path-route/aaa/bbb/" + Then the response status-code must be 200 + And the response must be served by the "single-slash-aaa-bbb" service + + Scenario: An Ingress with path rules / and path rules /aaa and /aaa/bbb should send traffic to the matching backend service + (prefix /, /aaa, /aaa/bbb matches request /aaa/bbb/ccc and route to single-slash-aaa-bbb) + + When I send a "GET" request to "http://multi-path-route/aaa/bbb/ccc" + Then the response status-code must be 200 + And the response must be served by the "single-slash-aaa-bbb" service + + Scenario: An Ingress with path rules / and path rules /aaa and /aaa/bbb should send traffic to the matching backend service + (prefix /, /aaa, /aaa/bbb matches request /ccc and route to single-slash) + + When I send a "GET" request to "http://multi-path-route/ccc" + Then the response status-code must be 200 + And the response must be served by the "single-slash" service + + Scenario: An Ingress with path rules / and path rules /aaa and /aaa/bbb should send traffic to the matching backend service + (prefix /, /aaa, /aaa/bbb matches request /aaa/ccc and route to single-slash-aaa) + + When I send a "GET" request to "http://multi-path-route/aaa/ccc" + Then the response status-code must be 200 + And the response must be served by the "single-slash-aaa" service diff --git a/test/e2e/go.mod b/test/e2e/go.mod new file mode 100644 index 00000000..ad8ae2b0 --- /dev/null +++ b/test/e2e/go.mod @@ -0,0 +1,17 @@ +module github.com/bfenetworks/ingress-bfe/test/e2e + +go 1.16 + +require ( + github.com/cucumber/gherkin-go/v11 v11.0.0 + github.com/cucumber/godog v0.12.3 + github.com/cucumber/messages-go/v10 v10.0.3 + github.com/cucumber/messages-go/v16 v16.0.1 // indirect + github.com/iancoleman/orderedmap v0.1.0 + golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4 + k8s.io/api v0.19.2 + k8s.io/apimachinery v0.19.2 + k8s.io/client-go v0.19.2 + k8s.io/klog/v2 v2.3.0 + sigs.k8s.io/yaml v1.2.0 +) diff --git a/test/e2e/go.sum b/test/e2e/go.sum new file mode 100644 index 00000000..503a57ca --- /dev/null +++ b/test/e2e/go.sum @@ -0,0 +1,913 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.34.0 h1:eOI3/cP2VTU6uZLDYAoic+eyzzB9YyGmJ7eIjl8rOPg= +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.38.0 h1:ROfEUZz+Gh5pa62DJWXSaonyu3StP6EA6lPEXPI6mCo= +cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= +cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= +cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= +cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= +cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= +cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= +cloud.google.com/go v0.51.0 h1:PvKAVQWCtlGUSlZkGW3QLelKaWq7KYv/MW1EboG8bfM= +cloud.google.com/go v0.51.0/go.mod h1:hWtGJ6gnXH+KgDv+V0zFGDvpi07n3z8ZNj3T1RW0Gcw= +cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= +cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= +cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= +cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= +cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= +cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= +cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= +cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= +cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= +cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg= +cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8= +cloud.google.com/go v0.81.0 h1:at8Tk2zUz63cLPR0JPWm5vp77pEZmzxEQBEfRKn1VV8= +cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0= +cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= +cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= +cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= +cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= +cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= +cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= +cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= +cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= +cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= +cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= +cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= +cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= +cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= +cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= +cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= +cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= +cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= +cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= +dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= +github.com/Azure/go-autorest/autorest v0.9.0 h1:MRvx8gncNaXJqOoLmhNjUAKh33JJF8LyxPhomEtOsjs= +github.com/Azure/go-autorest/autorest v0.9.0/go.mod h1:xyHB1BMZT0cuDHU7I0+g046+BFDTQ8rEZB0s4Yfa6bI= +github.com/Azure/go-autorest/autorest v0.9.6 h1:5YWtOnckcudzIw8lPPBcWOnmIFWMtHci1ZWAZulMSx0= +github.com/Azure/go-autorest/autorest v0.9.6/go.mod h1:/FALq9T/kS7b5J5qsQ+RSTUdAmGFqi0vUdVNNx8q630= +github.com/Azure/go-autorest/autorest v0.11.18 h1:90Y4srNYrwOtAgVo3ndrQkTYn6kf1Eg/AjTFJ8Is2aM= +github.com/Azure/go-autorest/autorest v0.11.18/go.mod h1:dSiJPy22c3u0OtOKDNttNgqpNFY/GeWa7GH/Pz56QRA= +github.com/Azure/go-autorest/autorest/adal v0.5.0 h1:q2gDruN08/guU9vAjuPWff0+QIrpH6ediguzdAzXAUU= +github.com/Azure/go-autorest/autorest/adal v0.5.0/go.mod h1:8Z9fGy2MpX0PvDjB1pEgQTmVqjGhiHBW7RJJEciWzS0= +github.com/Azure/go-autorest/autorest/adal v0.8.2 h1:O1X4oexUxnZCaEUGsvMnr8ZGj8HI37tNezwY4npRqA0= +github.com/Azure/go-autorest/autorest/adal v0.8.2/go.mod h1:ZjhuQClTqx435SRJ2iMlOxPYt3d2C/T/7TiQCVZSn3Q= +github.com/Azure/go-autorest/autorest/adal v0.9.13 h1:Mp5hbtOePIzM8pJVRa3YLrWWmZtoxRXqUEzCfJt3+/Q= +github.com/Azure/go-autorest/autorest/adal v0.9.13/go.mod h1:W/MM4U6nLxnIskrw4UwWzlHfGjwUS50aOsc/I3yuU8M= +github.com/Azure/go-autorest/autorest/date v0.1.0 h1:YGrhWfrgtFs84+h0o46rJrlmsZtyZRg470CqAXTZaGM= +github.com/Azure/go-autorest/autorest/date v0.1.0/go.mod h1:plvfp3oPSKwf2DNjlBjWF/7vwR+cUD/ELuzDCXwHUVA= +github.com/Azure/go-autorest/autorest/date v0.2.0 h1:yW+Zlqf26583pE43KhfnhFcdmSWlm5Ew6bxipnr/tbM= +github.com/Azure/go-autorest/autorest/date v0.2.0/go.mod h1:vcORJHLJEh643/Ioh9+vPmf1Ij9AEBM5FuBIXLmIy0g= +github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74= +github.com/Azure/go-autorest/autorest/mocks v0.1.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0= +github.com/Azure/go-autorest/autorest/mocks v0.2.0 h1:Ww5g4zThfD/6cLb4z6xxgeyDa7QDkizMkJKe0ysZXp0= +github.com/Azure/go-autorest/autorest/mocks v0.2.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0= +github.com/Azure/go-autorest/autorest/mocks v0.3.0 h1:qJumjCaCudz+OcqE9/XtEPfvtOjOmKaui4EOpFI6zZc= +github.com/Azure/go-autorest/autorest/mocks v0.3.0/go.mod h1:a8FDP3DYzQ4RYfVAxAN3SVSiiO77gL2j2ronKKP0syM= +github.com/Azure/go-autorest/autorest/mocks v0.4.1 h1:K0laFcLE6VLTOwNgSxaGbUcLPuGXlNkbVvq4cW4nIHk= +github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= +github.com/Azure/go-autorest/logger v0.1.0 h1:ruG4BSDXONFRrZZJ2GUXDiUyVpayPmb1GnWeHDdaNKY= +github.com/Azure/go-autorest/logger v0.1.0/go.mod h1:oExouG+K6PryycPJfVSxi/koC6LSNgds39diKLz7Vrc= +github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= +github.com/Azure/go-autorest/tracing v0.5.0 h1:TRn4WjSnkcSy5AEG3pnbtFSwNtwzjr4VYyQflFE619k= +github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk= +github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= +github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/PuerkitoBio/purell v1.0.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= +github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= +github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= +github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= +github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= +github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= +github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= +github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= +github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= +github.com/aslakhellesoy/gox v1.0.100/go.mod h1:AJl542QsKKG96COVsv0N74HHzVQgDIQPceVUh1aeU2M= +github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= +github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= +github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= +github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= +github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/cucumber/gherkin-go/v11 v11.0.0 h1:cwVwN1Qn2VRSfHZNLEh5x00tPBmZcjATBWDpxsR5Xug= +github.com/cucumber/gherkin-go/v11 v11.0.0/go.mod h1:CX33k2XU2qog4e+TFjOValoq6mIUq0DmVccZs238R9w= +github.com/cucumber/gherkin-go/v19 v19.0.3 h1:mMSKu1077ffLbTJULUfM5HPokgeBcIGboyeNUof1MdE= +github.com/cucumber/gherkin-go/v19 v19.0.3/go.mod h1:jY/NP6jUtRSArQQJ5h1FXOUgk5fZK24qtE7vKi776Vw= +github.com/cucumber/godog v0.12.3 h1:nBshklqcWho/joTFtSBfyD4KYkvftwwf0r0XpX6ajNU= +github.com/cucumber/godog v0.12.3/go.mod h1:u6SD7IXC49dLpPN35kal0oYEjsXZWee4pW6Tm9t5pIc= +github.com/cucumber/messages-go/v10 v10.0.1/go.mod h1:kA5T38CBlBbYLU12TIrJ4fk4wSkVVOgyh7Enyy8WnSg= +github.com/cucumber/messages-go/v10 v10.0.3 h1:m/9SD/K/A15WP7i1aemIv7cwvUw+viS51Ui5HBw1cdE= +github.com/cucumber/messages-go/v10 v10.0.3/go.mod h1:9jMZ2Y8ZxjLY6TG2+x344nt5rXstVVDYSdS5ySfI1WY= +github.com/cucumber/messages-go/v16 v16.0.0/go.mod h1:EJcyR5Mm5ZuDsKJnT2N9KRnBK30BGjtYotDKpwQ0v6g= +github.com/cucumber/messages-go/v16 v16.0.1 h1:fvkpwsLgnIm0qugftrw2YwNlio+ABe2Iu94Ap8GMYIY= +github.com/cucumber/messages-go/v16 v16.0.1/go.mod h1:EJcyR5Mm5ZuDsKJnT2N9KRnBK30BGjtYotDKpwQ0v6g= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= +github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= +github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96 h1:cenwrSVm+Z7QLSV/BsnenAOcDXdX4cMv4wP0B/5QbPg= +github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= +github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= +github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153 h1:yUdfgN0XgIJw7foRItutHYUIhlcKzcSf5vDpdhQAKTc= +github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= +github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= +github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= +github.com/form3tech-oss/jwt-go v3.2.3+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= +github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= +github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= +github.com/getkin/kin-openapi v0.76.0/go.mod h1:660oXbgy5JFMKreazJaQTw7o+X00qeSyhcnluiMv+Xg= +github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= +github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= +github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= +github.com/go-logr/logr v0.2.0 h1:QvGt2nLcHH0WK9orKa+ppBPAxREcH364nPUedEpK0TY= +github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= +github.com/go-logr/logr v1.2.0 h1:QK40JKJyMdUDz+h+xvCsru/bJhvG0UxvePV0ufL/AcE= +github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-openapi/jsonpointer v0.0.0-20160704185906-46af16f9f7b1/go.mod h1:+35s3my2LFTysnkMfxsJBAMHj/DoqoB9knIWoYG/Vk0= +github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= +github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= +github.com/go-openapi/jsonreference v0.0.0-20160704190145-13c6e3589ad9/go.mod h1:W3Z9FmVs9qj+KR4zFKmDPGiLdk1D9Rlm7cyMvf57TTg= +github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8= +github.com/go-openapi/spec v0.0.0-20160808142527-6aced65f8501/go.mod h1:J8+jY1nAiCcj+friV/PDoE1/3eeccG9LYBs0tYvLOWc= +github.com/go-openapi/swag v0.0.0-20160704191624-1d0bd113de87/go.mod h1:DXUve3Dpr1UfpPtxFw+EFuQ41HhCWZfha5jSVRG7C7I= +github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= +github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/gofrs/uuid v3.2.0+incompatible h1:y12jRkkFxsd7GpqdSZ+/KCs/fJbqpEXSGd4+jfEaewE= +github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= +github.com/gofrs/uuid v4.0.0+incompatible h1:1SD/1F5pU8p29ybwgQSwpQk+mwdRrXCYuPhW6m+TnJw= +github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= +github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= +github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls= +github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7 h1:5ZkaAPbicIKTF2I64qf5Fh8Aa83Q/dnOafMYV0OMwjA= +github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= +github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= +github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= +github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo= +github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.0.1 h1:gK4Kx5IaGY9CD5sPJ36FHiBJ6ZXl0kilRiiCj+jdYp4= +github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/gofuzz v1.0.0 h1:A8PeW59pxE9IoFRqBp37U+mSNaQoZ46F1f0f863XSXw= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g= +github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= +github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= +github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/googleapis/gnostic v0.4.1 h1:DLJCy1n/vrD4HPjOvYcT8aYQXpPIzoRZONaYwyycI+I= +github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3ir6b65WBswg= +github.com/googleapis/gnostic v0.5.1/go.mod h1:6U4PtQXGIEt/Z3h5MAT7FNofLnw9vXk2cUuW7uA/OeU= +github.com/googleapis/gnostic v0.5.5/go.mod h1:7+EbHbldMins07ALC74bsA81Ovc97DwqyJO1AENw9kA= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= +github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= +github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7 h1:pdN6V1QBWetyv/0+wjACpqVH+eVULgEjkurDLq3goeM= +github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= +github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= +github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= +github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= +github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= +github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= +github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= +github.com/hashicorp/go-immutable-radix v1.2.0 h1:l6UW37iCXwZkZoAbEYnptSHVE/cQ5bOTPYG5W3vf9+8= +github.com/hashicorp/go-immutable-radix v1.2.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= +github.com/hashicorp/go-immutable-radix v1.3.0 h1:8exGP7ego3OmkfksihtSouGMZ+hQrhxx+FVELeXpVPE= +github.com/hashicorp/go-immutable-radix v1.3.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= +github.com/hashicorp/go-memdb v1.2.1 h1:wI9btDjYUOJJHTCnRlAG/TkRyD/ij7meJMrLK9X31Cc= +github.com/hashicorp/go-memdb v1.2.1/go.mod h1:OSvLJ662Jim8hMM+gWGyhktyWk2xPCnWMc7DWIqtkGA= +github.com/hashicorp/go-memdb v1.3.0 h1:xdXq34gBOMEloa9rlGStLxmfX/dyIK8htOv36dQUwHU= +github.com/hashicorp/go-memdb v1.3.0/go.mod h1:Mluclgwib3R93Hk5fxEfiRhB+6Dar64wWh71LpNSe3g= +github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= +github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= +github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= +github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= +github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= +github.com/hashicorp/go-uuid v1.0.0 h1:RS8zrF7PhGwyNPOtxSClXXj9HA8feRnJzgnI1RJCSnM= +github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-version v1.0.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= +github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= +github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU= +github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc= +github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= +github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= +github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= +github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= +github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/iancoleman/orderedmap v0.1.0 h1:2orAxZBJsvimgEBmMWfXaFlzSG2fbQil5qzP3F6cCkg= +github.com/iancoleman/orderedmap v0.1.0/go.mod h1:N0Wam8K1arqPXNWjMo21EXnBPOPp36vB07FNRdD2geA= +github.com/iancoleman/orderedmap v0.2.0 h1:sq1N/TFpYH++aViPcaKjys3bDClUEU7s5B+z6jq8pNA= +github.com/iancoleman/orderedmap v0.2.0/go.mod h1:N0Wam8K1arqPXNWjMo21EXnBPOPp36vB07FNRdD2geA= +github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/imdario/mergo v0.3.5 h1:JboBksRwiiAJWvIYJVo46AfV+IAIKZpfrSzVKj42R4Q= +github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= +github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= +github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68= +github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= +github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= +github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= +github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= +github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= +github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= +github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= +github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= +github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= +github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= +github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= +github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= +github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= +github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= +github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= +github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.11.0 h1:JAKSXpt1YjtLA7YpPiqO9ss6sNXEsPfSGdwN0UHqzrw= +github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= +github.com/onsi/ginkgo v1.14.0 h1:2mOpI4JVVPBN+WQRa0WKH2eXR+Ey+uK4n7Zj0aYpIQA= +github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= +github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= +github.com/onsi/gomega v1.7.0 h1:XPnZz8VVBHjVsy1vzJmRwIcSwiUO+JFfrv/xGiigmME= +github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= +github.com/onsi/gomega v1.10.1 h1:o0+MgICZLuZ7xjH7Vx6zS/zcu93/BEp1VwkIW1mEXCE= +github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= +github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= +github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= +github.com/peterbourgon/diskv v2.0.1+incompatible h1:UBdAOUP5p4RWqPBg048CAvpKN+vxiaj6gdUUzhl4XmI= +github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= +github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= +github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= +github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= +github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= +github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= +github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= +github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= +github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= +github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= +github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= +github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= +github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= +github.com/spf13/cobra v1.1.1/go.mod h1:WnodtKOvamDL/PwE2M4iKs8aMDBZ5Q5klgD3qfVJQMI= +github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= +github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= +github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= +github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= +github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= +github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= +github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= +github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= +go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= +go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= +go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= +go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= +go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= +go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210817164053-32db794688a5 h1:HWj/xjIHfjYU5nVXpTM0s39J9CbLn7Cc5a7IC5rwsMQ= +golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= +golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= +golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= +golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= +golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= +golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= +golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3 h1:XQyxROzUlZH+WIQwySDgnISgOivlhjIEwaQaJEJrrN0= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f h1:J5lckAjkw6qYlOZNj90mLYNTEKDvWeuc1yieZ8qUzUE= +golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= +golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5 h1:2M3HP5CCK1Si9FQhwnzYhXdG6DXeebvUHFpre8QvbyI= +golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= +golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= +golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e h1:3G+cUijn7XD+S4eJFddp53Pv7+slrESplyjG25HgL+k= +golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200707034311-ab3426394381 h1:VXak5I6aEWmAXeQjA+QSZzlgNrpq9mjcfDemuexIKsU= +golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= +golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= +golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211209124913-491a49abca63 h1:iocB37TsdFuN6IBRZ+ry36wrkoV51/tl5vOWqkcPGvY= +golang.org/x/net v0.0.0-20211209124913-491a49abca63/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 h1:SVwTIAaPC2U/AvvLNZ2a7OVsmBpC8L5BlwK1whH3hm0= +golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6 h1:pE8b58s1HRDMi8RDc79m0HISf9D4TzseP40cEA6IGfs= +golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f h1:Qmd2pbz05z7z6lm0DrgQVVPuBm92jqujBKMHMOlOQEw= +golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4 h1:YUO/7uOKsKeq9UokNS62b8FYywz3ker1l1vDZRCRefw= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200622214017-ed371f2e16b4 h1:5/PjkGUjvEU5Gl6BxmvKRPpqo2uNMv4rcHBMwzk/st8= +golang.org/x/sys v0.0.0-20200622214017-ed371f2e16b4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210831042530-f4d43177bf5e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 h1:SvFZT6jyqRaOeXpc5h/JSfZenJ2O330aBsf7JfSUXmQ= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20191024005414-555d28b269f0 h1:/5xXl8Y5W96D+TtHSlonuFqGHIWVuyCkGJLwGh9JJFs= +golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac h1:7zkz7BUtwNFFqcowJ+RIgu2MaV/MapERkDIy+mwPyjs= +golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181011042414-1f849cf54d09/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4 h1:Toz2IK7k8rbltAXwNAxKcn9OzqyNfMUhUNjz3sL0NMk= +golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= +golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200505023115-26f46d2f7ef8/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= +golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= +golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.9 h1:j9KsMiaP1c3B0OTQGth0/k+miLGTgLsAFUCrF2vLcF8= +golang.org/x/tools v0.1.9/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= +google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= +google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= +google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= +google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= +google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE= +google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= +google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU= +google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.5.0 h1:KxkO13IPW4Lslp2bz+KHP2E3gtFlrIGNThxkZQ3g+4c= +google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= +google.golang.org/appengine v1.6.5 h1:tycE03LOZYQNhDpS27tcQdAzLCVMaj7QT2SXxebnpCM= +google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= +google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= +google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= +google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201019141844-1ed22bb0c154/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= +google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= +google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= +google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= +google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= +google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= +google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.24.0 h1:UhZDfRO8JRQru4/+LlLE0BRKGF8L+PICnvYZmx/fEGA= +google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU= +gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +k8s.io/api v0.19.2 h1:q+/krnHWKsL7OBZg/rxnycsl9569Pud76UJ77MvKXms= +k8s.io/api v0.19.2/go.mod h1:IQpK0zFQ1xc5iNIQPqzgoOwuFugaYHK4iCknlAQP9nI= +k8s.io/api v0.23.3 h1:KNrME8KHGr12Ozjf8ytOewKzZh6hl/hHUZeHddT3a38= +k8s.io/api v0.23.3/go.mod h1:w258XdGyvCmnBj/vGzQMj6kzdufJZVUwEM1U2fRJwSQ= +k8s.io/apimachinery v0.19.2 h1:5Gy9vQpAGTKHPVOh5c4plE274X8D/6cuEiTO2zve7tc= +k8s.io/apimachinery v0.19.2/go.mod h1:DnPGDnARWFvYa3pMHgSxtbZb7gpzzAZ1pTfaUNDVlmA= +k8s.io/apimachinery v0.23.3 h1:7IW6jxNzrXTsP0c8yXz2E5Yx/WTzVPTsHIx/2Vm0cIk= +k8s.io/apimachinery v0.23.3/go.mod h1:BEuFMMBaIbcOqVIJqNZJXGFTP4W6AycEpb5+m/97hrM= +k8s.io/client-go v0.19.2 h1:gMJuU3xJZs86L1oQ99R4EViAADUPMHHtS9jFshasHSc= +k8s.io/client-go v0.19.2/go.mod h1:S5wPhCqyDNAlzM9CnEdgTGV4OqhsW3jGO1UM1epwfJA= +k8s.io/client-go v0.23.3 h1:23QYUmCQ/W6hW78xIwm3XqZrrKZM+LWDqW2zfo+szJs= +k8s.io/client-go v0.23.3/go.mod h1:47oMd+YvAOqZM7pcQ6neJtBiFH7alOyfunYN48VsmwE= +k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= +k8s.io/gengo v0.0.0-20210813121822-485abfe95c7c/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= +k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= +k8s.io/klog/v2 v2.2.0 h1:XRvcwJozkgZ1UQJmfMGpvRthQHOvihEhYtDfAaxMz/A= +k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= +k8s.io/klog/v2 v2.3.0 h1:WmkrnW7fdrm0/DMClc+HIxtftvxVIPAhlVwMQo5yLco= +k8s.io/klog/v2 v2.3.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= +k8s.io/klog/v2 v2.30.0/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= +k8s.io/klog/v2 v2.40.1 h1:P4RRucWk/lFOlDdkAr3mc7iWFkgKrZY9qZMAgek06S4= +k8s.io/klog/v2 v2.40.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= +k8s.io/kube-openapi v0.0.0-20200805222855-6aeccd4b50c6/go.mod h1:UuqjUnNftUyPE5H64/qeyjQoUZhGpeFDVdxjTeEVN2o= +k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65/go.mod h1:sX9MT8g7NVZM5lVL/j8QyCCJe8YSMW30QvGZWaCIDIk= +k8s.io/utils v0.0.0-20200729134348-d5654de09c73 h1:uJmqzgNWG7XyClnU/mLPBWwfKKF1K8Hf8whTseBgJcg= +k8s.io/utils v0.0.0-20200729134348-d5654de09c73/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= +k8s.io/utils v0.0.0-20210802155522-efc7438f0176/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= +k8s.io/utils v0.0.0-20211116205334-6203023598ed h1:ck1fRPWPJWsMd8ZRFsWc6mh/zHp5fZ/shhbrgPUxDAE= +k8s.io/utils v0.0.0-20211116205334-6203023598ed/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= +rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= +rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= +rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= +sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6/go.mod h1:p4QtZmO4uMYipTQNzagwnNoseA6OxSUutVw05NhYDRs= +sigs.k8s.io/structured-merge-diff/v4 v4.0.1 h1:YXTMot5Qz/X1iBRJhAt+vI+HVttY0WkSqqhKxQ0xVbA= +sigs.k8s.io/structured-merge-diff/v4 v4.0.1/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= +sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= +sigs.k8s.io/structured-merge-diff/v4 v4.2.1/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4= +sigs.k8s.io/yaml v1.1.0 h1:4A07+ZFc2wgJwo8YNlQpr1rVlgUDlxXHhPJciaPY5gs= +sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= +sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q= +sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= +sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo= +sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= diff --git a/test/e2e/hack/boilerplate/boilerplate.go.txt b/test/e2e/hack/boilerplate/boilerplate.go.txt new file mode 100644 index 00000000..59e740c1 --- /dev/null +++ b/test/e2e/hack/boilerplate/boilerplate.go.txt @@ -0,0 +1,16 @@ +/* +Copyright YEAR The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + diff --git a/test/e2e/hack/boilerplate/boilerplate.py b/test/e2e/hack/boilerplate/boilerplate.py new file mode 100755 index 00000000..b8f5fe28 --- /dev/null +++ b/test/e2e/hack/boilerplate/boilerplate.py @@ -0,0 +1,215 @@ +#!/usr/bin/env python + +# Copyright 2015 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import print_function + +import argparse +import difflib +import glob +import json +import mmap +import os +import re +import sys +from datetime import date + +parser = argparse.ArgumentParser() +parser.add_argument( + "filenames", + help="list of files to check, all files if unspecified", + nargs='*') + +rootdir = os.path.dirname(__file__) + "/../../" +rootdir = os.path.abspath(rootdir) +parser.add_argument( + "--rootdir", default=rootdir, help="root directory to examine") + +default_boilerplate_dir = os.path.join(rootdir, "hack/boilerplate") +parser.add_argument( + "--boilerplate-dir", default=default_boilerplate_dir) + +parser.add_argument( + "-v", "--verbose", + help="give verbose output regarding why a file does not pass", + action="store_true") + +args = parser.parse_args() + +verbose_out = sys.stderr if args.verbose else open("/dev/null", "w") + + +def get_refs(): + refs = {} + + for path in glob.glob(os.path.join(args.boilerplate_dir, "boilerplate.*.txt")): + extension = os.path.basename(path).split(".")[1] + + ref_file = open(path, 'r') + ref = ref_file.read().splitlines() + ref_file.close() + refs[extension] = ref + + return refs + + +def file_passes(filename, refs, regexs): + try: + f = open(filename, 'r') + except Exception as exc: + print("Unable to open %s: %s" % (filename, exc), file=verbose_out) + return False + + data = f.read() + f.close() + + basename = os.path.basename(filename) + extension = file_extension(filename) + if extension != "": + ref = refs[extension] + else: + ref = refs[basename] + + # remove build tags from the top of Go files + if extension == "go": + p = regexs["go_build_constraints"] + (data, found) = p.subn("", data, 1) + + # remove shebang from the top of shell files + if extension == "sh": + p = regexs["shebang"] + (data, found) = p.subn("", data, 1) + + data = data.splitlines() + + # if our test file is smaller than the reference it surely fails! + if len(ref) > len(data): + print('File %s smaller than reference (%d < %d)' % + (filename, len(data), len(ref)), + file=verbose_out) + return False + + # trim our file to the same number of lines as the reference file + data = data[:len(ref)] + + p = regexs["year"] + for d in data: + if p.search(d): + print('File %s is missing the year' % filename, file=verbose_out) + return False + + # Replace all occurrences of the regex "CURRENT_YEAR|...|2016|2015|2014" with "YEAR" + p = regexs["date"] + for i, d in enumerate(data): + (data[i], found) = p.subn('YEAR', d) + if found != 0: + break + + # if we don't match the reference at this point, fail + if ref != data: + print("Header in %s does not match reference, diff:" % + filename, file=verbose_out) + if args.verbose: + print(file=verbose_out) + for line in difflib.unified_diff(ref, data, 'reference', filename, lineterm=''): + print(line, file=verbose_out) + print(file=verbose_out) + return False + + return True + + +def file_extension(filename): + return os.path.splitext(filename)[1].split(".")[-1].lower() + + +skipped_dirs = [ + '.git', + "vendor", + "test/e2e/framework/framework.go", + "images" +] + + +def normalize_files(files): + newfiles = [] + for pathname in files: + if any(x in pathname for x in skipped_dirs): + continue + newfiles.append(pathname) + for i, pathname in enumerate(newfiles): + if not os.path.isabs(pathname): + newfiles[i] = os.path.join(args.rootdir, pathname) + return newfiles + + +def get_files(extensions): + files = [] + if len(args.filenames) > 0: + files = args.filenames + else: + for root, dirs, walkfiles in os.walk(args.rootdir): + # don't visit certain dirs. This is just a performance improvement + # as we would prune these later in normalize_files(). But doing it + # cuts down the amount of filesystem walking we do and cuts down + # the size of the file list + for d in skipped_dirs: + if d in dirs: + dirs.remove(d) + + for name in walkfiles: + pathname = os.path.join(root, name) + files.append(pathname) + + files = normalize_files(files) + outfiles = [] + for pathname in files: + basename = os.path.basename(pathname) + extension = file_extension(pathname) + if extension in extensions or basename in extensions: + outfiles.append(pathname) + return outfiles + + +def get_regexs(): + regexs = {} + # Search for "YEAR" which exists in the boilerplate, but shouldn't in the real thing + regexs["year"] = re.compile('YEAR') + # dates can be 2014, 2015, 2016, ..., CURRENT_YEAR, company holder names can be anything + years = range(2014, date.today().year + 1) + regexs["date"] = re.compile( + '(%s)' % "|".join(map(lambda l: str(l), years))) + # strip // +build \n\n build constraints + regexs["go_build_constraints"] = re.compile( + r"^(// \+build.*\n)+\n", re.MULTILINE) + # strip #!.* from shell scripts + regexs["shebang"] = re.compile(r"^(#!.*\n)\n*", re.MULTILINE) + return regexs + + +def main(): + regexs = get_regexs() + refs = get_refs() + filenames = get_files(refs.keys()) + + for filename in filenames: + if not file_passes(filename, refs, regexs): + print(filename, file=sys.stdout) + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/test/e2e/hack/boilerplate/boilerplate.py.txt b/test/e2e/hack/boilerplate/boilerplate.py.txt new file mode 100644 index 00000000..a2e72e59 --- /dev/null +++ b/test/e2e/hack/boilerplate/boilerplate.py.txt @@ -0,0 +1,16 @@ +#!/usr/bin/env python + +# Copyright YEAR The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/test/e2e/hack/boilerplate/boilerplate.sh.txt b/test/e2e/hack/boilerplate/boilerplate.sh.txt new file mode 100644 index 00000000..384f325a --- /dev/null +++ b/test/e2e/hack/boilerplate/boilerplate.sh.txt @@ -0,0 +1,14 @@ +# Copyright YEAR The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/test/e2e/hack/check-go-version.sh b/test/e2e/hack/check-go-version.sh new file mode 100755 index 00000000..c2690d9e --- /dev/null +++ b/test/e2e/hack/check-go-version.sh @@ -0,0 +1,45 @@ +#!/bin/bash + +# Copyright 2020 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if [ -n "$DEBUG" ]; then + set -x +fi + +set -o errexit +set -o nounset +set -o pipefail + +MINIMUM_GO_VERSION=go1.15 + +if [[ -z "$(command -v go)" ]]; then + echo " +Can't find 'go' in PATH, please fix and retry. +See http://golang.org/doc/install for installation instructions. +" + exit 1 +fi + +IFS=" " read -ra go_version <<< "$(go version)" + +if [[ "${MINIMUM_GO_VERSION}" != $(echo -e "${MINIMUM_GO_VERSION}\n${go_version[2]}" | sort -s -t. -k 1,1 -k 2,2n -k 3,3n | head -n1) && "${go_version[2]}" != "devel" ]]; then + echo " +Detected go version: ${go_version[*]}. +ingress-nginx requires ${MINIMUM_GO_VERSION} or greater. + +Please install ${MINIMUM_GO_VERSION} or later. +" + exit 1 +fi diff --git a/test/e2e/hack/codegen.go b/test/e2e/hack/codegen.go new file mode 100644 index 00000000..be047233 --- /dev/null +++ b/test/e2e/hack/codegen.go @@ -0,0 +1,824 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "bytes" + "flag" + "fmt" + "go/ast" + "go/format" + "go/parser" + "go/printer" + "go/token" + "html/template" + "io/ioutil" + "log" + "os" + "path" + "path/filepath" + "reflect" + "regexp" + "strconv" + "strings" + "unicode" + + "github.com/cucumber/gherkin-go/v11" + "github.com/cucumber/messages-go/v10" + "github.com/iancoleman/orderedmap" + "golang.org/x/tools/go/ast/astutil" + "k8s.io/apimachinery/pkg/util/sets" + + "github.com/bfenetworks/ingress-bfe/test/e2e/pkg/files" +) + +var codeGenTemplate *template.Template + +func main() { + var ( + update bool + features []string + destPath string + generatorTemplate string + testMainPath string + + basePackage string + ) + + flag.BoolVar(&update, "update", false, "update files in place in case of missing steps or method definitions") + flag.StringVar(&destPath, "dest-path", "test", "path to generated test package location") + flag.StringVar(&generatorTemplate, "code-generator-template", "hack/codegen.tmpl", "path to the go template for code generation") + flag.StringVar(&testMainPath, "test-main", "", "path to the TestMain go file") + flag.StringVar(&basePackage, "base-package", "github.com/bfenetworks/ingress-bfe", "base go package") + + flag.Parse() + + // 1. verify flags + features = flag.CommandLine.Args() + if len(features) == 0 { + fmt.Println("Usage: codegen [-update=false] [-dest-path=steps [features]") + fmt.Println() + fmt.Println("Example: codegen features/default_backend.feature") + flag.CommandLine.Usage() + os.Exit(1) + } + + // 2. parse template + var err error + codeGenTemplate, err = template.New("codegen.tmpl").Funcs(templateFuncs).ParseFiles(generatorTemplate) + + if err != nil { + log.Fatalf("Unexpected error parsing template: %v", err) + } + + // 3. if features is a directory, iterate and search for files with extension .feature + if len(features) == 1 && files.IsDir(features[0]) { + root := features[0] + features = []string{} + + err := filepath.Walk(root, visitDir(&features)) + if err != nil { + log.Fatalf("Unexpected error reading directory %v: %v", root, err) + } + } + + // 4. iterate feature files + for _, path := range features { + err := processFeature(path, destPath, update, basePackage, testMainPath) + if err != nil { + log.Fatal(err) + } + } + + // 16. last step verifies the TestMain file + // uses all the defined features + if testMainPath == "" { + return + } + + featuresInTestMain, err := extractFeaturesMapKeys(testMainPath) + if err != nil { + log.Fatal(err) + } + + featuresInTestMainSet := sets.NewString(featuresInTestMain...) + featuresSet := sets.NewString(features...) + + if !featuresInTestMainSet.Equal(featuresSet) { + log.Printf(`Generated features mapping from .features files differ from the expected in TestMain file %v +expected %v +generated %v + +`, + testMainPath, features, featuresInTestMain) + } +} + +func processFeature(path, destPath string, update bool, basePackage, testMainPath string) error { + // 5. parse feature file + featureSteps, err := parseFeature(path) + if err != nil { + return fmt.Errorf("parsing feature file: %w", err) + } + + // 6. generate package name to use + packageName := generatePackage(path) + + // 7. check if go source file exists + goFile := filepath.Join(destPath, packageName, "steps.go") + isGoFileOk := files.Exists(goFile) + + mapping := &Mapping{ + Package: packageName, + FeatureFile: path, + Features: featureSteps, + NewFunctions: featureSteps, + GoFile: goFile, + } + + // 8. Extract functions from go source code + if isGoFileOk { + goFunctions, err := extractFuncs(goFile) + if err != nil { + return fmt.Errorf("extracting go functions: %w", err) + } + + mapping.GoDefinitions = goFunctions + } + + if isGoFileOk { + inFeatures := sets.NewString() + inGo := sets.NewString() + + for _, feature := range mapping.Features { + inFeatures.Insert(feature.Name) + } + + for _, gofunc := range mapping.GoDefinitions { + inGo.Insert(gofunc.Name) + } + + mapping.NewFunctions = []Function{} + + if newFunctions := inFeatures.Difference(inGo); newFunctions.Len() > 0 { + log.Printf("Feature file %v contains %v new function/s", mapping.FeatureFile, newFunctions.Len()) + + var funcs []Function + for _, f := range newFunctions.List() { + for _, feature := range mapping.Features { + if feature.Name == f { + funcs = append(funcs, feature) + break + } + } + } + + mapping.NewFunctions = funcs + } + + // 9. check signatures are ok + signatureChanges := extractSignatureChanges(mapping) + if len(signatureChanges) != 0 { + var argBuf bytes.Buffer + + for _, sc := range signatureChanges { + argBuf.WriteString(fmt.Sprintf(` +function %v + have %v + want %v +`, sc.Function, sc.Have, sc.Want)) + } + + return fmt.Errorf("source file %v has a different signature/s:\n %v", mapping.GoFile, argBuf.String()) + } + } + + // 10. New go feature file + if !isGoFileOk { + log.Printf("Generating new go file %v...", mapping.GoFile) + // 11. Feature to go source code + err = generateGoFile(mapping) + if err != nil { + return err + } + + featurePackage := filepath.Join(basePackage, destPath, packageName) + + // 12. update map variable in e2e_test + if testMainPath != "" { + err = updateFeatureMapVariable(mapping.FeatureFile, mapping.Package, featurePackage, testMainPath) + if err != nil { + return err + } + } + + return nil + } + + if !update { + if len(mapping.NewFunctions) != 0 { + return fmt.Errorf("generated code %s exist but out of date, set argument -update=true if you need update file", mapping.GoFile) + } + + return nil + } + + // 13. if update is set + log.Printf("Updating go file %v...", mapping.GoFile) + return updateGoTestFile(mapping.GoFile, mapping.NewFunctions) +} + +// Function holds the definition of a function in a go file or godog step +type Function struct { + // Name + Name string + // Expr Regexp to use in godog Step definition + Expr string + // Args function arguments + // k = name of the argument + // v = type of the argument + Args *orderedmap.OrderedMap +} + +type Mapping struct { + Package string + + FeatureFile string + Features []Function + + GoFile string + GoDefinitions []Function + + NewFunctions []Function +} + +// SignatureChange holds information about the definition of a go function +type SignatureChange struct { + Function string + Have string + Want string +} + +var templateFuncs = template.FuncMap{ + "backticked": func(s string) string { + return "`" + s + "`" + }, + "unescape": func(s string) template.HTML { + return template.HTML(s) + }, + "argsFromMap": argsFromMap, +} + +// parseFeature parses a godog feature file returning the unique +// steps definitions +func parseFeature(path string) ([]Function, error) { + data, err := files.Read(path) + if err != nil { + return nil, err + } + + gd, err := gherkin.ParseGherkinDocument(bytes.NewReader(data), (&messages.Incrementing{}).NewId) + if err != nil { + return nil, err + } + + scenarios := gherkin.Pickles(*gd, path, (&messages.Incrementing{}).NewId) + + funcs := []Function{} + for _, s := range scenarios { + funcs = parseSteps(s.Steps, funcs) + } + + return funcs, nil +} + +// extractFuncs reads a file containing go source code and returns +// the functions defined in the file. +func extractFuncs(filePath string) ([]Function, error) { + if !strings.HasSuffix(filePath, ".go") { + return nil, fmt.Errorf("only files with go extension are valid") + } + + fset := token.NewFileSet() + + node, err := parser.ParseFile(fset, filePath, nil, parser.ParseComments) + if err != nil { + return nil, err + } + + var funcs []Function + + var printErr error + ast.Inspect(node, func(n ast.Node) bool { + fn, ok := n.(*ast.FuncDecl) + if !ok { + return true + } + + index := 0 + args := orderedmap.New() + for _, p := range fn.Type.Params.List { + var typeNameBuf bytes.Buffer + + err := printer.Fprint(&typeNameBuf, fset, p.Type) + if err != nil { + printErr = err + return false + } + + if len(p.Names) == 0 { + argName := fmt.Sprintf("arg%d", index+1) + args.Set(argName, typeNameBuf.String()) + + index++ + continue + } + + for _, ag := range p.Names { + argName := ag.String() + args.Set(argName, typeNameBuf.String()) + index++ + } + } + + // Go functions do not have an expression + funcs = append(funcs, Function{Name: fn.Name.Name, Args: args}) + + return true + }) + + if printErr != nil { + return nil, printErr + } + + return funcs, nil +} + +func updateGoTestFile(filePath string, newFuncs []Function) error { + fileSet := token.NewFileSet() + + node, err := parser.ParseFile(fileSet, filePath, nil, parser.ParseComments) + if err != nil { + return err + } + + var featureFunc *ast.FuncDecl + ast.Inspect(node, func(n ast.Node) bool { + fn, ok := n.(*ast.FuncDecl) + if !ok { + return true + } + + if fn.Name.Name == "InitializeScenario" { + featureFunc = fn + } + + return true + }) + + if featureFunc == nil { + return fmt.Errorf("file %v does not contains a FeatureFunct function", filePath) + } + + // Add new functions + astf, err := toAstFunctions(newFuncs) + if err != nil { + return err + } + + node.Decls = append(node.Decls, astf...) + + // Update steps in InitializeScenario + astSteps, err := toContextStepsfuncs(newFuncs) + if err != nil { + return err + } + + featureFunc.Body.List = append(astSteps, featureFunc.Body.List...) + + var buffer bytes.Buffer + if err = format.Node(&buffer, fileSet, node); err != nil { + return fmt.Errorf("error formatting file %v: %w", filePath, err) + } + + fileInfo, err := os.Stat(filePath) + if err != nil { + return fmt.Errorf("error reading file %v: %w", filePath, err) + } + + return ioutil.WriteFile(filePath, buffer.Bytes(), fileInfo.Mode()) +} + +func toContextStepsfuncs(funcs []Function) ([]ast.Stmt, error) { + astStepsTpl := ` +package codegen +func InitializeScenario() { {{ range . }} + ctx.Step({{ backticked .Expr | unescape }}, {{ .Name }}){{end}} +} +` + astFile, err := astFromTemplate(astStepsTpl, funcs) + if err != nil { + return nil, err + } + + f := astFile.Decls[0].(*ast.FuncDecl) + + return f.Body.List, nil +} + +func toAstFunctions(funcs []Function) ([]ast.Decl, error) { + astFuncTpl := ` +package codegen +{{ range . }}func {{ .Name }}{{ argsFromMap .Args false }} error { + return godog.ErrPending +} + +{{end}} +` + astFile, err := astFromTemplate(astFuncTpl, funcs) + if err != nil { + return nil, err + } + + return astFile.Decls, nil +} + +func astFromTemplate(astFuncTpl string, funcs []Function) (*ast.File, error) { + buf := bytes.NewBuffer(make([]byte, 0)) + + astFuncs, err := template.New("ast").Funcs(templateFuncs).Parse(astFuncTpl) + if err != nil { + return nil, err + } + + err = astFuncs.Execute(buf, funcs) + if err != nil { + return nil, err + } + + fset := token.NewFileSet() + + astFile, err := parser.ParseFile(fset, "src.go", buf.String(), parser.ParseComments) + if err != nil { + return nil, err + } + + return astFile, nil +} + +// generatePackage returns the name of the +// package to use using the feature filename +func generatePackage(filePath string) string { + base := path.Base(filePath) + base = strings.ToLower(base) + base = strings.ReplaceAll(base, "_", "") + base = strings.ReplaceAll(base, ".feature", "") + + return base +} + +func argsFromMap(args *orderedmap.OrderedMap, onlyType bool) string { + s := "(" + + for _, k := range args.Keys() { + v, ok := args.Get(k) + if !ok { + continue + } + + if onlyType { + s += fmt.Sprintf("%v, ", v) + } else { + s += fmt.Sprintf("%v %v, ", k, v) + } + } + + if len(args.Keys()) > 0 { + s = s[0 : len(s)-2] + } + + return s + ")" +} + +func visitDir(files *[]string) filepath.WalkFunc { + return func(path string, info os.FileInfo, err error) error { + if err != nil { + log.Fatal(err) + } + + if filepath.Ext(path) != ".feature" { + return nil + } + + *files = append(*files, path) + return nil + } +} + +const mapVariableName = "features" + +// extractFeaturesMapKeys extracts the keys from the features map defined in +// the main test file defined in a variable: +// features = map[string]func(*godog.Suite){} +func extractFeaturesMapKeys(testPath string) ([]string, error) { + fset := token.NewFileSet() + + fileAst, err := parser.ParseFile(fset, testPath, nil, parser.ParseComments) + if err != nil { + return nil, err + } + + for _, declarations := range fileAst.Decls { + switch decl := declarations.(type) { + case *ast.GenDecl: + for _, spec := range decl.Specs { + spec, ok := spec.(*ast.ValueSpec) + if !ok { + continue + } + + for _, fn := range spec.Names { + if fn.Name != mapVariableName { + continue + } + + features := fn.Obj.Decl.(*ast.ValueSpec).Values + elts := features[0].(*ast.CompositeLit).Elts + + featureNames := []string{} + for _, elt := range elts { + val := elt.(*ast.KeyValueExpr).Key.(*ast.BasicLit).Value + s, err := strconv.Unquote(val) + if err != nil { + featureNames = append(featureNames, val) + continue + } + + featureNames = append(featureNames, s) + } + + return featureNames, nil + } + } + } + } + + return nil, fmt.Errorf("there is no features variable in file %v", testPath) +} + +func updateFeatureMapVariable(featureName, packageName, featurePackage, testPath string) error { + fset := token.NewFileSet() + + fileAst, err := parser.ParseFile(fset, testPath, nil, parser.ParseComments) + if err != nil { + return err + } + + if !astutil.UsesImport(fileAst, featurePackage) { + astutil.AddImport(fset, fileAst, featurePackage) + } + + pre := func(c *astutil.Cursor) bool { + if sel, ok := c.Node().(*ast.GenDecl); ok { + for _, spec := range sel.Specs { + spec, ok := spec.(*ast.ValueSpec) + if !ok { + continue + } + + for _, fn := range spec.Names { + if fn.Name != mapVariableName { + continue + } + + features := fn.Obj.Decl.(*ast.ValueSpec).Values[0] + features.(*ast.CompositeLit).Elts = append(features.(*ast.CompositeLit).Elts, + &ast.KeyValueExpr{ + Key: &ast.BasicLit{ + Kind: token.STRING, + Value: fmt.Sprintf("\n\"%v\"", featureName), + }, + Value: &ast.SelectorExpr{ + X: &ast.Ident{ + Name: packageName, + }, + Sel: &ast.Ident{ + Name: "InitializeScenario,\n", + }, + }, + }, + ) + + break + } + } + } + + return true + } + + astutil.Apply(fileAst, pre, nil) + + var buf bytes.Buffer + if err := format.Node(&buf, fset, fileAst); err != nil { + return err + } + + return ioutil.WriteFile(testPath, buf.Bytes(), 0644) +} + +func generateGoFile(mapping *Mapping) error { + buf := bytes.NewBuffer(make([]byte, 0)) + + err := codeGenTemplate.Execute(buf, mapping) + if err != nil { + return err + } + + // 13. if update is set + isDirOk := files.IsDir(mapping.GoFile) + if !isDirOk { + err := os.MkdirAll(filepath.Dir(mapping.GoFile), 0755) + if err != nil { + return err + } + } + + return ioutil.WriteFile(mapping.GoFile, buf.Bytes(), 0644) +} + +func extractSignatureChanges(mapping *Mapping) []SignatureChange { + var signatureChanges []SignatureChange + for _, feature := range mapping.Features { + for _, gofunc := range mapping.GoDefinitions { + if feature.Name != gofunc.Name { + continue + } + + // We need to compare function arguments checking only + // the number and type. Is not possible to rely in the name + // in the go code. + featKeys := feature.Args.Keys() + goKeys := gofunc.Args.Keys() + + if len(featKeys) != len(goKeys) { + signatureChanges = append(signatureChanges, SignatureChange{ + Function: gofunc.Name, + Have: argsFromMap(gofunc.Args, true), + Want: argsFromMap(feature.Args, true), + }) + + continue + } + + for index, k := range featKeys { + fv, _ := feature.Args.Get(k) + gv, _ := gofunc.Args.Get(goKeys[index]) + + if !reflect.DeepEqual(fv, gv) { + signatureChanges = append(signatureChanges, SignatureChange{ + Function: gofunc.Name, + Have: argsFromMap(gofunc.Args, true), + Want: argsFromMap(feature.Args, true), + }) + } + } + } + } + + return signatureChanges +} + +// Code below this comment comes from github.com/cucumber/godog +// (code defined in private methods) + +const ( + numberGroup = "(\\d+)" + stringGroup = "\"([^\"]*)\"" +) + +// parseStepArgs extracts arguments from an expression defined in a step RegExp. +// This code was extracted from +// https://github.com/cucumber/godog/blob/4da503aab2d0b71d380fbe8c48a6af9f729b6f5a/undefined_snippets_gen.go#L41 +func parseStepArgs(exp string, argument *messages.PickleStepArgument) *orderedmap.OrderedMap { + var ( + args []string + pos int + breakLoop bool + ) + + for !breakLoop { + part := exp[pos:] + ipos := strings.Index(part, numberGroup) + spos := strings.Index(part, stringGroup) + + switch { + case spos == -1 && ipos == -1: + breakLoop = true + case spos == -1: + pos += ipos + len(numberGroup) + args = append(args, "int") + case ipos == -1: + pos += spos + len(stringGroup) + args = append(args, "string") + case ipos < spos: + pos += ipos + len(numberGroup) + args = append(args, "int") + case spos < ipos: + pos += spos + len(stringGroup) + args = append(args, "string") + } + } + + if argument != nil { + if argument.GetDocString() != nil { + args = append(args, "*godog.DocString") + } + + if argument.GetDataTable() != nil { + args = append(args, "*godog.DocString") + } + } + + stepArgs := orderedmap.New() + + for i, v := range args { + k := fmt.Sprintf("arg%d", i+1) + stepArgs.Set(k, v) + } + + return stepArgs +} + +// some snippet formatting regexps +var snippetExprCleanup = regexp.MustCompile("([\\/\\[\\]\\(\\)\\\\^\\$\\.\\|\\?\\*\\+\\'])") +var snippetExprQuoted = regexp.MustCompile("(\\W|^)\"(?:[^\"]*)\"(\\W|$)") +var snippetMethodName = regexp.MustCompile("[^a-zA-Z\\_\\ ]") +var snippetNumbers = regexp.MustCompile("(\\d+)") + +// parseSteps converts a string step definition in a different one valid as a regular +// expression that can be used in a go Step definition. This original code is located in +// https://github.com/cucumber/godog/blob/4da503aab2d0b71d380fbe8c48a6af9f729b6f5a/fmt.go#L457 +func parseSteps(steps []*messages.Pickle_PickleStep, funcDefs []Function) []Function { + var index int + + for _, step := range steps { + text := step.Text + + expr := snippetExprCleanup.ReplaceAllString(text, "\\$1") + expr = snippetNumbers.ReplaceAllString(expr, "(\\d+)") + expr = snippetExprQuoted.ReplaceAllString(expr, "$1\"([^\"]*)\"$2") + expr = "^" + strings.TrimSpace(expr) + "$" + + name := snippetNumbers.ReplaceAllString(text, " ") + name = snippetExprQuoted.ReplaceAllString(name, " ") + name = strings.TrimSpace(snippetMethodName.ReplaceAllString(name, "")) + + var words []string + for i, w := range strings.Split(name, " ") { + switch { + case i != 0: + w = strings.Title(w) + case len(w) > 0: + w = string(unicode.ToLower(rune(w[0]))) + w[1:] + } + + words = append(words, w) + } + + name = strings.Join(words, "") + if len(name) == 0 { + index++ + name = fmt.Sprintf("StepDefinitioninition%d", index) + } + + var found bool + for _, f := range funcDefs { + if f.Expr == expr { + found = true + break + } + } + + if !found { + args := parseStepArgs(expr, step.Argument) + funcDefs = append(funcDefs, Function{Name: name, Expr: expr, Args: args}) + } + } + + return funcDefs +} diff --git a/test/e2e/hack/codegen.tmpl b/test/e2e/hack/codegen.tmpl new file mode 100644 index 00000000..0935230c --- /dev/null +++ b/test/e2e/hack/codegen.tmpl @@ -0,0 +1,51 @@ +/* +Copyright 2021 The BFE Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package {{ .Package }} + +import ( + "github.com/cucumber/godog" + "github.com/cucumber/messages-go/v16" + + "github.com/bfenetworks/ingress-bfe/test/e2e/pkg/kubernetes" + tstate "github.com/bfenetworks/ingress-bfe/test/e2e/pkg/state" +) + +var ( + state *tstate.Scenario +) + +// IMPORTANT: Steps definitions are generated and should not be modified +// by hand but rather through make codegen. DO NOT EDIT. + +// InitializeScenario configures the Feature to test +func InitializeScenario(ctx *godog.ScenarioContext) { {{- range .NewFunctions }} + ctx.Step({{ backticked .Expr | unescape }}, {{ .Name }}){{end}} + + ctx.BeforeScenario(func(*godog.Scenario) { + state = tstate.New() + }) + + ctx.AfterScenario(func(*messages.Pickle, error) { + // delete namespace an all the content + _ = kubernetes.DeleteNamespace(kubernetes.KubeClient, state.Namespace) + }) +} +{{ range .NewFunctions }} +func {{ .Name }}{{ argsFromMap .Args false }} error { + return godog.ErrPending +} +{{ end }} diff --git a/test/e2e/hack/kube-env.sh b/test/e2e/hack/kube-env.sh new file mode 100644 index 00000000..4415df15 --- /dev/null +++ b/test/e2e/hack/kube-env.sh @@ -0,0 +1,44 @@ +#!/bin/bash + +# Copyright 2014 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Some useful colors. +if [[ -z "${color_start-}" ]]; then + declare -r color_start="\033[" + declare -r color_red="${color_start}0;31m" + declare -r color_yellow="${color_start}0;33m" + declare -r color_green="${color_start}0;32m" + declare -r color_norm="${color_start}0m" +fi + +# Returns the server version as MMmmpp, with MM as the major +# component, mm the minor component, and pp as the patch +# revision. e.g. 0.7.1 is echoed as 701, and 1.0.11 would be +# 10011. (This makes for easy integer comparison in bash.) +function kube_server_version() { + local server_version + local major + local minor + local patch + + # This sed expression is the POSIX BRE to match strings like: + # Server Version: &version.Info{Major:"0", Minor:"7+", GitVersion:"v0.7.0-dirty", GitCommit:"ad44234f7152e9c66bc2853575445c7071335e57", GitTreeState:"dirty"} + # and capture the GitVersion portion (which has the patch level) + server_version=$(${KUBECTL} --match-server-version=false version | grep "Server Version:") + read major minor patch < <( + echo ${server_version} | \ + sed "s/.*GitVersion:\"v\([0-9]\{1,\}\)\.\([0-9]\{1,\}\)\.\([0-9]\{1,\}\).*/\1 \2 \3/") + printf "%02d%02d%02d" ${major} ${minor} ${patch} | sed 's/^0*//' +} diff --git a/test/e2e/hack/verify-all.sh b/test/e2e/hack/verify-all.sh new file mode 100755 index 00000000..764d7c3c --- /dev/null +++ b/test/e2e/hack/verify-all.sh @@ -0,0 +1,78 @@ +#!/bin/bash + +# Copyright 2014 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +KUBE_ROOT=$(dirname "${BASH_SOURCE}")/.. +source "${KUBE_ROOT}/hack/kube-env.sh" + +SILENT=true + +function is-excluded { + for e in $EXCLUDE; do + if [[ $1 -ef ${BASH_SOURCE} ]]; then + return + fi + if [[ $1 -ef "$KUBE_ROOT/hack/$e" ]]; then + return + fi + done + return 1 +} + +while getopts ":v" opt; do + case $opt in + v) + SILENT=false + ;; + \?) + echo "Invalid flag: -$OPTARG" >&2 + exit 1 + ;; + esac +done + +if $SILENT ; then + echo "Running in the silent mode, run with -v if you want to see script logs." +fi + +EXCLUDE="verify-all.sh verify-codegen.sh" + +ret=0 +for t in `ls $KUBE_ROOT/hack/verify-*.sh` +do + if is-excluded $t ; then + echo "Skipping $t" + continue + fi + if $SILENT ; then + echo -e "Verifying $t" + if bash "$t" &> /dev/null; then + echo -e "${color_green}SUCCESS${color_norm}" + else + echo -e "${color_red}FAILED${color_norm}" + ret=1 + fi + else + bash "$t" || ret=1 + fi +done + +exit $ret + +# ex: ts=2 sw=2 et filetype=sh diff --git a/test/e2e/hack/verify-boilerplate.sh b/test/e2e/hack/verify-boilerplate.sh new file mode 100755 index 00000000..2f8705b6 --- /dev/null +++ b/test/e2e/hack/verify-boilerplate.sh @@ -0,0 +1,35 @@ +#!/bin/bash + +# Copyright 2014 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +KUBE_ROOT=$(dirname "${BASH_SOURCE}")/.. + +boilerDir="${KUBE_ROOT}/hack/boilerplate" +boiler="${boilerDir}/boilerplate.py" + +files_need_boilerplate=($(${boiler} "$@")) + +# Run boilerplate check +if [[ ${#files_need_boilerplate[@]} -gt 0 ]]; then + for file in "${files_need_boilerplate[@]}"; do + echo "Boilerplate header is wrong for: ${file}" + done + + exit 1 +fi diff --git a/test/e2e/hack/verify-gherkin.sh b/test/e2e/hack/verify-gherkin.sh new file mode 100755 index 00000000..40a1c9dc --- /dev/null +++ b/test/e2e/hack/verify-gherkin.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +# Copyright 2014 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# GoFmt apparently is changing @ head... + +set -o errexit +set -o nounset +set -o pipefail + +KUBE_ROOT=$(dirname "${BASH_SOURCE}")/.. + +if ! command -v reformat-gherkin --version &> /dev/null; then + echo "Please install reformat-gherkin running \"pip install reformat-gherkin\"" + exit 1 +fi + +reformat-gherkin --check ${KUBE_ROOT}/features diff --git a/test/e2e/hack/verify-gofmt.sh b/test/e2e/hack/verify-gofmt.sh new file mode 100755 index 00000000..db31c046 --- /dev/null +++ b/test/e2e/hack/verify-gofmt.sh @@ -0,0 +1,43 @@ +#!/bin/bash + +# Copyright 2014 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# GoFmt apparently is changing @ head... + +set -o errexit +set -o nounset +set -o pipefail + +KUBE_ROOT=$(dirname "${BASH_SOURCE}")/.. + +cd "${KUBE_ROOT}" + +find_files() { + find . -not \( \ + \( \ + -wholename './.git' \ + -o -wholename '*/vendor/*' \ + -o -wholename '*bindata.go' \ + \) -prune \ + \) -name '*.go' +} + +GOFMT="gofmt -s" +bad_files=$(find_files | xargs $GOFMT -l) +if [[ -n "${bad_files}" ]]; then + echo "!!! '$GOFMT' needs to be run on the following files: " + echo "${bad_files}" + exit 1 +fi diff --git a/test/e2e/hack/verify-golint.sh b/test/e2e/hack/verify-golint.sh new file mode 100755 index 00000000..0d433df2 --- /dev/null +++ b/test/e2e/hack/verify-golint.sh @@ -0,0 +1,43 @@ +#!/bin/bash + +# Copyright 2014 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +KUBE_ROOT=$(dirname "${BASH_SOURCE}")/.. + +if ! command -v golint &> /dev/null; then + go get golang.org/x/lint/golint +fi + +cd "${KUBE_ROOT}" + +PACKAGES=($(go list ./test/...)) +bad_files=() +for package in "${PACKAGES[@]}"; do + out=$(golint -min_confidence=0.9 "${package}" | grep -v -E '(should not use dot imports)' || :) + if [[ -n "${out}" ]]; then + bad_files+=("${out}") + fi +done +if [[ "${#bad_files[@]}" -ne 0 ]]; then + echo "!!! golint problems: " + echo "${bad_files[@]}" + exit 1 +fi + +# ex: ts=2 sw=2 et filetype=sh diff --git a/test/e2e/images/echoserver/Dockerfile b/test/e2e/images/echoserver/Dockerfile new file mode 100644 index 00000000..b14d3638 --- /dev/null +++ b/test/e2e/images/echoserver/Dockerfile @@ -0,0 +1,32 @@ +# Copyright 2019 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Build +FROM golang:1.16-alpine3.14 as builder + +ENV CGO_ENABLED=0 + +WORKDIR /echoserver/ + +COPY echoserver.go . + +RUN GO111MODULE=off go build -trimpath -ldflags="-buildid= -s -w" -o echoserver . + +# Use distroless as minimal base image to package the binary +# Refer to https://github.com/GoogleContainerTools/distroless for more details +FROM alpine:3.14 +WORKDIR / +COPY --from=builder /echoserver / + +ENTRYPOINT ["/echoserver"] diff --git a/test/e2e/images/echoserver/Makefile b/test/e2e/images/echoserver/Makefile new file mode 100644 index 00000000..ded39063 --- /dev/null +++ b/test/e2e/images/echoserver/Makefile @@ -0,0 +1,26 @@ +# Copyright 2020 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +TAG ?= 0.0.1 + +REGISTRY ?= local +IMAGE = echoserver + +.PHONY: build-image +build-image: ## Build the ingress conformance image + docker build -t $(REGISTRY)/$(IMAGE):$(TAG) . + +.PHONY: publish-image +publish-image: + docker push $(REGISTRY)/$(IMAGE):$(TAG) diff --git a/test/e2e/images/echoserver/echoserver b/test/e2e/images/echoserver/echoserver new file mode 100755 index 00000000..f0f6dc27 Binary files /dev/null and b/test/e2e/images/echoserver/echoserver differ diff --git a/test/e2e/images/echoserver/echoserver.go b/test/e2e/images/echoserver/echoserver.go new file mode 100644 index 00000000..1ceac393 --- /dev/null +++ b/test/e2e/images/echoserver/echoserver.go @@ -0,0 +1,232 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "crypto/tls" + "crypto/x509" + "encoding/json" + "encoding/pem" + "fmt" + "io/ioutil" + "net/http" + "os" + "strings" +) + +// RequestAssertions contains information about the request and the Ingress +type RequestAssertions struct { + Path string `json:"path"` + Host string `json:"host"` + Method string `json:"method"` + Proto string `json:"proto"` + Headers map[string][]string `json:"headers"` + + Context `json:",inline"` + + TLS *TLSAssertions `json:"tls,omitempty"` +} + +// TLSAssertions contains information about the TLS connection. +type TLSAssertions struct { + Version string `json:"version"` + PeerCertificates []string `json:"peerCertificates,omitempty"` + ServerName string `json:"serverName"` + NegotiatedProtocol string `json:"negotiatedProtocol,omitempty"` + CipherSuite string `json:"cipherSuite"` +} + +type preserveSlashes struct { + mux http.Handler +} + +func (s *preserveSlashes) ServeHTTP(w http.ResponseWriter, r *http.Request) { + r.URL.Path = strings.Replace(r.URL.Path, "//", "/", -1) + s.mux.ServeHTTP(w, r) +} + +// Context contains information about the context where the echoserver is running +type Context struct { + Namespace string `json:"namespace"` + Ingress string `json:"ingress"` + Service string `json:"service"` + Pod string `json:"pod"` +} + +var context Context + +func main() { + httpPort := os.Getenv("HTTP_PORT") + if httpPort == "" { + httpPort = "3000" + } + + httpsPort := os.Getenv("HTTPS_PORT") + if httpsPort == "" { + httpsPort = "8443" + } + + context = Context{ + Namespace: os.Getenv("NAMESPACE"), + Ingress: os.Getenv("INGRESS_NAME"), + Service: os.Getenv("SERVICE_NAME"), + Pod: os.Getenv("POD_NAME"), + } + + httpMux := http.NewServeMux() + httpMux.HandleFunc("/health", healthHandler) + httpMux.HandleFunc("/", echoHandler) + httpHandler := &preserveSlashes{httpMux} + + errchan := make(chan error) + + go func() { + fmt.Printf("Starting server, listening on port %s (http)\n", httpPort) + err := http.ListenAndServe(fmt.Sprintf(":%s", httpPort), httpHandler) + if err != nil { + errchan <- err + } + }() + + // Enable HTTPS if certificate and private key are given. + if os.Getenv("TLS_SERVER_CERT") != "" && os.Getenv("TLS_SERVER_PRIVKEY") != "" { + go func() { + fmt.Printf("Starting server, listening on port %s (https)\n", httpsPort) + err := listenAndServeTLS(fmt.Sprintf(":%s", httpsPort), os.Getenv("TLS_SERVER_CERT"), os.Getenv("TLS_SERVER_PRIVKEY"), os.Getenv("TLS_CLIENT_CACERTS"), httpHandler) + if err != nil { + errchan <- err + } + }() + } + + select { + case err := <-errchan: + panic(fmt.Sprintf("Failed to start listening: %s\n", err.Error())) + } +} + +func healthHandler(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(200) + w.Write([]byte(`OK`)) +} + +func echoHandler(w http.ResponseWriter, r *http.Request) { + fmt.Printf("Echoing back request made to %s to client (%s)\n", r.RequestURI, r.RemoteAddr) + requestAssertions := RequestAssertions{ + r.RequestURI, + r.Host, + r.Method, + r.Proto, + r.Header, + + context, + + tlsStateToAssertions(r.TLS), + } + + js, err := json.MarshalIndent(requestAssertions, "", " ") + if err != nil { + processError(w, err, http.StatusInternalServerError) + return + } + + w.Header().Set("Content-Type", "application/json") + w.Header().Set("X-Content-Type-Options", "nosniff") + w.Write(js) +} + +func processError(w http.ResponseWriter, err error, code int) { + w.Header().Set("Content-Type", "application/json") + w.Header().Set("X-Content-Type-Options", "nosniff") + body, err := json.Marshal(struct { + Message string `json:"message"` + }{ + err.Error(), + }) + if err != nil { + w.WriteHeader(code) + fmt.Fprintln(w, err) + return + } + + w.WriteHeader(code) + w.Write(body) +} + +func listenAndServeTLS(addr string, serverCert string, serverPrivKey string, clientCA string, handler http.Handler) error { + var config tls.Config + + // Optionally enable client certificate validation when client CA certificates are given. + if clientCA != "" { + ca, err := ioutil.ReadFile(clientCA) + if err != nil { + return err + } + + certPool := x509.NewCertPool() + if ok := certPool.AppendCertsFromPEM(ca); !ok { + return fmt.Errorf("unable to append certificate in %q to CA pool", clientCA) + } + + // Verify certificate against given CA but also allow unauthenticated connections. + config.ClientAuth = tls.VerifyClientCertIfGiven + config.ClientCAs = certPool + } + + srv := &http.Server{ + Addr: addr, + Handler: handler, + TLSConfig: &config, + } + + return srv.ListenAndServeTLS(serverCert, serverPrivKey) +} + +func tlsStateToAssertions(connectionState *tls.ConnectionState) *TLSAssertions { + if connectionState != nil { + var state TLSAssertions + + switch connectionState.Version { + case tls.VersionTLS13: + state.Version = "TLSv1.3" + case tls.VersionTLS12: + state.Version = "TLSv1.2" + case tls.VersionTLS11: + state.Version = "TLSv1.1" + case tls.VersionTLS10: + state.Version = "TLSv1.0" + } + + state.NegotiatedProtocol = connectionState.NegotiatedProtocol + state.ServerName = connectionState.ServerName + state.CipherSuite = tls.CipherSuiteName(connectionState.CipherSuite) + + // Convert peer certificates to PEM blocks. + for _, c := range connectionState.PeerCertificates { + var out strings.Builder + pem.Encode(&out, &pem.Block{ + Type: "CERTIFICATE", + Bytes: c.Raw, + }) + state.PeerCertificates = append(state.PeerCertificates, out.String()) + } + + return &state + } + + return nil +} diff --git a/test/e2e/images/reports/Dockerfile b/test/e2e/images/reports/Dockerfile new file mode 100644 index 00000000..b338675a --- /dev/null +++ b/test/e2e/images/reports/Dockerfile @@ -0,0 +1,24 @@ +# Copyright 2020 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# build stage +FROM node:lts-alpine + +WORKDIR /app + +COPY src /app + +RUN npm install --production + +CMD ["node", "/app/index.js"] diff --git a/test/e2e/images/reports/Makefile b/test/e2e/images/reports/Makefile new file mode 100644 index 00000000..fa01b958 --- /dev/null +++ b/test/e2e/images/reports/Makefile @@ -0,0 +1,26 @@ +# Copyright 2020 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +TAG ?= 0.0.1 + +REGISTRY ?= local +IMAGE = reports-builder + +.PHONY: build-image +build-image: ## Build the e2e test report image + docker build -t $(REGISTRY)/$(IMAGE):$(TAG) . + +.PHONY: publish-image +publish-image: + docker push $(REGISTRY)/$(IMAGE):$(TAG) diff --git a/test/e2e/images/reports/README.md b/test/e2e/images/reports/README.md new file mode 100644 index 00000000..9e2c25c9 --- /dev/null +++ b/test/e2e/images/reports/README.md @@ -0,0 +1,50 @@ +# Report builder for conformance tests + +### Environment variables: + +#### Mandatory + +| Variable | Description | +| ------------- | ------------- | +| INPUT_DIRECTORY | Directory that contains the cucumber json files | +| OUTPUT_DIRECTORY | Directory where the reports will be generated | + +#### Optional + +| Variable | Description | +| ------------- | ------------- | +| INGRESS_CONTROLLER | Information about the ingress controller | +| CONTROLLER_VERSION | ingress controller version | + +## Building + +```console +make +``` + +### Generation of reports + +```console +docker run \ + -e BUILD=$(git rev-parse --short HEAD) \ + -e INPUT_DIRECTORY=/input \ + -e OUTPUT_DIRECTORY=/output \ + -v $PWD:/input:ro \ + -v $PWD/output:/output \ + local/reports-builder:0.0 +``` + +### Display + +The reports are plain HTML files. The file located in `OUTPUT_DIRECTORY/index.html` renders the initial page. + +Using any web server capable of render html is enough. +Like: + +```console +cd $OUTPUT_DIRECTORY + +python -m http.server 8000 +Serving HTTP on 0.0.0.0 port 8000 (http://0.0.0.0:8000/) ... + +``` diff --git a/test/e2e/images/reports/src/.gitignore b/test/e2e/images/reports/src/.gitignore new file mode 100644 index 00000000..53752db2 --- /dev/null +++ b/test/e2e/images/reports/src/.gitignore @@ -0,0 +1 @@ +output diff --git a/test/e2e/images/reports/src/cucumber-html-reporter/.eslintrc.yml b/test/e2e/images/reports/src/cucumber-html-reporter/.eslintrc.yml new file mode 100644 index 00000000..c5807881 --- /dev/null +++ b/test/e2e/images/reports/src/cucumber-html-reporter/.eslintrc.yml @@ -0,0 +1,9 @@ +env: + browser: true + commonjs: true + es2020: true +extends: + - airbnb-base +parserOptions: + ecmaVersion: 11 +rules: {} diff --git a/test/e2e/images/reports/src/cucumber-html-reporter/.gitignore b/test/e2e/images/reports/src/cucumber-html-reporter/.gitignore new file mode 100644 index 00000000..3c3629e6 --- /dev/null +++ b/test/e2e/images/reports/src/cucumber-html-reporter/.gitignore @@ -0,0 +1 @@ +node_modules diff --git a/test/e2e/images/reports/src/cucumber-html-reporter/LICENSE b/test/e2e/images/reports/src/cucumber-html-reporter/LICENSE new file mode 100644 index 00000000..09bef9c2 --- /dev/null +++ b/test/e2e/images/reports/src/cucumber-html-reporter/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2018 Wim Selles + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/test/e2e/images/reports/src/cucumber-html-reporter/README.MD b/test/e2e/images/reports/src/cucumber-html-reporter/README.MD new file mode 100644 index 00000000..e9560951 --- /dev/null +++ b/test/e2e/images/reports/src/cucumber-html-reporter/README.MD @@ -0,0 +1,14 @@ +Cucumber HTML Reporter +=============================== + +Based on [multiple-cucumber-html-reporter](github/wswebcreation/multiple-cucumber-html-reporter) + +## Credits + +In the search for a reporting tools for Cucumber I found a few tools that helped me a lot: + +- [multiple-cucumber-html-reporter](github/wswebcreation/multiple-cucumber-html-reporter) +- [cucumber-html-repository](https://github.com/gkushang/cucumber-html-reporter) +- [cucumber-html-report](https://github.com/leinonen/cucumber-html-report) +- [cucumber-protractor-report](https://github.com/JesterXL/cucumber-protractor-report) +- [grunt-protractor-cucumber-html-report](https://github.com/robhil/grunt-protractor-cucumber-html-report) diff --git a/test/e2e/images/reports/src/cucumber-html-reporter/lib/collect-jsons.js b/test/e2e/images/reports/src/cucumber-html-reporter/lib/collect-jsons.js new file mode 100644 index 00000000..872cecc4 --- /dev/null +++ b/test/e2e/images/reports/src/cucumber-html-reporter/lib/collect-jsons.js @@ -0,0 +1,64 @@ +const { readFileSync, statSync } = require('fs-extra'); +const { findJsonFiles, formatToLocalIso } = require('./utils'); +const { parseFeatureHooks, parseMetadata } = require('./parse.cucumber.data'); + +module.exports = function collectJSONS(options) { + const jsonOutput = []; + const files = findJsonFiles(options.jsonDir); + + if (files.length === 0) { + console.log(`WARNING: No JSON files found in '${options.jsonDir}'. NO REPORT CAN BE CREATED!`); + return []; + } + + files.map((file) => { + let data; + // Cucumber json can be empty, it's likely being created by another process (#47) + // or the data could not be a valid JSON-file + try { + data = JSON.parse(readFileSync(file).toString()); + } catch (e) { + data = []; + console.log(`WARNING: File: '${file}' had no valid JSON data due to error:'${e}'. CONTENT WAS NOT LOADED!`); + } + + const jsonData = Array.isArray(data) ? data : [data]; + const stats = statSync(file); + const reportTime = formatToLocalIso(stats.birthtime); + + jsonData.map((json) => { + json = parseMetadata(json, options.metadata); + + if (options.displayReportTime) { + json.metadata = { + ...json.metadata, + ...{ reportTime }, + }; + } + + // Only check the feature hooks if there are elements (fail safe) + const { elements } = json; + + if (elements) { + json.elements = elements.map((scenario) => { + const { before, after } = scenario; + + if (before) { + scenario.steps = parseFeatureHooks(before, 'Before') + .concat(scenario.steps); + } + if (after) { + scenario.steps = scenario.steps + .concat(parseFeatureHooks(after, 'After')); + } + + return scenario; + }); + } + + jsonOutput.push(json); + }); + }); + + return jsonOutput; +}; diff --git a/test/e2e/images/reports/src/cucumber-html-reporter/lib/generate-report.js b/test/e2e/images/reports/src/cucumber-html-reporter/lib/generate-report.js new file mode 100755 index 00000000..643e4876 --- /dev/null +++ b/test/e2e/images/reports/src/cucumber-html-reporter/lib/generate-report.js @@ -0,0 +1,380 @@ +const fs = require('fs-extra'); +const path = require('path'); +const ejs = require('ejs'); +const collectJSONS = require('./collect-jsons'); +const { + calculatePercentage, + createReportFolders, + formatDuration, + getCustomStyleSheet, + getGenericJsContent, + getStyleSheet, +} = require('./utils'); +const { parseScenarioSteps } = require('./parse.cucumber.data'); + +const INDEX_HTML = 'index.html'; +const FEATURE_FOLDER = 'features'; + +function generateReport(options) { + if (!options) { + throw new Error('Options need to be provided.'); + } + + if (!options.jsonDir) { + throw new Error('A path which holds the JSON files should be provided.'); + } + + if (!options.reportPath) { + throw new Error('An output path for the reports should be defined, no path was provided.'); + } + + const customMetadata = options.customMetadata || false; + const customData = options.customData || null; + const style = getStyleSheet(options.overrideStyle) + getCustomStyleSheet(options.customStyle); + const ingress = options.ingress || { name:'XXXXXXXX', version: 'XXXXXXXX'}; + const reportPath = path.resolve(process.cwd(), options.reportPath); + const durationInMS = options.durationInMS || false; + const pageFooter = options.pageFooter || false; + const buildTime = options.buildTime || "N/A"; + + createReportFolders(reportPath); + + const allFeatures = collectJSONS(options); + + const suite = { + customMetadata, + customData, + style, + name: '', + version: 'version', + time: new Date(), + features: allFeatures, + ingress, + totalFeaturesCount: { + ambiguous: { + count: 0, + percentage: 0, + }, + failed: { + count: 0, + percentage: 0, + }, + passed: { + count: 0, + percentage: 0, + }, + notDefined: { + count: 0, + percentage: 0, + }, + pending: { + count: 0, + percentage: 0, + }, + skipped: { + count: 0, + percentage: 0, + }, + total: 0, + }, + totalScenariosCount: { + ambiguous: { + count: 0, + percentage: 0, + }, + failed: { + count: 0, + percentage: 0, + }, + passed: { + count: 0, + percentage: 0, + }, + notDefined: { + count: 0, + percentage: 0, + }, + pending: { + count: 0, + percentage: 0, + }, + skipped: { + count: 0, + percentage: 0, + }, + total: 0, + }, + totalTime: 0, + }; + + parseFeatures(suite); + + // Percentages + suite.totalFeaturesCount = calculatePercentage(suite.totalFeaturesCount); + + createFeaturesOverviewIndexPage(suite); + createFeatureIndexPages(suite); + + // console.log(JSON.stringify(suite, null,2)); + console.log(`\n +===================================================================================== + Multiple Cucumber HTML report generated in: + + ${path.join(reportPath, INDEX_HTML)} +=====================================================================================\n`); + // console.log(JSON.stringify(suite, null, 2)); + + function parseFeatures(suite) { + suite.features.forEach((feature) => { + feature.totalFeatureScenariosCount = { + ambiguous: { + count: 0, + percentage: 0, + }, + failed: { + count: 0, + percentage: 0, + }, + passed: { + count: 0, + percentage: 0, + }, + notDefined: { + count: 0, + percentage: 0, + }, + pending: { + count: 0, + percentage: 0, + }, + skipped: { + count: 0, + percentage: 0, + }, + total: 0, + }; + feature.duration = 0; + feature.time = '0s'; + feature.isFailed = false; + feature.isAmbiguous = false; + feature.isSkipped = false; + feature.isNotdefined = false; + feature.isPending = false; + suite.totalFeaturesCount.total++; + feature.id = `${feature.id}`.replace(/[^a-zA-Z0-9-_]/g, '-'); + + if (!feature.elements) { + return; + } + + feature = parseScenarios(feature, suite); + + if (feature.isFailed) { + feature.failed++; + suite.totalFeaturesCount.failed.count++; + } else if (feature.isAmbiguous) { + feature.ambiguous++; + suite.totalFeaturesCount.ambiguous.count++; + } else if (feature.isNotdefined) { + feature.notDefined++; + suite.totalFeaturesCount.notDefined.count++; + } else if (feature.isPending) { + feature.pending++; + suite.totalFeaturesCount.pending.count++; + } else if (feature.isSkipped) { + feature.skipped++; + suite.totalFeaturesCount.skipped.count++; + } else { + feature.passed++; + suite.totalFeaturesCount.passed.count++; + } + + if (feature.duration) { + feature.totalTime += feature.duration; + feature.time = formatDuration(durationInMS, feature.duration); + } + + // Percentages + feature.totalFeatureScenariosCount = calculatePercentage(feature.totalFeatureScenariosCount); + suite.totalScenariosCount = calculatePercentage(suite.totalScenariosCount); + }); + } + + /** + * Parse each scenario within a feature + * @param {object} feature a feature with all the scenarios in it + * @return {object} return the parsed feature + * @private + */ + function parseScenarios(feature) { + feature.elements.forEach((scenario) => { + scenario.passed = 0; + scenario.failed = 0; + scenario.notDefined = 0; + scenario.skipped = 0; + scenario.pending = 0; + scenario.ambiguous = 0; + scenario.duration = 0; + scenario.time = '0s'; + scenario = parseScenarioSteps(scenario, durationInMS); + + if (scenario.duration > 0) { + feature.duration += scenario.duration; + scenario.time = formatDuration(durationInMS, scenario.duration); + } + + if (scenario.hasOwnProperty('description') && scenario.description) { + scenario.description = scenario.description.replace(new RegExp('\r?\n', 'g'), '
'); + } + + if (scenario.failed > 0) { + suite.totalScenariosCount.total++; + suite.totalScenariosCount.failed.count++; + feature.totalFeatureScenariosCount.total++; + feature.isFailed = true; + + return feature.totalFeatureScenariosCount.failed.count++; + } + + if (scenario.ambiguous > 0) { + suite.totalScenariosCount.total++; + suite.totalScenariosCount.ambiguous.count++; + feature.totalFeatureScenariosCount.total++; + feature.isAmbiguous = true; + + return feature.totalFeatureScenariosCount.ambiguous.count++; + } + + if (scenario.notDefined > 0) { + suite.totalScenariosCount.total++; + suite.totalScenariosCount.notDefined.count++; + feature.totalFeatureScenariosCount.total++; + feature.isNotdefined = true; + + return feature.totalFeatureScenariosCount.notDefined.count++; + } + + if (scenario.pending > 0) { + suite.totalScenariosCount.total++; + suite.totalScenariosCount.pending.count++; + feature.totalFeatureScenariosCount.total++; + feature.isPending = true; + + return feature.totalFeatureScenariosCount.pending.count++; + } + + if (scenario.skipped > 0) { + suite.totalScenariosCount.total++; + suite.totalScenariosCount.skipped.count++; + feature.totalFeatureScenariosCount.total++; + + return feature.totalFeatureScenariosCount.skipped.count++; + } + + /* istanbul ignore else */ + if (scenario.passed > 0) { + suite.totalScenariosCount.total++; + suite.totalScenariosCount.passed.count++; + feature.totalFeatureScenariosCount.total++; + + return feature.totalFeatureScenariosCount.passed.count++; + } + }); + + feature.isSkipped = feature.totalFeatureScenariosCount.total === feature.totalFeatureScenariosCount.skipped.count; + + return feature; + } + + function getScenarioStatus(scenario) { + if (scenario.failed) { + return 'danger'; + }else if (scenario.ambiguous) { + return 'warning'; + }else if (scenario.pending) { + return 'info' + }else if (scenario.skipped) { + return 'secondary' + }else if (scenario.passed) { + return 'success' + } + } + + /** + * Generate the features overview + * @param {object} suite JSON object with all the features and scenarios + * @private + */ + function createFeaturesOverviewIndexPage(suite) { + ejs.renderFile( + path.join(__dirname, '..', 'templates', 'features-overview.index.ejs'), + { + ...{ suite }, + ...{ + genericScript: getGenericJsContent(), + pageFooter, + buildTime, + ingress, + styles: suite.style, + }, + getScenarioStatus, + }, + { + rmWhitespace: true + }, + (err, str) => { + if (err) { + console.log('err = ', err); + return; + } + + fs.writeFileSync( + path.resolve(reportPath, INDEX_HTML), + str, + ); + }, + ); + } + + /** + * Generate the feature pages + * @param suite suite JSON object with all the features and scenarios + * @private + */ + function createFeatureIndexPages(suite) { + suite.features.forEach((feature) => { + const featurePage = path.resolve(reportPath, `${FEATURE_FOLDER}/${feature.id}.html`); + ejs.renderFile( + path.join(__dirname, '..', 'templates', 'feature-overview.index.ejs'), + { + ...{ suite }, + ...{ feature }, + ...{ + genericScript: getGenericJsContent(), + pageFooter, + buildTime, + ingress, + styles: suite.style, + }, + getScenarioStatus, + }, + {}, + (err, str) => { + if (err) { + console.log('err = ', err); + return; + } + + fs.writeFileSync( + featurePage, + str, + ); + }, + ); + }); + } +} + +module.exports = { + generate: generateReport, +}; diff --git a/test/e2e/images/reports/src/cucumber-html-reporter/lib/parse.cucumber.data.js b/test/e2e/images/reports/src/cucumber-html-reporter/lib/parse.cucumber.data.js new file mode 100644 index 00000000..187d68a6 --- /dev/null +++ b/test/e2e/images/reports/src/cucumber-html-reporter/lib/parse.cucumber.data.js @@ -0,0 +1,190 @@ +const { Base64 } = require('js-base64'); +const { escapeHtml, formatDuration, isBase64 } = require('./utils'); + +const RESULT_STATUS = { + passed: 'passed', + failed: 'failed', + skipped: 'skipped', + pending: 'pending', + notDefined: 'undefined', + ambiguous: 'ambiguous', +}; + +/** + * Parse the feature hook and return a step hook + * + * @param {Array} data + * @param {string} keyword + * + * @returns { + * { + * arguments: array, + * keyword: string, + * name: string, + * result: { + * status: string, + * }, + * line: string, + * match: { + * location: string + * }, + * embeddings: [] + * }[] + * } + */ +function parseFeatureHooks(data, keyword) { + return data.map((step) => { + const match = step.match && step.match.location ? step.match : { location: 'can not be determined' }; + + return { + arguments: step.arguments || [], + embeddings: step.embeddings || [], + keyword, + line: '', + match, + name: 'Hook', + result: step.result, + }; + }); +} + +/** + * Parse metadata and provide default data if needed + * + * @param {object} json + * @param {object} metadata + * + * @returns {{*}} + */ +function parseMetadata(json, metadata) { + const defaultMetadata = { + metadata: { + browser: { + name: 'not known', + version: 'not known', + }, + device: 'not known', + platform: { + name: 'not known', + version: 'not known', + }, + }, + }; + + return { + ...(metadata && !json.metadata ? { metadata } : defaultMetadata), + ...json, + }; +} + +/** + * Parse the scenario steps + * + * @param {object} scenario + * @param {boolean} durationInMS + * + * @returns {*} + */ +function parseScenarioSteps(scenario, durationInMS) { + scenario.steps.forEach((step) => { + if (step.embeddings !== undefined) { + step.attachments = []; + step.embeddings.forEach((embedding, embeddingIndex) => { + if (embedding.mime_type === 'application/json' || embedding.media && embedding.media.type === 'application/json') { + step.json = (step.json + ? step.json + : []).concat( + [typeof embedding.data === 'string' + ? JSON.parse(embedding.data) + : embedding.data, + ], + ); + } else if (embedding.mime_type === 'text/html' || (embedding.media && embedding.media.type === 'text/html')) { + step.html = (step.html ? step.html : []).concat([ + isBase64(embedding.data) ? Base64.decode(embedding.data) : embedding.data, + ]); + } else if (embedding.mime_type === 'text/plain' || (embedding.media && embedding.media.type === 'text/plain')) { + step.text = (step.text ? step.text : []).concat([ + isBase64(embedding.data) ? escapeHtml(Base64.decode(embedding.data)) : escapeHtml(embedding.data), + ]); + } else if (embedding.mime_type === 'image/png' || (embedding.media && embedding.media.type === 'image/png')) { + step.image = (step.image ? step.image : []).concat([`data:image/png;base64,${embedding.data}`]); + step.embeddings[embeddingIndex] = {}; + } else { + let embeddingtype = 'text/plain'; + if (embedding.mime_type) { + embeddingtype = embedding.mime_type; + } else if (embedding.media && embedding.media.type) { + embeddingtype = embedding.media.type; + } + step.attachments.push({ + data: `data:${embeddingtype};base64,${embedding.data}`, + type: embeddingtype, + }); + step.embeddings[embeddingIndex] = {}; + } + }); + } + + if (step.doc_string !== undefined) { + step.id = `${scenario.id}.${step.name}`.replace(/[^a-zA-Z0-9-_]/g, '-'); + step.restWireData = step.doc_string.value + .split(/[>]/) + .join('') + .replace(new RegExp('\r?\n', 'g'), '
'); + } + + // Don't log steps to the report that: + // - don't have a result + // - or are hidden with: + // - no text/image + // - attachments which are empty + // - and the result it not failed + // 9-10 times these are hooks + if (!step.result + || (step.hidden + && !step.text + && !step.image + && step.attachments + && step.attachments.length === 0 + && step.result.status !== RESULT_STATUS.failed + )) { + return; + } + + if (step.result.duration) { + scenario.duration += step.result.duration; + step.time = formatDuration(durationInMS, step.result.duration); + } + + if (step.result.status === RESULT_STATUS.passed) { + return scenario.passed++; + } + + if (step.result.status === RESULT_STATUS.failed) { + return scenario.failed++; + } + + if (step.result.status === RESULT_STATUS.notDefined) { + return scenario.notDefined++; + } + + if (step.result.status === RESULT_STATUS.pending) { + return scenario.pending++; + } + + if (step.result.status === RESULT_STATUS.ambiguous) { + return scenario.ambiguous++; + } + + scenario.skipped++; + }); + + return scenario; +} + +module.exports = { + parseFeatureHooks, + parseMetadata, + parseScenarioSteps, +}; diff --git a/test/e2e/images/reports/src/cucumber-html-reporter/lib/utils.js b/test/e2e/images/reports/src/cucumber-html-reporter/lib/utils.js new file mode 100644 index 00000000..01935870 --- /dev/null +++ b/test/e2e/images/reports/src/cucumber-html-reporter/lib/utils.js @@ -0,0 +1,209 @@ +const { join, resolve } = require('path'); +const { + accessSync, constants, ensureDirSync, readdirSync, readFileSync, +} = require('fs-extra'); +const moment = require('moment'); + +/** + * Find all JSON Files + * + * @param {string} dir + * + * @returns {string[]} + */ +function findJsonFiles(dir) { + const folder = resolve(process.cwd(), dir); + + try { + return readdirSync(folder) + .filter((file) => file.slice(-5) === '.json') + .map((file) => join(folder, file)); + } catch (e) { + throw new Error(`There were issues reading JSON-files from '${folder}'.`); + } +} + +/** + * Format input date to YYYY/MM/DD HH:mm:ss + * + * @param {Date} date + * + * @returns {string} formatted date in ISO format local time + */ +function formatToLocalIso(date) { + return moment(date).format('YYYY/MM/DD HH:mm:ss'); +} + +/** + * Create the report folders + * + * @param {string} folder + */ +function createReportFolders(folder) { + ensureDirSync(folder); + ensureDirSync(resolve(folder, 'features')); +} + +/** + * Format the duration to HH:mm:ss.SSS + * + * @param {boolean} durationInMS + * @param {number} ns + * + * @return {string} + */ +function formatDuration(durationInMS, ns) { + // `moment.utc(#)` needs ms, we now use device by 1000000 to calculate ns to ms + ft = moment.utc(durationInMS ? ns : ns / 1000000).format('m[m]s[s]'); + if (ft.startsWith("0m")) { + return ft.substr(2, ft.length); + } + + return ft; +} + +/** + * Escape html in data string + * + * @param {*} data + * + * @return {*} + */ +function escapeHtml(data) { + return (typeof data === 'string' || data instanceof String) + ? data.replace(//g, '>') + : data; +} + +/** + * Check if the string a base64 string + * + * @param {string} string + * + * @return {boolean} + */ +function isBase64(string) { + const notBase64 = /[^A-Z0-9+\/=]/i; + const stringLength = string.length; + + if (!stringLength || stringLength % 4 !== 0 || notBase64.test(string)) { + return false; + } + + const firstPaddingChar = string.indexOf('='); + + return firstPaddingChar === -1 + || firstPaddingChar === stringLength - 1 + || (firstPaddingChar === stringLength - 2 && string[stringLength - 1] === '='); +} + +/** + * Read a file and return it's content + * + * @param {string} fileName + * + * @return {*} Content of the file + */ +function getTemplateFileContent(fileName) { + return readFileSync(join(__dirname, '..', 'templates', fileName), 'utf-8'); +} + +/** + * Get the generic JS file content + * + * @returns {string} + */ +function getGenericJsContent() { + return getTemplateFileContent('generic.js'); +} + +/** + * Get the custom style sheet content + * + * @param {string} fileName + * + * @returns {string} + */ +function getCustomStyleSheet(fileName = '') { + if (fileName) { + try { + // This is for getting the content of custom CSS files + accessSync(fileName, constants.R_OK); + + return readFileSync(fileName, 'utf-8'); + } catch (err) { + console.log(`WARNING: Custom stylesheet: '${fileName}' could not be loaded due to '${err}'.`); + } + } + + return ''; +} + +/** + * Get the style sheet content + * + * @param {string} fileName + * + * @returns {string} + */ +function getStyleSheet(fileName = '') { + if (fileName) { + try { + // This is for getting the content of custom CSS files + accessSync(fileName, constants.R_OK); + + return readFileSync(fileName, 'utf-8'); + } catch (err) { + console.log(`WARNING: Override stylesheet: '${fileName}' could not be loaded due to '${err}'. The default will be loaded.`); + } + } + + return getTemplateFileContent('style.css'); +} + +/** + * Calculate the percentage of keys + * + * @example: + * { + * ambiguous: { + * count: 0, + * percentage: 0 + * }, + * failed: { + * count: 0, + * percentage: 0 + * }, + * passed: { + * count: 0, + * percentage: 0 + * }, + * total: 0, + * } + * + * @param {object} obj + * + * @returns {object} + */ +function calculatePercentage(obj) { + Object.keys(obj).forEach((key) => { + if (key !== 'total') { + obj[key].percentage = ((obj[key].count / obj.total) * 100).toFixed(2); + } + }); + + return obj; +} + +module.exports = { + calculatePercentage, + createReportFolders, + escapeHtml, + findJsonFiles, + formatDuration, + formatToLocalIso, + getCustomStyleSheet, + getGenericJsContent, + getStyleSheet, + isBase64, +}; diff --git a/test/e2e/images/reports/src/cucumber-html-reporter/package.json b/test/e2e/images/reports/src/cucumber-html-reporter/package.json new file mode 100644 index 00000000..e7244533 --- /dev/null +++ b/test/e2e/images/reports/src/cucumber-html-reporter/package.json @@ -0,0 +1,35 @@ +{ + "name": "multiple-cucumber-html-reporter", + "version": "1.18.0", + "description": "Generate beautiful Cucumber reports for multiple instances purposes", + "keywords": [ + "cucumber", + "html", + "test report", + "multiple-cucumber-html-reporter", + "html report", + "json to html" + ], + "main": "lib/generate-report.js", + "license": "MIT", + "repository": { + "type": "git", + "url": "git+https://github.com/wswebcreation/multiple-cucumber-html-reporter.git" + }, + "author": "wswebcreation", + "homepage": "https://github.com/wswebcreation/multiple-cucumber-html-reporter#readme", + "dependencies": { + "ejs": "^3.1.3", + "fs-extra": "^9.0.1", + "js-base64": "^3.4.1", + "moment": "^2.27.0" + }, + "devDependencies": { + "coveralls": "^3.1.0", + "eslint": "^7.7.0", + "eslint-config-airbnb-base": "^14.2.0", + "eslint-plugin-import": "^2.22.0", + "jest": "^26.2.1", + "np": "^6.3.2" + } +} diff --git a/test/e2e/images/reports/src/cucumber-html-reporter/templates/components/features-overview.chart.ejs b/test/e2e/images/reports/src/cucumber-html-reporter/templates/components/features-overview.chart.ejs new file mode 100644 index 00000000..8128d59e --- /dev/null +++ b/test/e2e/images/reports/src/cucumber-html-reporter/templates/components/features-overview.chart.ejs @@ -0,0 +1,62 @@ +
+
+ +
+
+
Features
+ + + + + + + + + + + + + + + + + <%if(suite.totalFeaturesCount.ambiguous.count > 0){ %> + + + + + <%} %> + <%if(suite.totalFeaturesCount.notDefined.count > 0){ %> + + + + + <%} %> + <%if(suite.totalFeaturesCount.pending.count > 0){ %> + + + + + <%} %> + <%if(suite.totalFeaturesCount.skipped.count > 0){ %> + + + + + <%} %> + +
StatusProgress
+

Passed

+ 		<%- suite.totalFeaturesCount.passed.percentage %>%
+

Failed

+ 		<%- suite.totalFeaturesCount.failed.percentage %>%
+

Ambiguous

+ 		<%- suite.totalFeaturesCount.ambiguous.percentage %>%
+

Not Defined

+ 		<%- suite.totalFeaturesCount.notDefined.percentage %>%
+

Pending

+ 		<%- suite.totalFeaturesCount.pending.percentage %>%
+

Skipped

+ 		<%- suite.totalFeaturesCount.skipped.percentage %>%
+
+
diff --git a/test/e2e/images/reports/src/cucumber-html-reporter/templates/components/features-overview.ejs b/test/e2e/images/reports/src/cucumber-html-reporter/templates/components/features-overview.ejs new file mode 100644 index 00000000..590974b5 --- /dev/null +++ b/test/e2e/images/reports/src/cucumber-html-reporter/templates/components/features-overview.ejs @@ -0,0 +1,73 @@ +

Features overview

+ + + + + + + + + + <%if(+suite.totalScenariosCount.pending.count > 0) { %><%} %> + <%if(+suite.totalScenariosCount.skipped.count > 0) { %><%} %> + <%if(+suite.totalScenariosCount.notDefined.count > 0) { %><%} %> + <%if(+suite.totalScenariosCount.ambiguous.count > 0) { %><%} %> + + + + + <%suite.features.forEach(function(feature, featureIndex) { %> + + + + + + + <%if(+suite.totalScenariosCount.skipped.count > 0) { %> + <%} %><%if(+suite.totalScenariosCount.pending.count > 0) { %> + <%} %><%if(+suite.totalScenariosCount.notDefined.count > 0) { %> + <%} %><%if(+suite.totalScenariosCount.ambiguous.count > 0) { %> + <%} %> + + <%}); %> + +
Feature nameStatusTotalPassedFailedPendingSkipUndefinedAmbiguousDuration
+ <%- feature.name %> + + <%if (feature.tags) { %> + <%var amount = feature.tags.length; %> + <%var tags = feature.tags.reduce((tags, tag) => tags + tag.name + ' ', ''); %> + <%if (amount > 0 ){ %> + bookmark<% if(amount > 1 ) { %>s<% } %> + <%} %> + <%} %> + + <%var statusIcon; %> + <%var status; %> + <%var statusColor; %> + <%if (feature.isFailed) { %> + <%status = 'Failed'; %> + <%statusIcon = 'error'; %> + <%statusColor = 'text-danger'; %> + <%} else if (feature.isAmbiguous) { %> + <%status = 'Ambiguous'; %> + <%statusIcon = 'face'; %> + <%statusColor = 'ambiguous-color'; %> + <%} else if (feature.isNotdefined) { %> + <%status = 'Not Defined'; %> + <%statusIcon = 'help'; %> + <%statusColor = 'not-defined-color'; %> + <%} else if (feature.isPending) { %> + <%status = 'Pending'; %> + <%statusIcon = 'pending'; %> + <%statusColor = 'pending-color'; %> + <%} else if (feature.isSkipped) { %> + <%status = 'Skipped'; %> + <%statusIcon = 'double_arrow'; %> + <%statusColor = 'skipped-color'; %> + <%} else { %> + <%status = 'Passed'; %> + <%statusIcon = 'check_circle'; %> + <%statusColor = 'passed-color'; %> + <%} %> + <%- statusIcon %><%- feature.totalFeatureScenariosCount.total %><%- feature.totalFeatureScenariosCount.passed.count %><%- feature.totalFeatureScenariosCount.failed.count %><%- feature.totalFeatureScenariosCount.skipped.count %><%- feature.totalFeatureScenariosCount.pending.count %><%- feature.totalFeatureScenariosCount.notDefined.count %><%- feature.totalFeatureScenariosCount.ambiguous.count %><%- feature.time %>
diff --git a/test/e2e/images/reports/src/cucumber-html-reporter/templates/components/scenarios-overview.chart.ejs b/test/e2e/images/reports/src/cucumber-html-reporter/templates/components/scenarios-overview.chart.ejs new file mode 100644 index 00000000..662d20d6 --- /dev/null +++ b/test/e2e/images/reports/src/cucumber-html-reporter/templates/components/scenarios-overview.chart.ejs @@ -0,0 +1,54 @@ +
+
+ +
+
+
Scenarios
+ + + + + + + + + + + + + + + <%if (scenarios.ambiguous.count > 0) { %> + + + + + <%} %><%if (scenarios.notDefined.count > 0) { %> + + + + <%} %><%if (scenarios.pending.count > 0) { %> + + + + <%} %><%if (scenarios.skipped.count > 0) { %> + + + + <%} %> + +
StatusProgress
+

Passed

+ 		<%- scenarios.passed.percentage %>%
+

Failed

+ 		<%- scenarios.failed.percentage %>%
+

Ambiguous

+ 		<%- scenarios.ambiguous.percentage %>%
+

Not defined

+ 		<%- scenarios.notDefined.percentage %>%
+

Pending

+ 		<%- scenarios.pending.percentage %>%
+

Skipped

+ 		<%- scenarios.skipped.percentage %>%
+
+
diff --git a/test/e2e/images/reports/src/cucumber-html-reporter/templates/components/scenarios.ejs b/test/e2e/images/reports/src/cucumber-html-reporter/templates/components/scenarios.ejs new file mode 100755 index 00000000..1abf62aa --- /dev/null +++ b/test/e2e/images/reports/src/cucumber-html-reporter/templates/components/scenarios.ejs @@ -0,0 +1,180 @@ +<% scenarios && scenarios.forEach(function(scenario, scenarioIndex) { %> +
+
+
+
+ + <%- scenario.keyword %>: <%- scenario.name %> + +

<%- scenario.description %>

+
+ +
+ <% scenario.steps.forEach( function(step, stepIndex) { %> + <% if(!step.hidden || step.image || step.text || step.html || step.attachment) { %> +
+ <% if(step.result) { %> + <% if(step.result.status === 'passed') { %> +
+ check_circle +
+ <% } else if(step.result.status === 'failed') { %> +
error +
+ <% } else if(step.result.status === 'skipped') { %> +
+ double_arrow +
+ <% } else if(step.result.status === 'pending') { %> +
+ pending +
+ <% } else if(step.result.status === 'ambiguous') { %> +
+ face +
+ <% } else { %> +
+ help +
+ <% } %> + <% } %> +
+ <%- step.keyword %> + <% if (step.time) { -%> + <%- step.time %> + <% } else { -%> + 0s + <% } -%> + <% if(step.name) { -%> + <% + var stepName = step.name.replace(/(?!^)"(.*?)"/g, "$1"); + stepName = stepName.replace(/[response status\-code must be ](\d+)/g, " $1"); + %> + <%- stepName -%> + <% if(step.restWireData) { %> + +Show Parameter +
+ + 
<%- step.restWireData %>
+ +
+ <% } %> + <% } %> +
+ <% if(step.result) { %> + <% if (step.result.error_message) { %>+Show Error<% } %> + <% if (step.text || step.json) { %>+Show Info<% } %> + <% if (step.html) { %>+Show Info<% } %> + <% if (step.image) { %>+Screenshot<% } %> + <% if (step.attachments && step.attachments.length > 0) { %> + [ + <% step.attachments.forEach(function(attachment, attachmentIndex) { %> + Attachment + <% if ( attachmentIndex < (step.attachments.length - 1) ) { %> + , + <% } %> + <% }); %> + ] + <% } %> + <% } %> +
+ <% if(step.result) { %> + <% if (step.result.error_message) { %> +
+ 
<%- step.result.error_message.replace(//g, ')') %>
+
+ <% } %> + <% if (step.json) { %> +
+ 
+          <% try { %><%- JSON.stringify(step.json, undefined, 2) %>
+          <% } catch (error) { %><%- step.json %>
+          <% } %>
+
+
+ <% } %> + <% if (step.text) { %> +
+ 
<%- step.text.join('
') %>
+
+ <% } %> + <% if (step.html) { %> +
+
<%- step.html.join('
') %>
+
+ <% } %> + <% if (step.image) { %> +
+ <% for( var i = 0; i < step.image.length; i++ ) { %> + + <% } %> +
+ <% } %> + <% if (step.attachments && step.attachments.length > 0) { %> + <% step.attachments.forEach(function(attachment, attachmentIndex) { %> +
+ +
+ <% }); %> + <% } %> + <% } %> + <% if(step.arguments) { %> + <% for( var i = 0; i < step.arguments.length; i++ ) { %> +
+ <% if(step.arguments[i]["rows"]) { %> + + <% var rows = step.arguments[i]["rows"]; %> + + <% var cells = rows[0]["cells"]; %> + <% for( var k = 0; k < cells.length; k++ ) { %> + + <% } %> + + <% for( var j = 1; j < rows.length; j++ ) { %> + + <% var cells = rows[j]["cells"]; %> + <% for( var k = 0; k < cells.length; k++ ) { %> + + <% } %> + + <% } %> +
<%- cells[k] %>
<%- cells[k] %>
+ <% } %> +
+ <% if(step.arguments[i]["content"]) { %> + 
<%- step.arguments[i]["content"].replace(//g, ')') %>
+ <% } %> + <% } %> + <% } %> + <% } %> + <% }); %> +
+
+
+ Duration: <%- scenario.time %> query_builder +
+ <% if (scenario.ambiguous) { %> + <%- scenario.ambiguous %> + <% } %> + <% if (scenario.pending) { %> + <%- scenario.pending %> + <% } %> + <% if (scenario.skipped) { %> + <%- scenario.skipped %> + <% } %> + <% if (scenario.failed) { %> + <%- scenario.failed %> + <% } %> + <% if (scenario.passed) { %> + <%- scenario.passed %> + <% } %> + <% if (scenario.notDefine) { %> + <%- scenario.notDefined %> + <% } %> +
+
+
+
+<% }); %> diff --git a/test/e2e/images/reports/src/cucumber-html-reporter/templates/feature-overview.index.ejs b/test/e2e/images/reports/src/cucumber-html-reporter/templates/feature-overview.index.ejs new file mode 100644 index 00000000..f159022a --- /dev/null +++ b/test/e2e/images/reports/src/cucumber-html-reporter/templates/feature-overview.index.ejs @@ -0,0 +1,113 @@ + + + + + + bfe-ingress-controller e2e test + + + + + + + + + + +
+
+
+ <%if (feature.tags) { %> + <%var amount = feature.tags.length; %> + <%if (amount > 0 ){ %> + <%feature.tags.forEach((tag) => { %> + <%- tag.name %> + <%}); %> + <%} %> + <%} %> +

Feature: <%- feature.name %>

+

<%- feature.description %>

+
+
+ <%- include('components/scenarios-overview.chart.ejs', {overviewPage: false, scenarios: feature.totalFeatureScenariosCount }) %> +
+
+ +
+ <%- include('components/scenarios.ejs', {scenarios: feature.elements}) %> +
+ + <%if (pageFooter) { %> +
+
+ <%- pageFooter %> +
+
+ <%} %> +
+ + + + + + + + + diff --git a/test/e2e/images/reports/src/cucumber-html-reporter/templates/features-overview.index.ejs b/test/e2e/images/reports/src/cucumber-html-reporter/templates/features-overview.index.ejs new file mode 100644 index 00000000..54de5224 --- /dev/null +++ b/test/e2e/images/reports/src/cucumber-html-reporter/templates/features-overview.index.ejs @@ -0,0 +1,142 @@ + + + + + + bfe-ingress-controller e2e test + + + + + + + + + + + +
+
+
+ <%- include('components/features-overview.chart.ejs') %> +
+
+ <%- include('components/scenarios-overview.chart.ejs', {overviewPage: true, scenarios: suite.totalScenariosCount}) %> +
+
+
+
+ <%- include('components/features-overview.ejs') %> +
+
<%if (pageFooter) { %> +
+
+ <%- pageFooter %> +
+
<%} %> +
+ + + + + + + + + diff --git a/test/e2e/images/reports/src/cucumber-html-reporter/templates/generic.js b/test/e2e/images/reports/src/cucumber-html-reporter/templates/generic.js new file mode 100644 index 00000000..e79e60fb --- /dev/null +++ b/test/e2e/images/reports/src/cucumber-html-reporter/templates/generic.js @@ -0,0 +1,3 @@ +$('body').tooltip({ + selector: '[data-toggle="tooltip"]' +}); \ No newline at end of file diff --git a/test/e2e/images/reports/src/cucumber-html-reporter/templates/style.css b/test/e2e/images/reports/src/cucumber-html-reporter/templates/style.css new file mode 100644 index 00000000..aeb759b9 --- /dev/null +++ b/test/e2e/images/reports/src/cucumber-html-reporter/templates/style.css @@ -0,0 +1,209 @@ +/* colors */ +.ambiguous-color { + color: #E74C3C !important; +} + +.failed-color { + color: #E74C3C !important; +} + +.not-defined-color { + color: #F39C12 !important; +} + +.passed-color { + color: #1ABB9C !important; +} + +.pending-color { + color: #FFD119 !important; +} + +.skipped-color { + color: #3498DB !important; +} + +/* backgrounds */ +.ambiguous-background { + background: #b73122 !important; +} + +.failed-background { + background: #E74C3C !important; +} + +.not-defined-background { + background: #F39C12 !important; +} + +.passed-background { + background: #1ABB9C !important; +} + +.pending-background { + background: #FFD119 !important; +} + +.skipped-background { + background: #3498DB !important; +} + +ul.quick-list { + padding-left: 0; + display: inline-block; +} + +ul.quick-list li, +table.quick-list tr { + padding-left: 10px; + list-style: none; + margin: 0; + padding-bottom: 6px; + padding-top: 4px; + white-space: nowrap; + text-overflow: ellipsis; + overflow: hidden; +} + +ul.quick-list li .meta-data-title, +table.quick-list td.meta-data-title { + display: inline-block; + min-width: 75px; + font-weight: bold; +} + +ul.quick-list li span, +table.quick-list td { + line-height: 28px; +} + +ul.quick-list li, +table.quick-list tr { + border-bottom: 1px solid #efefef; + padding: 0.5em 0; +} + +ul.quick-list li:last-child, +table.quick-list tr:last-child { + border-bottom: none; +} + +ul.quick-list li i { + padding-right: 10px; + color: #757679; +} + +.screenshot { + max-height: 100%; + max-width: 100%; +} + +/* Features / Scenarios */ + +.scenario-step-container { + margin-bottom: 10px; +} + +.scenario-step-container .label { + display: inline-block; + text-align: center; + width: 30px; +} + +.scenario-step-container .text { + display: inline; +} + +.scenario-step-container .duration { + position: relative; + float: right; +} + +.scenario-step-container .text .keyword.highlight { + font-size: 1.2em; + font-weight: 700; +} + +.scenario-scroll-bar { + overflow-x: scroll; +} + +.scenario-step-collapse, +.scenario-scroll-bar .arguments { + margin-left: 30px; + width: auto; +} + +/* NUEVO */ + +td p { + padding: 0; + margin: 0 +} + +td.percentage { + text-align: right; +} + +div.chart { + position: relative; +} + +body { + background-color: #fff !important +} + +h2 { + border-left: 3px solid orange; + padding-left: 4px; + font-size: 1.5rem +} + +h5 { + border-left: 2px solid orange; + padding-left: 4px +} + +th { + font-weight: bolder +} + +.step { + color: #ffffff; + display: inline-block; + font-size: 14px; + height: 30px; + margin-right: 1px; + padding: 5px; + text-align: center; + width: 30px; +} + +.step:first-of-type { + margin-left: 30px; +} + +code { + padding: 20px; + background-color: #f8f8f8; + display: block; +} + +.card { + border: 1px solid rgba(0, 0, 0, .12); +} + +body { + min-height: 75rem; + padding-top: 4.5rem; +} + +.scenario-step-collapse { + padding: 0; + margin: 0; +} + +kbd { + background-color: lightgrey; + color: black; +} diff --git a/test/e2e/images/reports/src/index.js b/test/e2e/images/reports/src/index.js new file mode 100644 index 00000000..30d36e9d --- /dev/null +++ b/test/e2e/images/reports/src/index.js @@ -0,0 +1,16 @@ +const report = require("./cucumber-html-reporter"); +const assert = require("assert"); + +assert(process.env.INPUT_DIRECTORY, "Environment variable INPUT_DIRECTORY is not optional"); +assert(process.env.OUTPUT_DIRECTORY, "Environment variable OUTPUT_DIRECTORY is not optional"); + +report.generate({ + jsonDir: process.env.INPUT_DIRECTORY, + reportPath: process.env.OUTPUT_DIRECTORY, + pageFooter: '

BFE ingress controller e2e test

', + ingress: { + controller: process.env.INGRESS_CONTROLLER || 'N/A', + version: process.env.CONTROLLER_VERSION || 'N/A' + }, + buildTime: process.env.BUILD +}); diff --git a/test/e2e/images/reports/src/package.json b/test/e2e/images/reports/src/package.json new file mode 100644 index 00000000..14bf31d6 --- /dev/null +++ b/test/e2e/images/reports/src/package.json @@ -0,0 +1,14 @@ +{ + "name": "ingress-conformance-reports", + "version": "0.0.1", + "dependencies": { + "ejs": "^3.1.5", + "fs-extra": "^9.0.1", + "js-base64": "^3.4.5", + "moment": "^2.27.0" + }, + "scripts": { + "run": "node index" + }, + "license": "Apache-2.0" +} diff --git a/test/e2e/pkg/files/files.go b/test/e2e/pkg/files/files.go new file mode 100644 index 00000000..95fef1fd --- /dev/null +++ b/test/e2e/pkg/files/files.go @@ -0,0 +1,60 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package files + +import ( + "fmt" + "io/ioutil" + "os" +) + +// Read tries to retrieve the desired file content from +// one of the registered file sources. +func Read(path string) ([]byte, error) { + if exists := Exists(path); !exists { + return nil, fmt.Errorf("file %v does not exists", path) + } + + data, err := ioutil.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("fatal error retrieving test file %s: %s", path, err) + } + + return data, nil +} + +// Exists checks whether a file could be read. Unexpected errors +// are handled by calling the fail function, which then should +// abort the current test. +func Exists(path string) bool { + _, err := os.Stat(path) + if os.IsNotExist(err) { + return false + } + + return true +} + +// IsDir reports whether path is a directory +func IsDir(path string) bool { + info, err := os.Stat(path) + if os.IsNotExist(err) { + return false + } + + return info.IsDir() +} diff --git a/test/e2e/pkg/http/http.go b/test/e2e/pkg/http/http.go new file mode 100644 index 00000000..4186ee30 --- /dev/null +++ b/test/e2e/pkg/http/http.go @@ -0,0 +1,200 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package http + +import ( + "crypto/tls" + "crypto/x509" + "encoding/json" + "fmt" + "io/ioutil" + "net/http" + "net/http/httputil" + "regexp" + "strings" + "time" +) + +var ( + // HTTPClientTimeout specifies a time limit for requests made by a client + HTTPClientTimeout = 10 * time.Second + // EnableHttpDebug enable dump of requests and responses of HTTP requests (useful for debug) + EnableDebug = false +) + +// CapturedRequest contains the original HTTP request metadata as received +// by the echoserver handling the test request. +type CapturedRequest struct { + Path string `json:"path"` + Host string `json:"host"` + Method string `json:"method"` + Proto string `json:"proto"` + Headers map[string][]string `json:"headers"` + + Namespace string `json:"namespace"` + Ingress string `json:"ingress"` + Service string `json:"service"` + Pod string `json:"pod"` +} + +// CapturedResponse contains the HTTP response metadata from the echoserver. +type CapturedResponse struct { + StatusCode int + ContentLength int64 + Proto string + Headers map[string][]string + TLSHostname string + + Certificate *x509.Certificate +} + +// CaptureRoundTrip will perform an HTTP request and return the CapturedRequest and CapturedResponse tuple +func CaptureRoundTrip(method, scheme, hostname, path, location string, headerInfo http.Header) (*CapturedRequest, *CapturedResponse, error) { + var capturedTLSHostname string + var certificate *x509.Certificate + + tr := &http.Transport{ + DisableCompression: true, + DisableKeepAlives: true, + TLSClientConfig: &tls.Config{ + // Skip all usual TLS verifications, since we are using self-signed certificates. + InsecureSkipVerify: true, + VerifyPeerCertificate: func(certificates [][]byte, _ [][]*x509.Certificate) error { + certs := make([]*x509.Certificate, len(certificates)) + for i, asn1Data := range certificates { + cert, err := x509.ParseCertificate(asn1Data) + if err != nil { + return fmt.Errorf("tls: failed to parse certificate from server: " + err.Error()) + } + certs[i] = cert + } + capturedTLSHostname = certs[0].DNSNames[0] + certificate = certs[0] + return nil + }, + }, + } + + if scheme == "https" && hostname != "" { + tr.TLSClientConfig.ServerName = hostname + } + + client := &http.Client{ + Transport: tr, + Timeout: HTTPClientTimeout, + CheckRedirect: func(req *http.Request, via []*http.Request) error { + return http.ErrUseLastResponse + }, + } + + url := fmt.Sprintf("%s://%s/%s", scheme, location, strings.TrimPrefix(path, "/")) + req, err := http.NewRequest(method, url, nil) + if err != nil { + return nil, nil, err + } + + if hostname != "" { + req.Host = hostname + } + + if headerInfo != nil { + req.Header = headerInfo + } + + if EnableDebug { + dump, err := httputil.DumpRequestOut(req, true) + if err != nil { + return nil, nil, err + } + + fmt.Printf("Sending request:\n%s\n\n", formatDump(dump, "> ")) + } + + resp, err := client.Do(req) + if err != nil { + return nil, nil, err + } + defer resp.Body.Close() + + if EnableDebug { + dump, err := httputil.DumpResponse(resp, true) + if err != nil { + return nil, nil, err + } + + fmt.Printf("Received response:\n%s\n\n", formatDump(dump, "< ")) + } + + // check if the result is a redirect and return a new request + // this avoids the issue of URLs without valid DNS names and + // also sends the traffic to the ingress controller IP address or FQDN + if isRedirect(resp.StatusCode) { + redirectURL, err := resp.Location() + if err != nil { + return nil, nil, err + } + + return CaptureRoundTrip(method, redirectURL.Scheme, redirectURL.Hostname(), redirectURL.Path, location, headerInfo) + } + + capReq := CapturedRequest{} + body, _ := ioutil.ReadAll(resp.Body) + + // we cannot assume the response is JSON + if isJSON(body) { + err = json.Unmarshal(body, &capReq) + if err != nil { + return nil, nil, fmt.Errorf("unexpected error reading response: %w", err) + } + } + + capRes := &CapturedResponse{ + resp.StatusCode, + resp.ContentLength, + resp.Proto, + resp.Header, + capturedTLSHostname, + certificate, + } + + return &capReq, capRes, nil +} + +func isJSON(content []byte) bool { + var js map[string]interface{} + return json.Unmarshal(content, &js) == nil +} + +func isRedirect(statusCode int) bool { + switch statusCode { + case http.StatusMovedPermanently, + http.StatusFound, + http.StatusSeeOther, + http.StatusTemporaryRedirect, + http.StatusPermanentRedirect: + return true + } + + return false +} + +var startLineRegex = regexp.MustCompile(`(?m)^`) + +func formatDump(data []byte, prefix string) string { + data = startLineRegex.ReplaceAllLiteral(data, []byte(prefix)) + return string(data) +} diff --git a/test/e2e/pkg/kubernetes/deployment.go b/test/e2e/pkg/kubernetes/deployment.go new file mode 100644 index 00000000..f201e6e4 --- /dev/null +++ b/test/e2e/pkg/kubernetes/deployment.go @@ -0,0 +1,312 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kubernetes + +import ( + "context" + "fmt" + "strconv" + "strings" + "time" + + appsv1 "k8s.io/api/apps/v1" + autoscalingv1 "k8s.io/api/autoscaling/v1" + corev1 "k8s.io/api/core/v1" + networking "k8s.io/api/networking/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/kubernetes" + "sigs.k8s.io/yaml" + + "github.com/bfenetworks/ingress-bfe/test/e2e/pkg/kubernetes/templates" +) + +// EchoService name of the deployment for the echo app +const EchoService = "echo" + +// EchoContainer container image name +const EchoContainer = "local/echoserver:0.0.1" + +// NewEchoDeployment creates a new deployment of the echoserver image in a particular namespace. +func NewEchoDeployment(kubeClientSet kubernetes.Interface, namespace, name, serviceName, servicePortName string, servicePort int32) error { + deploymentName := fmt.Sprintf("%v-%v", name, serviceName) + + deployment, err := kubeClientSet.AppsV1().Deployments(namespace).Get(context.TODO(), deploymentName, metav1.GetOptions{}) + if err != nil { + if !apierrors.IsNotFound(err) { + return err + } + + // if the deployment doesn't exists is still returned + deployment = nil + } + + // assume an existing deployment is ok + if deployment != nil { + return nil + } + + deploymentData := struct { + Name string + MatchLabels string + Labels string + Image string + Ingress string + Service string + PortName string + }{ + deploymentName, + deploymentName, + deploymentName, + EchoContainer, + name, + serviceName, + servicePortName, + } + + manifest, err := templates.Render("deployment", deploymentData) + if err != nil { + return err + } + + deployment, err = deploymentFromManifest(manifest) + if err != nil { + return err + } + + err = displayYamlDefinition(deployment) + if err != nil { + return fmt.Errorf("unable show yaml definition: %v", err) + } + + _, err = kubeClientSet.AppsV1().Deployments(namespace).Create(context.TODO(), deployment, metav1.CreateOptions{}) + if err != nil { + return fmt.Errorf("creating deployment (%v): %w", deployment.Name, err) + } + + serviceData := struct { + Name string + Selector string + Port int32 + }{ + serviceName, + deploymentName, + servicePort, + } + + manifest, err = templates.Render("service", serviceData) + if err != nil { + return err + } + + service, err := serviceFromManifest(manifest) + if err != nil { + return err + } + + if servicePortName != "" { + service.Spec.Ports[0].Name = servicePortName + } + + // if no port is defined, use default 8080 + if servicePort == 0 { + service.Spec.Ports[0].Port = 8080 + } + + err = displayYamlDefinition(service) + if err != nil { + return fmt.Errorf("unable show yaml definition: %v", err) + } + + service, err = kubeClientSet.CoreV1().Services(namespace).Create(context.TODO(), service, metav1.CreateOptions{}) + if err != nil { + return fmt.Errorf("creating service (%v): %w", service.Name, err) + } + + err = waitForEndpoints(kubeClientSet, WaitForEndpointsTimeout, service.Namespace, service.Name, 1) + if err != nil { + return fmt.Errorf("waiting for service (%v) endpoints available: %w", service.Name, err) + } + + return nil +} + +// DeploymentsFromIngress creates the required deployments for the services defined in the ingress object +func DeploymentsFromIngress(kubeClientSet kubernetes.Interface, ingress *networking.Ingress) error { + + if ingress.Spec.DefaultBackend != nil { + serviceName := ingress.Spec.DefaultBackend.Service.Name + servicePort := ingress.Spec.DefaultBackend.Service.Port.Number + + err := NewEchoDeployment(kubeClientSet, ingress.Namespace, ingress.Name, serviceName, "", servicePort) + if err != nil { + return err + } + } + + for _, rule := range ingress.Spec.Rules { + if rule.HTTP == nil { + continue + } + + for _, path := range rule.HTTP.Paths { + serviceName := path.Backend.Service.Name + servicePort := path.Backend.Service.Port.Number + + err := NewEchoDeployment(kubeClientSet, ingress.Namespace, ingress.Name, serviceName, "", servicePort) + if err != nil { + return err + } + } + } + + return nil +} + +// DeploymentsFromIngressForBalance creates the required deployments for the services defined in the ingress object or in the param of service info +func DeploymentsFromIngressForBalance(kubeClientSet kubernetes.Interface, ingress *networking.Ingress, serviceInfo string) error { + + if ingress.Spec.DefaultBackend != nil { + serviceName := ingress.Spec.DefaultBackend.Service.Name + servicePort := ingress.Spec.DefaultBackend.Service.Port.Number + + err := NewEchoDeployment(kubeClientSet, ingress.Namespace, ingress.Name, serviceName, "", servicePort) + if err != nil { + return err + } + } + + if serviceInfo != "" { + serviceNameList := strings.Split(serviceInfo, "|") + for i := 0; i < len(serviceNameList); i++ { + ipPort := strings.Split(serviceNameList[i], ":") + if len(ipPort) < 2 { + return fmt.Errorf("error ip port for service info") + } + serviceName := ipPort[0] + servicePort, err := strconv.ParseInt(ipPort[1], 10, 32) + if err != nil { + return err + } + err = NewEchoDeployment(kubeClientSet, ingress.Namespace, ingress.Name, serviceName, "", int32(servicePort)) + if err != nil { + return err + } + } + } else { + for _, rule := range ingress.Spec.Rules { + if rule.HTTP == nil { + continue + } + + for _, path := range rule.HTTP.Paths { + serviceName := path.Backend.Service.Name + servicePort := path.Backend.Service.Port.Number + + err := NewEchoDeployment(kubeClientSet, ingress.Namespace, ingress.Name, serviceName, "", servicePort) + if err != nil { + return err + } + } + } + } + + return nil +} + +// ScaleIngressBackendDeployment changes the replicas count of a deployment defined in an ingress service backend +func ScaleIngressBackendDeployment(kubeClientSet kubernetes.Interface, namespace, name, serviceName string, replicas int) error { + deploymentName := fmt.Sprintf("%v-%v", name, serviceName) + + scale := &autoscalingv1.Scale{ + ObjectMeta: metav1.ObjectMeta{ + Name: deploymentName, + Namespace: namespace, + }, + Spec: autoscalingv1.ScaleSpec{ + Replicas: int32(replicas), + }, + } + + _, err := kubeClientSet.AppsV1().Deployments(namespace).UpdateScale(context.TODO(), deploymentName, scale, metav1.UpdateOptions{}) + if err != nil { + return err + } + + err = waitForEndpoints(kubeClientSet, WaitForEndpointsTimeout, namespace, serviceName, replicas) + if err != nil { + return fmt.Errorf("waiting for service (%v) endpoints available: %w", serviceName, err) + } + + time.Sleep(60 * time.Second) + + return nil +} + +// deploymentFromManifest deserializes a Deployment definition from a yaml string +func deploymentFromManifest(manifest string) (*appsv1.Deployment, error) { + deployment := &appsv1.Deployment{} + if err := yaml.Unmarshal([]byte(manifest), &deployment); err != nil { + return nil, fmt.Errorf("deserializing deployment from manifest: %w\n%v", err, manifest) + } + + return deployment, nil +} + +// serviceFromManifest deserializes a Deployment definition from a yaml string +func serviceFromManifest(manifest string) (*corev1.Service, error) { + deployment := &corev1.Service{} + if err := yaml.Unmarshal([]byte(manifest), &deployment); err != nil { + return nil, fmt.Errorf("deserializing service from manifest: %w", err) + } + + return deployment, nil +} + +// waitForEndpoints waits for a given amount of time until the number of endpoints = expectedEndpoints. +func waitForEndpoints(kubeClientSet kubernetes.Interface, timeout time.Duration, ns, name string, expectedEndpoints int) error { + if expectedEndpoints == 0 { + return nil + } + + return wait.Poll(5*time.Second, timeout, func() (bool, error) { + endpoint, err := kubeClientSet.CoreV1().Endpoints(ns).Get(context.TODO(), name, metav1.GetOptions{}) + if apierrors.IsNotFound(err) { + return false, nil + } + + if countReadyEndpoints(endpoint) == expectedEndpoints { + return true, nil + } + + return false, nil + }) +} + +func countReadyEndpoints(e *corev1.Endpoints) int { + if e == nil || e.Subsets == nil { + return 0 + } + + num := 0 + for _, sub := range e.Subsets { + num += len(sub.Addresses) + } + + return num +} diff --git a/test/e2e/pkg/kubernetes/kubernetes.go b/test/e2e/pkg/kubernetes/kubernetes.go new file mode 100644 index 00000000..3b9b737a --- /dev/null +++ b/test/e2e/pkg/kubernetes/kubernetes.go @@ -0,0 +1,521 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kubernetes + +import ( + "bytes" + "context" + "crypto/rand" + "crypto/rsa" + "crypto/x509" + "crypto/x509/pkix" + "encoding/json" + "encoding/pem" + "fmt" + "io" + "math/big" + "net" + "os" + "path/filepath" + "runtime" + "strings" + "time" + + corev1 "k8s.io/api/core/v1" + networking "k8s.io/api/networking/v1" + apierrs "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + apiruntime "k8s.io/apimachinery/pkg/runtime" + utilnet "k8s.io/apimachinery/pkg/util/net" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/kubernetes" + clientset "k8s.io/client-go/kubernetes" + restclient "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" + "k8s.io/client-go/util/flowcontrol" + "sigs.k8s.io/yaml" + + // ensure auth plugins are loaded + _ "k8s.io/client-go/plugin/pkg/client/auth" +) + +// IngressClassValue sets the value of the class of Ingresses +var IngressClassValue string + +// K8sNodeAddr sets ip addr of a k8s node +var K8sNodeAddr string + +// KubeClient Kubernetes API client +var KubeClient *kubernetes.Clientset + +var ( + BfeAnnotationPrefix = "bfe.ingress.kubernetes.io/" + StatusAnnotationKey = fmt.Sprintf("%s%s", BfeAnnotationPrefix, "bfe-ingress-status") +) + +type StatusMsg struct { + Status string `json:"status"` + Message string `json:"message"` +} + +// LoadClientset returns clientset for connecting to kubernetes clusters. +func LoadClientset() (*clientset.Clientset, error) { + config, err := restclient.InClusterConfig() + if err != nil { + // Attempt to use local KUBECONFIG + loadingRules := clientcmd.NewDefaultClientConfigLoadingRules() + kubeconfig := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(loadingRules, &clientcmd.ConfigOverrides{}) + // use the current context in kubeconfig + var err error + + config, err = kubeconfig.ClientConfig() + if err != nil { + return nil, err + } + } + + config.RateLimiter = flowcontrol.NewTokenBucketRateLimiter(100, 100) + + // TODO: add version information? + config.UserAgent = fmt.Sprintf( + "%s (%s/%s) ingress-conformance", + filepath.Base(os.Args[0]), + runtime.GOOS, + runtime.GOARCH, + ) + + client, err := clientset.NewForConfig(config) + if err != nil { + return nil, err + } + + return client, nil +} + +// NewNamespace creates a new namespace using ingress-conformance- as prefix. +func NewNamespace(c kubernetes.Interface) (string, error) { + nsParam := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "ingress-conformance-", + Labels: map[string]string{ + "app.kubernetes.io/name": "ingress-conformance", + }, + }, + } + + var err error + + err = displayYamlDefinition(nsParam) + if err != nil { + return "", fmt.Errorf("unable show yaml definition: %v", err) + } + + ns, err := c.CoreV1().Namespaces().Create(context.TODO(), nsParam, metav1.CreateOptions{}) + if err != nil { + return "", fmt.Errorf("unable to create namespace: %v", err) + } + + return ns.Name, nil +} + +// DeleteNamespace deletes a namespace and all the objects inside +func DeleteNamespaceBlocking(c kubernetes.Interface, namespace string) error { + grace := int64(0) + pb := metav1.DeletePropagationForeground + + if err := c.CoreV1().Namespaces().Delete(context.TODO(), namespace, metav1.DeleteOptions{ + GracePeriodSeconds: &grace, + PropagationPolicy: &pb, + }); err != nil { + return err + } + + return waitForNamespaceDeleted(c, namespace) +} + +// DeleteNamespace deletes a namespace and all the objects inside +func DeleteNamespace(c kubernetes.Interface, namespace string) error { + grace := int64(0) + pb := metav1.DeletePropagationBackground + + return c.CoreV1().Namespaces().Delete(context.TODO(), namespace, metav1.DeleteOptions{ + GracePeriodSeconds: &grace, + PropagationPolicy: &pb, + }) +} + +// WaitForIngressAddress waits for the Ingress to acquire an address. +func waitForNamespaceDeleted(c clientset.Interface, namespace string) error { + err := wait.PollImmediate(ingressWaitInterval, WaitForIngressAddressTimeout, func() (bool, error) { + _, err := c.CoreV1().Namespaces().Get(context.TODO(), namespace, metav1.GetOptions{}) + if apierrs.IsNotFound(err) { + return true, nil + } + return false, nil + }) + + return err +} + +// CleanupNamespaces removes namespaces created by conformance tests +func CleanupNamespaces(c kubernetes.Interface) error { + namespaces, err := c.CoreV1().Namespaces().List(context.TODO(), metav1.ListOptions{ + LabelSelector: "app.kubernetes.io/name=ingress-conformance", + }) + + if err != nil { + return err + } + + for _, namespace := range namespaces.Items { + err := DeleteNamespace(c, namespace.Name) + if err != nil { + return err + } + } + + return nil +} + +// NewIngress creates a new ingress +func NewIngress(c kubernetes.Interface, namespace string, ingress *networking.Ingress) error { + err := displayYamlDefinition(ingress) + if err != nil { + return fmt.Errorf("unable show yaml definition: %v", err) + } + + if _, err := c.NetworkingV1().Ingresses(namespace).Create(context.TODO(), ingress, metav1.CreateOptions{}); err != nil { + return err + } + + return nil +} + +// IngressFromSpec deserializes an Ingress definition using an IngressSpec +func IngressFromSpec(name, namespace, ingressSpec string) (*networking.Ingress, error) { + if namespace == metav1.NamespaceNone || namespace == metav1.NamespaceDefault { + return nil, fmt.Errorf("ingress definitions in the default namespace are not allowed (%v)", namespace) + } + + ingress := &networking.Ingress{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + } + + if err := yaml.Unmarshal([]byte(ingressSpec), &ingress.Spec); err != nil { + return nil, fmt.Errorf("deserializing Ingress from spec: %w", err) + } + + if ingress.Spec.IngressClassName == nil { + ingress.Spec.IngressClassName = &IngressClassValue + } + + return ingress, nil +} + +// IngressFromManifest deserializes an Ingress definition using an Ingress +func IngressFromManifest(namespace, manifest string) (*networking.Ingress, error) { + if namespace == metav1.NamespaceNone || namespace == metav1.NamespaceDefault { + return nil, fmt.Errorf("Ingress definitions in the default namespace are not allowed (%v)", namespace) + } + + ingress := &networking.Ingress{} + if err := yaml.Unmarshal([]byte(manifest), &ingress); err != nil { + return nil, fmt.Errorf("deserializing Ingress from manifest: %w", err) + } + + ingress.SetNamespace(namespace) + + if ingress.Spec.IngressClassName == nil { + ingress.Spec.IngressClassName = &IngressClassValue + } + + return ingress, nil +} + +// NewSelfSignedSecret creates a self signed SSL certificate and store it in a secret +func NewSelfSignedSecret(c clientset.Interface, namespace, secretName string, hosts []string) error { + if len(hosts) == 0 { + return fmt.Errorf("require a non-empty hosts for Subject Alternate Name values") + } + + var serverKey, serverCert bytes.Buffer + + host := strings.Join(hosts, ",") + + if err := generateRSACert(host, &serverKey, &serverCert); err != nil { + return err + } + + data := map[string][]byte{ + corev1.TLSCertKey: serverCert.Bytes(), + corev1.TLSPrivateKeyKey: serverKey.Bytes(), + } + + newSecret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: secretName, + }, + Type: corev1.SecretTypeTLS, + Data: data, + } + + err := displayYamlDefinition(newSecret) + if err != nil { + return fmt.Errorf("unable show yaml definition: %v", err) + } + + if _, err := c.CoreV1().Secrets(namespace).Create(context.TODO(), newSecret, metav1.CreateOptions{}); err != nil { + return err + } + + return nil +} + +const ( + // ingressWaitInterval time to wait between checks for a condition + ingressWaitInterval = 3 * time.Second +) + +var ( + // WaitForIngressAddressTimeout maximum wait time for valid ingress status value + WaitForIngressAddressTimeout = 3 * time.Minute + // WaitForEndpointsTimeout maximum wait time for ready endpoints + WaitForEndpointsTimeout = 3 * time.Minute + + // EnableOutputYamlDefinitions display yaml definitions of Kubernetes objects before creation + EnableOutputYamlDefinitions = false + + // IngressControllerNameSpace the namespace of the ingress controller + IngressControllerNameSpace = "ingress-bfe" + // IngressControllerServiceName the service name of the ingress controller + IngressControllerServiceName = "bfe-controller-service" +) + +// WaitForIngressAddress waits for the Ingress to acquire an address. +func WaitForIngressAddress(c clientset.Interface, namespace, name string) (map[string]string, error) { + var address map[string]string + err := wait.PollImmediate(ingressWaitInterval, WaitForIngressAddressTimeout, func() (bool, error) { + ipOrNameList, err := getIngressAddressOfNodePort(c, namespace, name) + if err != nil || len(ipOrNameList) == 0 { + if isRetryableAPIError(err) { + return false, nil + } + + return false, err + } + address = ipOrNameList + return true, nil + }) + + if err != nil { + return nil, fmt.Errorf("waiting for ingress status update: %w", err) + } + + return address, nil +} + +// getIngressAddress returns the ips/hostnames associated with the Ingress. +func getIngressAddress(c clientset.Interface, ns, name string) ([]string, error) { + ing, err := c.NetworkingV1().Ingresses(ns).Get(context.TODO(), name, metav1.GetOptions{}) + if err != nil { + return nil, err + } + + var addresses []string + + for _, a := range ing.Status.LoadBalancer.Ingress { + if a.IP != "" { + addresses = append(addresses, a.IP) + } + + if a.Hostname != "" { + addresses = append(addresses, a.Hostname) + } + } + + return addresses, nil +} + +// getBfeIngressAddressByNodePort returns the service port name and port associated with the Ingress. +func getIngressAddressOfNodePort(c clientset.Interface, ns, name string) (map[string]string, error) { + ing, err := c.NetworkingV1().Ingresses(ns).Get(context.TODO(), name, metav1.GetOptions{}) + if err != nil { + return nil, err + } + + var statusMsg StatusMsg + + addresses := make(map[string]string) + + if ing.Annotations != nil { + if statusMessage, ok := ing.Annotations[StatusAnnotationKey]; ok && statusMessage != "" { + statusMessage := ing.Annotations[StatusAnnotationKey] + err := json.Unmarshal([]byte(statusMessage), &statusMsg) + if err != nil { + return nil, err + } + if statusMsg.Status == "success" { + service, err := c.CoreV1().Services(IngressControllerNameSpace).Get(context.TODO(), IngressControllerServiceName, metav1.GetOptions{}) + if err != nil { + return nil, err + } + ports := service.Spec.Ports + for i := 0; i < len(ports); i++ { + addresses[ports[i].Name] = fmt.Sprintf("%s:%d", K8sNodeAddr, ports[i].NodePort) + } + } else if statusMsg.Status == "error" { + return nil, fmt.Errorf(statusMsg.Message) + } + } + } + return addresses, nil +} + +func getIngressAddressOfClusterIp(c clientset.Interface, ns, name string) (map[string]string, error) { + ing, err := c.NetworkingV1().Ingresses(ns).Get(context.TODO(), name, metav1.GetOptions{}) + if err != nil { + return nil, err + } + + var statusMsg StatusMsg + + addresses := make(map[string]string) + + if ing.Annotations != nil { + if statusMessage, ok := ing.Annotations[StatusAnnotationKey]; ok && statusMessage != "" { + err := json.Unmarshal([]byte(statusMessage), &statusMsg) + if err != nil { + return nil, err + } + if statusMsg.Status == "success" { + service, err := c.CoreV1().Services(IngressControllerNameSpace).Get(context.TODO(), "bfe-controller-service", metav1.GetOptions{}) + if err != nil { + fmt.Println("error: ", err.Error()) + return nil, err + } + ports := service.Spec.Ports + for i := 0; i < len(ports); i++ { + addresses[ports[i].Name] = fmt.Sprintf("%s:%d", service.Spec.ClusterIP, service.Spec.Ports[i].Port) + } + } else { + fmt.Printf("statusMessage: %s", statusMessage) + return nil, fmt.Errorf("ingress status is not success") + } + } + } + fmt.Println("address", addresses) + return addresses, nil +} + +// isRetryableAPIError checks if an API error allows retries or not +func isRetryableAPIError(err error) bool { + // These errors may indicate a transient error that we can retry in tests. + if apierrs.IsInternalError(err) || apierrs.IsTimeout(err) || apierrs.IsServerTimeout(err) || + apierrs.IsTooManyRequests(err) || utilnet.IsProbableEOF(err) || utilnet.IsConnectionReset(err) { + return true + } + + // If the error sends the Retry-After header, we respect it as an explicit confirmation we should retry. + if _, shouldRetry := apierrs.SuggestsClientDelay(err); shouldRetry { + return true + } + + // in case backend start slowly + if err != nil && (strings.Contains(err.Error(), "no avail backend") || strings.Contains(err.Error(), "conflict with")) { + return true + } + + return false +} + +const ( + rsaBits = 2048 + validFor = 365 * 24 * time.Hour +) + +// generateRSACert generates a basic self signed certificate valir for a year +func generateRSACert(host string, keyOut, certOut io.Writer) error { + priv, err := rsa.GenerateKey(rand.Reader, rsaBits) + if err != nil { + return fmt.Errorf("failed to generate key: %v", err) + } + notBefore := time.Now() + notAfter := notBefore.Add(validFor) + + serialNumberLimit := new(big.Int).Lsh(big.NewInt(1), 128) + serialNumber, err := rand.Int(rand.Reader, serialNumberLimit) + + if err != nil { + return fmt.Errorf("failed to generate serial number: %s", err) + } + + template := x509.Certificate{ + SerialNumber: serialNumber, + Subject: pkix.Name{ + CommonName: "default", + Organization: []string{"Acme Co"}, + }, + NotBefore: notBefore, + NotAfter: notAfter, + + KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + BasicConstraintsValid: true, + } + + hosts := strings.Split(host, ",") + for _, h := range hosts { + if ip := net.ParseIP(h); ip != nil { + template.IPAddresses = append(template.IPAddresses, ip) + } else { + template.DNSNames = append(template.DNSNames, h) + } + } + + derBytes, err := x509.CreateCertificate(rand.Reader, &template, &template, &priv.PublicKey, priv) + + if err != nil { + return fmt.Errorf("failed to create certificate: %s", err) + } + + if err := pem.Encode(certOut, &pem.Block{Type: "CERTIFICATE", Bytes: derBytes}); err != nil { + return fmt.Errorf("failed creating cert: %v", err) + } + + if err := pem.Encode(keyOut, &pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(priv)}); err != nil { + return fmt.Errorf("failed creating key: %v", err) + } + + return nil +} + +func displayYamlDefinition(obj apiruntime.Object) error { + if !EnableOutputYamlDefinitions { + return nil + } + + output, err := yaml.Marshal(obj) + if err != nil { + return err + } + + _, err = fmt.Fprint(os.Stdout, fmt.Sprintf("---\n%s\n", output)) + return err +} diff --git a/test/e2e/pkg/kubernetes/templates/templates.go b/test/e2e/pkg/kubernetes/templates/templates.go new file mode 100644 index 00000000..a062e0c9 --- /dev/null +++ b/test/e2e/pkg/kubernetes/templates/templates.go @@ -0,0 +1,128 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package templates + +import ( + "bytes" + "fmt" + text_template "text/template" +) + +var k8sTemplates = map[string]string{ + "deployment": ` +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ .Name }} +spec: + replicas: 1 + strategy: + type: RollingUpdate + selector: + matchLabels: + app: {{ .MatchLabels }} + template: + metadata: + labels: + app: {{ .Labels }} + spec: + containers: + - name: ingress-conformance-echo + image: {{ .Image }} + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: INGRESS_NAME + value: {{ .Ingress }} + - name: SERVICE_NAME + value: {{ .Service }} + ports: + - name: {{ .PortName }} + containerPort: 3000 + livenessProbe: + httpGet: + path: /health + port: 3000 + scheme: HTTP + initialDelaySeconds: 1 + periodSeconds: 1 + timeoutSeconds: 1 + successThreshold: 1 + failureThreshold: 10 + readinessProbe: + httpGet: + path: /health + port: 3000 + scheme: HTTP + initialDelaySeconds: 1 + periodSeconds: 1 + timeoutSeconds: 1 + successThreshold: 1 + failureThreshold: 10 +`, + "service": ` +apiVersion: v1 +kind: Service +metadata: + name: {{ .Name }} +spec: + type: NodePort + selector: + app: {{ .Selector }} + ports: + - port: {{ .Port }} + targetPort: 3000 +`, +} + +var templates = map[string]*text_template.Template{} + +// Load parses templates required to deploy Kubernetes objects +func Load() error { + for name, template := range k8sTemplates { + tmpl, err := text_template.New(name).Parse(template) + if err != nil { + return err + } + + templates[name] = tmpl + } + + return nil +} + +// Render executes a parsed template to the specified data object +func Render(name string, data interface{}) (string, error) { + tmpl, ok := templates[name] + if !ok { + return "", fmt.Errorf("there is no template with name %v", name) + } + + var tpl bytes.Buffer + err := tmpl.Execute(&tpl, data) + if err != nil { + return "", err + } + + return tpl.String(), nil +} diff --git a/test/e2e/pkg/state/state.go b/test/e2e/pkg/state/state.go new file mode 100644 index 00000000..4ab34020 --- /dev/null +++ b/test/e2e/pkg/state/state.go @@ -0,0 +1,189 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package state + +import ( + "fmt" + net_http "net/http" + "strings" + + "github.com/bfenetworks/ingress-bfe/test/e2e/pkg/http" +) + +// Scenario holds state for a test scenario +type Scenario struct { + Namespace string + IngressName string + + SecretName string + + CapturedRequest *http.CapturedRequest + CapturedResponse *http.CapturedResponse + + IPOrFQDN map[string]string + NodeIPPort map[string]int32 +} + +// New creates a new state to use in a test Scenario +func New() *Scenario { + return &Scenario{} +} + +// CaptureRoundTrip will perform an HTTP request and return the CapturedRequest and CapturedResponse tuple +func (s *Scenario) CaptureRoundTrip(method, scheme, hostname, path string, headerInfo net_http.Header) error { + var location string + + // scheme == http or https + location, ok := s.IPOrFQDN[scheme] + if !ok { + return fmt.Errorf("scheme is not found in addr: %+v", s.IPOrFQDN) + } + + capturedRequest, capturedResponse, err := http.CaptureRoundTrip(method, scheme, hostname, path, location, headerInfo) + if err != nil { + return err + } + + s.CapturedRequest = capturedRequest + s.CapturedResponse = capturedResponse + + return nil +} + +// AssertStatusCode returns an error if the captured response status code does not match the expected value +func (s *Scenario) AssertStatusCode(statusCode int) error { + if s.CapturedResponse.StatusCode != statusCode { + return fmt.Errorf("expected status code %v but %v was returned", statusCode, s.CapturedResponse.StatusCode) + } + + return nil +} + +// AssertServedBy returns an error if the captured request was not served by the expected service +func (s *Scenario) AssertServedBy(service string) error { + if s.CapturedRequest.Service != service { + return fmt.Errorf("expected the request to be served by %v but it was served by %v", service, s.CapturedRequest.Service) + } + + return nil +} + +// AssertRequestHost returns an error if the captured request host does not match the expected value +func (s *Scenario) AssertRequestHost(host string) error { + if s.CapturedRequest.Host != host { + return fmt.Errorf("expected the request host to be %v but was %v", host, s.CapturedRequest.Host) + } + + return nil +} + +// AssertTLSHostname returns an error if the captured TLS response hostname does not match the expected value +func (s *Scenario) AssertTLSHostname(hostname string) error { + if s.CapturedResponse.TLSHostname != hostname { + return fmt.Errorf("expected the response TLS hostname to be %v but was %v", hostname, s.CapturedResponse.TLSHostname) + } + + return nil +} + +// AssertResponseProto returns an error if the captured response proto does not match the expected value +func (s *Scenario) AssertResponseProto(proto string) error { + if s.CapturedResponse.Proto != proto { + return fmt.Errorf("expected the response protocol to be %v but it was %v", proto, s.CapturedResponse.Proto) + } + + return nil +} + +// AssertRequestProto returns an error if the captured request proto does not match the expected value +func (s *Scenario) AssertRequestProto(proto string) error { + if s.CapturedRequest.Proto != proto { + return fmt.Errorf("expected the request protocol to be %v but it was %v", proto, s.CapturedRequest.Proto) + } + + return nil +} + +// AssertMethod returns an error if the captured request method does not match the expected value +func (s *Scenario) AssertMethod(method string) error { + if s.CapturedRequest.Method != method { + return fmt.Errorf("expected the request method to be %v but it was %v", method, s.CapturedRequest.Method) + } + + return nil +} + +// AssertRequestPath returns an error if the captured request path does not match the expected value +func (s *Scenario) AssertRequestPath(path string) error { + if !strings.HasPrefix(path, "/") { + path = fmt.Sprintf("/%s", path) + } + + if s.CapturedRequest.Path != path { + return fmt.Errorf("expected the request path to be %v but it was %v", path, s.CapturedRequest.Path) + } + + return nil +} + +// AssertResponseHeader returns an error if the captured response headers do not contain the expected headerKey, +// or if the matching response header value does not match the expected headerValue. +// If the headerValue string equals `*`, the header value check is ignored. +func (s *Scenario) AssertResponseHeader(headerKey string, headerValue string) error { + if headerValues := s.CapturedResponse.Headers[headerKey]; headerValues == nil { + return fmt.Errorf("expected response headers to contain %v but it only contained %v", headerKey, s.CapturedResponse.Headers) + } else if headerValue != "*" { + for _, value := range headerValues { + if value == headerValue { + return nil + } + } + + return fmt.Errorf("expected response headers %v to contain a %v value but it contained %v", headerKey, headerValue, headerValues) + } + + return nil +} + +// AssertRequestHeader returns an error if the captured request headers do not contain the expected headerKey, +// or if the matching request header value does not match the expected headerValue. +// If the headerValue string equals `*`, the header value check is ignored. +func (s *Scenario) AssertRequestHeader(headerKey string, headerValue string) error { + if headerValues := s.CapturedRequest.Headers[headerKey]; headerValues == nil { + return fmt.Errorf("expected request headers to contain %v but it only contained %v", headerKey, s.CapturedRequest.Headers) + } else if headerValue != "*" { + for _, value := range headerValues { + if value == headerValue { + return nil + } + } + + return fmt.Errorf("expected request headers %v to contain a %v value but it contained %v", headerKey, headerValue, headerValues) + } + + return nil +} + +// AssertResponseCertificate returns nil if the captured certificate for the named host is valid. +// Otherwise it returns an error describing the mismatch. +func (s *Scenario) AssertResponseCertificate(hostname string) error { + if s.CapturedResponse == nil || s.CapturedResponse.Certificate == nil { + return fmt.Errorf("hostname verification requires executing a request and also target an HTTPS URL") + } + + return s.CapturedResponse.Certificate.VerifyHostname(hostname) +} diff --git a/test/e2e/run.sh b/test/e2e/run.sh new file mode 100755 index 00000000..06e93050 --- /dev/null +++ b/test/e2e/run.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash + +# Copyright 2020 The BFE Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -ex + +trap TERM + +cd "$(dirname "$0")" + +RESULTS_DIR="${RESULTS_DIR:-$PWD/result}" +if [[ -d $RESULTS_DIR ]]; then + rm -rf $RESULTS_DIR/* +else + mkdir $RESULTS_DIR +fi + +make build + +CUCUMBER_OUTPUT_FORMAT="${CUCUMBER_OUTPUT_FORMAT:-pretty}" +WAIT_FOR_STATUS_TIMEOUT="${WAIT_FOR_STATUS_TIMEOUT:-5m}" +TEST_TIMEOUT="${TEST_TIMEOUT:-0}" +TEST_PARALLEL="${TEST_PARALLEL:-5}" + +./e2e_test \ + --output-directory="${RESULTS_DIR}" \ + --feature="${CUCUMBER_FEATURE}" \ + --format="${CUCUMBER_OUTPUT_FORMAT}" \ + --wait-time-for-ingress-status="${WAIT_FOR_STATUS_TIMEOUT}" \ + --wait-time-for-ready="${WAIT_FOR_STATUS_TIMEOUT}" \ + --test.timeout="${TEST_TIMEOUT}" \ + --feature-parallel="${TEST_PARALLEL}" +ret=$? + +exit 0 diff --git a/test/e2e/steps/annotations/balance/loadbalance/steps.go b/test/e2e/steps/annotations/balance/loadbalance/steps.go new file mode 100644 index 00000000..923f731b --- /dev/null +++ b/test/e2e/steps/annotations/balance/loadbalance/steps.go @@ -0,0 +1,158 @@ +/* +Copyright 2021 The BFE Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package loadbalance + +import ( + "fmt" + "net/url" + "strings" + "time" + + "github.com/cucumber/godog" + "github.com/cucumber/messages-go/v16" + "k8s.io/apimachinery/pkg/util/sets" + + "github.com/bfenetworks/ingress-bfe/test/e2e/pkg/kubernetes" + tstate "github.com/bfenetworks/ingress-bfe/test/e2e/pkg/state" +) + +var ( + state *tstate.Scenario + + resultStatus map[int]sets.String + resultService sets.String +) + +// IMPORTANT: Steps definitions are generated and should not be modified +// by hand but rather through make codegen. DO NOT EDIT. + +// InitializeScenario configures the Feature to test +func InitializeScenario(ctx *godog.ScenarioContext) { + ctx.Step(`^an Ingress with service info "([^"]*)" resource in a new random namespace$`, anIngressWithServiceInfoResourceInANewRandomNamespace) + ctx.Step(`^The Ingress status shows the IP address or FQDN where it is exposed$`, theIngressStatusShowsTheIPAddressOrFQDNWhereItIsExposed) + ctx.Step(`^I send (\d+) "([^"]*)" requests to "([^"]*)"$`, iSendRequestsTo) + ctx.Step(`^the response status-code must be (\d+) the response body should contain the IP address of (\d+) different Kubernetes pods$`, theResponseStatuscodeMustBeTheResponseBodyShouldContainTheIPAddressOfDifferentKubernetesPods) + ctx.Step(`^the response must be served by one of "([^"]*)" service$`, theResponseMustBeServedByOneOfService) + ctx.Step(`^The Ingress status should not be success$`, theIngressStatusShouldNotBeSuccess) + + ctx.BeforeScenario(func(*godog.Scenario) { + state = tstate.New() + resultStatus = make(map[int]sets.String, 0) + resultService = make(sets.String) + }) + + ctx.AfterScenario(func(*messages.Pickle, error) { + // delete namespace an all the content + _ = kubernetes.DeleteNamespace(kubernetes.KubeClient, state.Namespace) + }) +} + +func anIngressWithServiceInfoResourceInANewRandomNamespace(serviceInfo string, spec *godog.DocString) error { + ns, err := kubernetes.NewNamespace(kubernetes.KubeClient) + if err != nil { + return err + } + + state.Namespace = ns + + ingress, err := kubernetes.IngressFromManifest(state.Namespace, spec.Content) + if err != nil { + return err + } + + err = kubernetes.DeploymentsFromIngressForBalance(kubernetes.KubeClient, ingress, serviceInfo) + if err != nil { + return err + } + + err = kubernetes.NewIngress(kubernetes.KubeClient, state.Namespace, ingress) + if err != nil { + return err + } + + state.IngressName = ingress.GetName() + + return nil +} + +func theIngressStatusShowsTheIPAddressOrFQDNWhereItIsExposed() error { + ingress, err := kubernetes.WaitForIngressAddress(kubernetes.KubeClient, state.Namespace, state.IngressName) + if err != nil { + return err + } + + state.IPOrFQDN = ingress + + time.Sleep(3 * time.Second) + + return err +} + +func iSendRequestsTo(totalRequest int, method string, rawURL string) error { + u, err := url.Parse(rawURL) + if err != nil { + return err + } + + for iteration := 1; iteration <= totalRequest; iteration++ { + err := state.CaptureRoundTrip(method, u.Scheme, u.Host, u.Path, nil) + if err != nil { + return err + } + + if resultStatus[state.CapturedResponse.StatusCode] == nil { + resultStatus[state.CapturedResponse.StatusCode] = sets.NewString() + } + + resultStatus[state.CapturedResponse.StatusCode].Insert(state.CapturedRequest.Pod) + resultService.Insert(state.CapturedRequest.Service) + } + + return nil +} + +func theResponseStatuscodeMustBeTheResponseBodyShouldContainTheIPAddressOfDifferentKubernetesPods(statusCode int, pods int) error { + results, ok := resultStatus[statusCode] + if !ok { + return fmt.Errorf("no reponses for status code %v returned", statusCode) + } + + if results.Len() != pods { + return fmt.Errorf("expected %v different POD IP addresses/FQDN for status code %v but %v was returned", pods, statusCode, results.Len()) + } + + return nil +} + +func theResponseMustBeServedByOneOfService(serviceInfo string) error { + serviceList := strings.Split(serviceInfo, "|") + for i := 0; i < len(serviceList); i++ { + if !resultService.Has(serviceList[i]) { + return fmt.Errorf("service info %s not exist in request info", serviceList[i]) + } + } + return nil +} + +func theIngressStatusShouldNotBeSuccess() error { + _, err := kubernetes.WaitForIngressAddress(kubernetes.KubeClient, state.Namespace, state.IngressName) + if err == nil { + return fmt.Errorf("create ingress should return error") + } + + return nil +} diff --git a/test/e2e/steps/annotations/route/cookie/steps.go b/test/e2e/steps/annotations/route/cookie/steps.go new file mode 100644 index 00000000..e4c2ace0 --- /dev/null +++ b/test/e2e/steps/annotations/route/cookie/steps.go @@ -0,0 +1,131 @@ +/* +Copyright 2021 The BFE Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cookie + +import ( + "encoding/json" + "fmt" + "net/http" + "net/url" + "time" + + "github.com/cucumber/godog" + "github.com/cucumber/messages-go/v16" + "k8s.io/apimachinery/pkg/util/sets" + + "github.com/bfenetworks/ingress-bfe/test/e2e/pkg/kubernetes" + tstate "github.com/bfenetworks/ingress-bfe/test/e2e/pkg/state" +) + +var ( + state *tstate.Scenario + + resultStatus map[int]sets.String +) + +// IMPORTANT: Steps definitions are generated and should not be modified +// by hand but rather through make codegen. DO NOT EDIT. + +// InitializeScenario configures the Feature to test +func InitializeScenario(ctx *godog.ScenarioContext) { + ctx.Step(`^an Ingress resource in a new random namespace$`, anIngressResourceInANewRandomNamespace) + ctx.Step(`^The Ingress status shows the IP address or FQDN where it is exposed$`, theIngressStatusShowsTheIPAddressOrFQDNWhereItIsExposed) + ctx.Step(`^I send a "([^"]*)" request to "([^"]*)" with header$`, iSendARequestToWithHeader) + ctx.Step(`^the response status-code must be (\d+)$`, theResponseStatuscodeMustBe) + ctx.Step(`^the response must be served by the "([^"]*)" service$`, theResponseMustBeServedByTheService) + ctx.Step(`^I send a "([^"]*)" request to "([^"]*)"$`, iSendARequestTo) + + ctx.BeforeScenario(func(*godog.Scenario) { + state = tstate.New() + }) + + ctx.AfterScenario(func(*messages.Pickle, error) { + // delete namespace an all the content + _ = kubernetes.DeleteNamespace(kubernetes.KubeClient, state.Namespace) + }) +} + +func anIngressResourceInANewRandomNamespace(spec *godog.DocString) error { + ns, err := kubernetes.NewNamespace(kubernetes.KubeClient) + if err != nil { + return err + } + + state.Namespace = ns + + ingress, err := kubernetes.IngressFromManifest(state.Namespace, spec.Content) + if err != nil { + return err + } + + err = kubernetes.DeploymentsFromIngress(kubernetes.KubeClient, ingress) + if err != nil { + return err + } + + err = kubernetes.NewIngress(kubernetes.KubeClient, state.Namespace, ingress) + if err != nil { + return err + } + + state.IngressName = ingress.GetName() + + return nil +} + +func theIngressStatusShowsTheIPAddressOrFQDNWhereItIsExposed() error { + ingress, err := kubernetes.WaitForIngressAddress(kubernetes.KubeClient, state.Namespace, state.IngressName) + if err != nil { + return err + } + + state.IPOrFQDN = ingress + + time.Sleep(3 * time.Second) + + return err +} + +func iSendARequestToWithHeader(method, rawURL string, header *godog.DocString) error { + var headerInfo http.Header + if header.Content != "" { + if err := json.Unmarshal([]byte(header.Content), &headerInfo); err != nil { + return fmt.Errorf("err in jsonEncoder.Encode: ", err.Error()) + } + } + u, err := url.Parse(rawURL) + if err != nil { + return err + } + return state.CaptureRoundTrip(method, u.Scheme, u.Host, u.Path, headerInfo) +} + +func theResponseStatuscodeMustBe(statusCode int) error { + return state.AssertStatusCode(statusCode) +} + +func theResponseMustBeServedByTheService(service string) error { + return state.AssertServedBy(service) +} + +func iSendARequestTo(method string, rawURL string) error { + u, err := url.Parse(rawURL) + if err != nil { + return err + } + return state.CaptureRoundTrip(method, u.Scheme, u.Host, u.Path, nil) +} diff --git a/test/e2e/steps/annotations/route/header/steps.go b/test/e2e/steps/annotations/route/header/steps.go new file mode 100644 index 00000000..55aaa608 --- /dev/null +++ b/test/e2e/steps/annotations/route/header/steps.go @@ -0,0 +1,142 @@ +/* +Copyright 2021 The BFE Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package header + +import ( + "encoding/json" + "fmt" + "net/http" + "net/url" + "time" + + "github.com/cucumber/godog" + "github.com/cucumber/messages-go/v16" + "k8s.io/apimachinery/pkg/util/sets" + + "github.com/bfenetworks/ingress-bfe/test/e2e/pkg/kubernetes" + tstate "github.com/bfenetworks/ingress-bfe/test/e2e/pkg/state" +) + +var ( + state *tstate.Scenario + + resultStatus map[int]sets.String +) + +// IMPORTANT: Steps definitions are generated and should not be modified +// by hand but rather through make codegen. DO NOT EDIT. + +// InitializeScenario configures the Feature to test +func InitializeScenario(ctx *godog.ScenarioContext) { + ctx.Step(`^an Ingress resource in a new random namespace$`, anIngressResourceInANewRandomNamespace) + ctx.Step(`^The Ingress status shows the IP address or FQDN where it is exposed$`, theIngressStatusShowsTheIPAddressOrFQDNWhereItIsExposed) + ctx.Step(`^I send a "([^"]*)" request to "([^"]*)" with header$`, iSendARequestToWithHeader) + ctx.Step(`^the response status-code must be (\d+)$`, theResponseStatuscodeMustBe) + ctx.Step(`^the response must be served by the "([^"]*)" service$`, theResponseMustBeServedByTheService) + ctx.Step(`^I send a "([^"]*)" request to "([^"]*)"$`, iSendARequestTo) + ctx.Step(`^The Ingress status should not be success$`, theIngressStatusShouldNotBeSuccess) + + ctx.BeforeScenario(func(*godog.Scenario) { + state = tstate.New() + }) + + ctx.AfterScenario(func(*messages.Pickle, error) { + // delete namespace an all the content + _ = kubernetes.DeleteNamespace(kubernetes.KubeClient, state.Namespace) + }) +} + +func anIngressResourceInANewRandomNamespace(spec *godog.DocString) error { + ns, err := kubernetes.NewNamespace(kubernetes.KubeClient) + if err != nil { + return err + } + + state.Namespace = ns + + ingress, err := kubernetes.IngressFromManifest(state.Namespace, spec.Content) + if err != nil { + return err + } + + err = kubernetes.DeploymentsFromIngress(kubernetes.KubeClient, ingress) + if err != nil { + return err + } + + err = kubernetes.NewIngress(kubernetes.KubeClient, state.Namespace, ingress) + if err != nil { + return err + } + + state.IngressName = ingress.GetName() + + return nil + +} + +func theIngressStatusShowsTheIPAddressOrFQDNWhereItIsExposed() error { + ingress, err := kubernetes.WaitForIngressAddress(kubernetes.KubeClient, state.Namespace, state.IngressName) + if err != nil { + return err + } + + state.IPOrFQDN = ingress + + time.Sleep(3 * time.Second) + + return err +} + +func iSendARequestToWithHeader(method, rawURL string, header *godog.DocString) error { + var headerInfo http.Header + if header.Content != "" { + if err := json.Unmarshal([]byte(header.Content), &headerInfo); err != nil { + return fmt.Errorf("err in jsonEncoder.Encode: ", err.Error()) + } + } + u, err := url.Parse(rawURL) + if err != nil { + return err + } + return state.CaptureRoundTrip(method, u.Scheme, u.Host, u.Path, headerInfo) +} + +func theResponseStatuscodeMustBe(statusCode int) error { + return state.AssertStatusCode(statusCode) +} + +func theResponseMustBeServedByTheService(service string) error { + return state.AssertServedBy(service) +} + +func iSendARequestTo(method string, rawURL string) error { + u, err := url.Parse(rawURL) + if err != nil { + return err + } + return state.CaptureRoundTrip(method, u.Scheme, u.Host, u.Path, nil) +} + +func theIngressStatusShouldNotBeSuccess() error { + _, err := kubernetes.WaitForIngressAddress(kubernetes.KubeClient, state.Namespace, state.IngressName) + if err == nil { + return fmt.Errorf("create ingress should return error") + } + + return nil +} diff --git a/test/e2e/steps/annotations/route/priority/steps.go b/test/e2e/steps/annotations/route/priority/steps.go new file mode 100644 index 00000000..0dc0fd47 --- /dev/null +++ b/test/e2e/steps/annotations/route/priority/steps.go @@ -0,0 +1,128 @@ +/* +Copyright 2021 The BFE Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package priority + +import ( + "encoding/json" + "fmt" + "net/http" + "net/url" + "time" + + "github.com/cucumber/godog" + "github.com/cucumber/messages-go/v16" + + "github.com/bfenetworks/ingress-bfe/test/e2e/pkg/kubernetes" + tstate "github.com/bfenetworks/ingress-bfe/test/e2e/pkg/state" +) + +var ( + state *tstate.Scenario + namespaces []string +) + +// IMPORTANT: Steps definitions are generated and should not be modified +// by hand but rather through make codegen. DO NOT EDIT. + +// InitializeScenario configures the Feature to test +func InitializeScenario(ctx *godog.ScenarioContext) { + ctx.Step(`^an Ingress resource in a new random namespace$`, anIngressResourceInANewRandomNamespace) + ctx.Step(`^The Ingress status shows the IP address or FQDN where it is exposed$`, theIngressStatusShowsTheIPAddressOrFQDNWhereItIsExposed) + ctx.Step(`^I send a "([^"]*)" request to "([^"]*)" with header$`, iSendARequestToWithHeader) + ctx.Step(`^the response status-code must be (\d+)$`, theResponseStatuscodeMustBe) + ctx.Step(`^the response must be served by the "([^"]*)" service$`, theResponseMustBeServedByTheService) + + ctx.BeforeScenario(func(*godog.Scenario) { + state = tstate.New() + namespaces = []string{} + }) + + ctx.AfterScenario(func(*messages.Pickle, error) { + // delete namespace an all the content + _ = kubernetes.DeleteNamespace(kubernetes.KubeClient, state.Namespace) + for _, ns := range namespaces { + _ = kubernetes.DeleteNamespace(kubernetes.KubeClient, ns) + } + }) +} + +func anIngressResourceInANewRandomNamespace(spec *godog.DocString) error { + ns, err := kubernetes.NewNamespace(kubernetes.KubeClient) + if err != nil { + return err + } + + if state.Namespace != "" { + namespaces = append(namespaces, state.Namespace) + } + + state.Namespace = ns + + ingress, err := kubernetes.IngressFromManifest(state.Namespace, spec.Content) + if err != nil { + return err + } + + err = kubernetes.DeploymentsFromIngress(kubernetes.KubeClient, ingress) + if err != nil { + return err + } + + err = kubernetes.NewIngress(kubernetes.KubeClient, state.Namespace, ingress) + if err != nil { + return err + } + + state.IngressName = ingress.GetName() + + return nil +} + +func theIngressStatusShowsTheIPAddressOrFQDNWhereItIsExposed() error { + ingress, err := kubernetes.WaitForIngressAddress(kubernetes.KubeClient, state.Namespace, state.IngressName) + if err != nil { + return err + } + + state.IPOrFQDN = ingress + + time.Sleep(3 * time.Second) + + return err +} + +func iSendARequestToWithHeader(method, rawURL string, header *godog.DocString) error { + var headerInfo http.Header + if header.Content != "" { + if err := json.Unmarshal([]byte(header.Content), &headerInfo); err != nil { + return fmt.Errorf("err in jsonEncoder.Encode: %s", err) + } + } + u, err := url.Parse(rawURL) + if err != nil { + return err + } + return state.CaptureRoundTrip(method, u.Scheme, u.Host, u.Path, headerInfo) +} + +func theResponseStatuscodeMustBe(statusCode int) error { + return state.AssertStatusCode(statusCode) +} + +func theResponseMustBeServedByTheService(service string) error { + return state.AssertServedBy(service) +} diff --git a/test/e2e/steps/conformance/defaultbackend/steps.go b/test/e2e/steps/conformance/defaultbackend/steps.go new file mode 100644 index 00000000..82991a19 --- /dev/null +++ b/test/e2e/steps/conformance/defaultbackend/steps.go @@ -0,0 +1,169 @@ +/* +Copyright 2021 The BFE Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package defaultbackend + +import ( + "fmt" + "time" + + "github.com/cucumber/godog" + "github.com/cucumber/messages-go/v16" + + "github.com/bfenetworks/ingress-bfe/test/e2e/pkg/kubernetes" + tstate "github.com/bfenetworks/ingress-bfe/test/e2e/pkg/state" +) + +var ( + state *tstate.Scenario +) + +// IMPORTANT: Steps definitions are generated and should not be modified +// by hand but rather through make codegen. DO NOT EDIT. + +// InitializeScenario configures the Feature to test +func InitializeScenario(ctx *godog.ScenarioContext) { + ctx.Step(`^a new random namespace$`, aNewRandomNamespace) + ctx.Step(`^an Ingress resource named "([^"]*)" with this spec:$`, anIngressResourceNamedWithThisSpec) + ctx.Step(`^The Ingress status shows the IP address or FQDN where it is exposed$`, theIngressStatusShowsTheIPAddressOrFQDNWhereItIsExposed) + ctx.Step(`^I send a "([^"]*)" request to http:\/\/"([^"]*)"\/"([^"]*)"$`, iSendARequestToHttp) + ctx.Step(`^the response status-code must be (\d+)$`, theResponseStatuscodeMustBe) + ctx.Step(`^the response must be served by the "([^"]*)" service$`, theResponseMustBeServedByTheService) + ctx.Step(`^the response proto must be "([^"]*)"$`, theResponseProtoMustBe) + ctx.Step(`^the response headers must contain with matching $`, theResponseHeadersMustContainKeyWithMatchingValue) + ctx.Step(`^the request method must be "([^"]*)"$`, theRequestMethodMustBe) + ctx.Step(`^the request path must be "([^"]*)"$`, theRequestPathMustBe) + ctx.Step(`^the request proto must be "([^"]*)"$`, theRequestProtoMustBe) + ctx.Step(`^the request headers must contain with matching $`, theRequestHeadersMustContainKeyWithMatchingValue) + + ctx.BeforeScenario(func(*godog.Scenario) { + state = tstate.New() + }) + + ctx.AfterScenario(func(*messages.Pickle, error) { + // delete namespace an all the content + _ = kubernetes.DeleteNamespace(kubernetes.KubeClient, state.Namespace) + }) +} + +func aNewRandomNamespace() error { + ns, err := kubernetes.NewNamespace(kubernetes.KubeClient) + if err != nil { + return err + } + + state.Namespace = ns + return nil +} + +func anIngressResourceNamedWithThisSpec(name string, spec *godog.DocString) error { + ingress, err := kubernetes.IngressFromSpec(name, state.Namespace, spec.Content) + if err != nil { + return err + } + + err = kubernetes.DeploymentsFromIngress(kubernetes.KubeClient, ingress) + if err != nil { + return err + } + + err = kubernetes.NewIngress(kubernetes.KubeClient, state.Namespace, ingress) + if err != nil { + return err + } + + state.IngressName = name + + return nil +} + +func theIngressStatusShowsTheIPAddressOrFQDNWhereItIsExposed() error { + ingress, err := kubernetes.WaitForIngressAddress(kubernetes.KubeClient, state.Namespace, state.IngressName) + if err != nil { + return err + } + + state.IPOrFQDN = ingress + + time.Sleep(3 * time.Second) + + return err +} + +func iSendARequestToHttp(method string, hostname string, path string) error { + return state.CaptureRoundTrip(method, "http", hostname, path, nil) +} + +func theResponseStatuscodeMustBe(statusCode int) error { + return state.AssertStatusCode(statusCode) +} + +func theResponseMustBeServedByTheService(service string) error { + return state.AssertServedBy(service) +} + +func theResponseProtoMustBe(proto string) error { + return state.AssertResponseProto(proto) +} + +func theResponseHeadersMustContainKeyWithMatchingValue(headers *godog.Table) error { + return assertHeaderTable(headers, state.AssertResponseHeader) +} + +func theRequestMethodMustBe(method string) error { + return state.AssertMethod(method) +} + +func theRequestPathMustBe(path string) error { + return state.AssertRequestPath(path) +} + +func theRequestProtoMustBe(proto string) error { + return state.AssertRequestProto(proto) +} + +func theRequestHeadersMustContainKeyWithMatchingValue(headers *godog.Table) error { + return assertHeaderTable(headers, state.AssertRequestHeader) +} + +func assertHeaderTable(headerTable *godog.Table, assertF func(key string, value string) error) error { + if len(headerTable.Rows) < 1 { + return fmt.Errorf("expected a table with at least one row") + } + + for i, row := range headerTable.Rows { + if len(row.Cells) != 2 { + return fmt.Errorf("expected a table with 2 cells, it contained %v", len(row.Cells)) + } + + headerKey := row.Cells[0].Value + headerValue := row.Cells[1].Value + + if i == 0 { + if headerKey != "key" && headerValue != "value" { + return fmt.Errorf("expected a table with a header row of 'key' and 'value' but got '%v' and '%v'", headerKey, headerValue) + } + // Skip the header row + continue + } + + if err := assertF(headerKey, headerValue); err != nil { + return err + } + } + + return nil +} diff --git a/test/e2e/steps/conformance/hostrules/steps.go b/test/e2e/steps/conformance/hostrules/steps.go new file mode 100644 index 00000000..3d48fffd --- /dev/null +++ b/test/e2e/steps/conformance/hostrules/steps.go @@ -0,0 +1,147 @@ +/* +Copyright 2021 The BFE Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package hostrules + +import ( + "net/url" + "time" + + "github.com/cucumber/godog" + "github.com/cucumber/messages-go/v16" + + "github.com/bfenetworks/ingress-bfe/test/e2e/pkg/kubernetes" + tstate "github.com/bfenetworks/ingress-bfe/test/e2e/pkg/state" +) + +var ( + state *tstate.Scenario +) + +// IMPORTANT: Steps definitions are generated and should not be modified +// by hand but rather through make codegen. DO NOT EDIT. + +// InitializeScenario configures the Feature to test +func InitializeScenario(ctx *godog.ScenarioContext) { + ctx.Step(`^a new random namespace$`, aNewRandomNamespace) + ctx.Step(`^a self-signed TLS secret named "([^"]*)" for the "([^"]*)" hostname$`, aSelfsignedTLSSecretNamedForTheHostname) + ctx.Step(`^an Ingress resource$`, anIngressResource) + ctx.Step(`^The Ingress status shows the IP address or FQDN where it is exposed$`, theIngressStatusShowsTheIPAddressOrFQDNWhereItIsExposed) + ctx.Step(`^I send a "([^"]*)" request to "([^"]*)"$`, iSendARequestTo) + ctx.Step(`^the secure connection must verify the "([^"]*)" hostname$`, theSecureConnectionMustVerifyTheHostname) + ctx.Step(`^the response status-code must be (\d+)$`, theResponseStatuscodeMustBe) + ctx.Step(`^the response must be served by the "([^"]*)" service$`, theResponseMustBeServedByTheService) + ctx.Step(`^the request host must be "([^"]*)"$`, theRequestHostMustBe) + + ctx.BeforeScenario(func(*godog.Scenario) { + state = tstate.New() + }) + + ctx.AfterScenario(func(*messages.Pickle, error) { + // delete namespace an all the content + _ = kubernetes.DeleteNamespace(kubernetes.KubeClient, state.Namespace) + }) +} + +func aNewRandomNamespace() error { + ns, err := kubernetes.NewNamespace(kubernetes.KubeClient) + if err != nil { + return err + } + + state.Namespace = ns + return nil +} + +func aSelfsignedTLSSecretNamedForTheHostname(secretName string, host string) error { + err := kubernetes.NewSelfSignedSecret(kubernetes.KubeClient, state.Namespace, secretName, []string{host}) + if err != nil { + return err + } + + state.SecretName = secretName + + return nil + +} + +func anIngressResource(spec *godog.DocString) error { + ingress, err := kubernetes.IngressFromManifest(state.Namespace, spec.Content) + if err != nil { + return err + } + + err = kubernetes.DeploymentsFromIngress(kubernetes.KubeClient, ingress) + if err != nil { + return err + } + + err = kubernetes.NewIngress(kubernetes.KubeClient, state.Namespace, ingress) + if err != nil { + return err + } + + state.IngressName = ingress.GetName() + + return nil +} + +func theIngressStatusShowsTheIPAddressOrFQDNWhereItIsExposed() error { + ingress, err := kubernetes.WaitForIngressAddress(kubernetes.KubeClient, state.Namespace, state.IngressName) + if err != nil { + return err + } + + state.IPOrFQDN = ingress + + time.Sleep(3 * time.Second) + + return err +} + +func iSendARequestTo(method string, rawURL string) error { + u, err := url.Parse(rawURL) + if err != nil { + return err + } + return state.CaptureRoundTrip(method, u.Scheme, u.Host, u.Path, nil) +} + +func theSecureConnectionMustVerifyTheHostname(hostname string) error { + err := state.AssertTLSHostname(hostname) + if err != nil { + return err + } + + err = state.AssertResponseCertificate(hostname) + if err != nil { + return err + } + + return nil +} + +func theResponseStatuscodeMustBe(statusCode int) error { + return state.AssertStatusCode(statusCode) +} + +func theResponseMustBeServedByTheService(service string) error { + return state.AssertServedBy(service) +} + +func theRequestHostMustBe(host string) error { + return state.AssertRequestHost(host) +} diff --git a/test/e2e/steps/conformance/ingressclass/steps.go b/test/e2e/steps/conformance/ingressclass/steps.go new file mode 100644 index 00000000..11e2f79d --- /dev/null +++ b/test/e2e/steps/conformance/ingressclass/steps.go @@ -0,0 +1,86 @@ +/* +Copyright 2021 The BFE Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package ingressclass + +import ( + "fmt" + + "github.com/cucumber/godog" + "github.com/cucumber/messages-go/v16" + + "github.com/bfenetworks/ingress-bfe/test/e2e/pkg/kubernetes" + tstate "github.com/bfenetworks/ingress-bfe/test/e2e/pkg/state" +) + +var ( + state *tstate.Scenario +) + +// IMPORTANT: Steps definitions are generated and should not be modified +// by hand but rather through make codegen. DO NOT EDIT. + +// InitializeScenario configures the Feature to test +func InitializeScenario(ctx *godog.ScenarioContext) { + ctx.Step(`^an Ingress resource in a new random namespace$`, anIngressResourceInANewRandomNamespace) + ctx.Step(`^The Ingress status should not contain the IP address or FQDN$`, theIngressStatusShouldNotContainTheIPAddressOrFQDN) + + ctx.BeforeScenario(func(*godog.Scenario) { + state = tstate.New() + }) + + ctx.AfterScenario(func(*messages.Pickle, error) { + // delete namespace an all the content + _ = kubernetes.DeleteNamespace(kubernetes.KubeClient, state.Namespace) + }) +} + +func anIngressResourceInANewRandomNamespace(spec *godog.DocString) error { + ns, err := kubernetes.NewNamespace(kubernetes.KubeClient) + if err != nil { + return err + } + + state.Namespace = ns + + ingress, err := kubernetes.IngressFromManifest(state.Namespace, spec.Content) + if err != nil { + return err + } + + err = kubernetes.DeploymentsFromIngress(kubernetes.KubeClient, ingress) + if err != nil { + return err + } + + err = kubernetes.NewIngress(kubernetes.KubeClient, state.Namespace, ingress) + if err != nil { + return err + } + + state.IngressName = ingress.GetName() + + return nil +} + +func theIngressStatusShouldNotContainTheIPAddressOrFQDN() error { + _, err := kubernetes.WaitForIngressAddress(kubernetes.KubeClient, state.Namespace, state.IngressName) + if err == nil { + return fmt.Errorf("waiting for Ingress status should not return an IP address or FQDN") + } + + return nil +} diff --git a/test/e2e/steps/conformance/loadbalancing/steps.go b/test/e2e/steps/conformance/loadbalancing/steps.go new file mode 100644 index 00000000..0f3f075d --- /dev/null +++ b/test/e2e/steps/conformance/loadbalancing/steps.go @@ -0,0 +1,138 @@ +/* +Copyright 2021 The BFE Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package loadbalancing + +import ( + "fmt" + "net/url" + "time" + + "github.com/cucumber/godog" + "github.com/cucumber/messages-go/v16" + "k8s.io/apimachinery/pkg/util/sets" + + "github.com/bfenetworks/ingress-bfe/test/e2e/pkg/kubernetes" + tstate "github.com/bfenetworks/ingress-bfe/test/e2e/pkg/state" +) + +var ( + state *tstate.Scenario + + resultStatus map[int]sets.String +) + +// IMPORTANT: Steps definitions are generated and should not be modified +// by hand but rather through make codegen. DO NOT EDIT. + +// InitializeScenario configures the Feature to test +func InitializeScenario(ctx *godog.ScenarioContext) { + ctx.Step(`^an Ingress resource in a new random namespace$`, anIngressResourceInANewRandomNamespace) + ctx.Step(`^The Ingress status shows the IP address or FQDN where it is exposed$`, theIngressStatusShowsTheIPAddressOrFQDNWhereItIsExposed) + ctx.Step(`^The backend deployment "([^"]*)" for the ingress resource is scaled to (\d+)$`, theBackendDeploymentForTheIngressResourceIsScaledTo) + ctx.Step(`^I send (\d+) requests to "([^"]*)"$`, iSendRequestsTo) + ctx.Step(`^all the responses status-code must be (\d+) and the response body should contain the IP address of (\d+) different Kubernetes pods$`, allTheResponsesStatuscodeMustBeAndTheResponseBodyShouldContainTheIPAddressOfDifferentKubernetesPods) + + ctx.BeforeScenario(func(*godog.Scenario) { + state = tstate.New() + resultStatus = make(map[int]sets.String, 0) + }) + + ctx.AfterScenario(func(*messages.Pickle, error) { + // delete namespace an all the content + _ = kubernetes.DeleteNamespace(kubernetes.KubeClient, state.Namespace) + }) +} + +func anIngressResourceInANewRandomNamespace(spec *godog.DocString) error { + ns, err := kubernetes.NewNamespace(kubernetes.KubeClient) + if err != nil { + return err + } + + state.Namespace = ns + + ingress, err := kubernetes.IngressFromManifest(state.Namespace, spec.Content) + if err != nil { + return err + } + + err = kubernetes.DeploymentsFromIngress(kubernetes.KubeClient, ingress) + if err != nil { + return err + } + + err = kubernetes.NewIngress(kubernetes.KubeClient, state.Namespace, ingress) + if err != nil { + return err + } + + state.IngressName = ingress.GetName() + + return nil +} + +func theIngressStatusShowsTheIPAddressOrFQDNWhereItIsExposed() error { + ingress, err := kubernetes.WaitForIngressAddress(kubernetes.KubeClient, state.Namespace, state.IngressName) + if err != nil { + return err + } + + state.IPOrFQDN = ingress + + time.Sleep(3 * time.Second) + + return err +} + +func theBackendDeploymentForTheIngressResourceIsScaledTo(deployment string, replicas int) error { + return kubernetes.ScaleIngressBackendDeployment(kubernetes.KubeClient, state.Namespace, state.IngressName, deployment, replicas) +} + +func iSendRequestsTo(totalRequest int, rawURL string) error { + u, err := url.Parse(rawURL) + if err != nil { + return err + } + + for iteration := 1; iteration <= totalRequest; iteration++ { + err := state.CaptureRoundTrip("GET", u.Scheme, u.Host, u.Path, nil) + if err != nil { + return err + } + + if resultStatus[state.CapturedResponse.StatusCode] == nil { + resultStatus[state.CapturedResponse.StatusCode] = sets.NewString() + } + + resultStatus[state.CapturedResponse.StatusCode].Insert(state.CapturedRequest.Pod) + } + + return nil +} + +func allTheResponsesStatuscodeMustBeAndTheResponseBodyShouldContainTheIPAddressOfDifferentKubernetesPods(statusCode int, pods int) error { + results, ok := resultStatus[statusCode] + if !ok { + return fmt.Errorf("no reponses for status code %v returned", statusCode) + } + + if results.Len() != pods { + return fmt.Errorf("expected %v different POD IP addresses/FQDN for status code %v but %v was returned", pods, statusCode, results.Len()) + } + + return nil +} diff --git a/test/e2e/steps/conformance/pathrules/steps.go b/test/e2e/steps/conformance/pathrules/steps.go new file mode 100644 index 00000000..91233a98 --- /dev/null +++ b/test/e2e/steps/conformance/pathrules/steps.go @@ -0,0 +1,121 @@ +/* +Copyright 2021 The BFE Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package pathrules + +import ( + "net/url" + "time" + + "github.com/cucumber/godog" + + "github.com/bfenetworks/ingress-bfe/test/e2e/pkg/kubernetes" + tstate "github.com/bfenetworks/ingress-bfe/test/e2e/pkg/state" +) + +var ( + state *tstate.Scenario = tstate.New() +) + +// IMPORTANT: Steps definitions are generated and should not be modified +// by hand but rather through make codegen. DO NOT EDIT. + +// InitializeScenario configures the Feature to test +func InitializeScenario(ctx *godog.ScenarioContext) { + ctx.Step(`^an Ingress resource in a new random namespace$`, anIngressResourceInANewRandomNamespace) + ctx.Step(`^The Ingress status shows the IP address or FQDN where it is exposed$`, theIngressStatusShowsTheIPAddressOrFQDNWhereItIsExposed) + ctx.Step(`^I send a "([^"]*)" request to "([^"]*)"$`, iSendARequestTo) + ctx.Step(`^the response status-code must be (\d+)$`, theResponseStatuscodeMustBe) + ctx.Step(`^the response must be served by the "([^"]*)" service$`, theResponseMustBeServedByTheService) + +} + +func InitializeSuite(ctx *godog.TestSuiteContext) { + ctx.BeforeSuite(func() { + state = tstate.New() + }) + + ctx.AfterSuite(func() { + // delete namespace an all the content + _ = kubernetes.DeleteNamespace(kubernetes.KubeClient, state.Namespace) + }) + +} + +func anIngressResourceInANewRandomNamespace(spec *godog.DocString) error { + if state.Namespace != "" && state.IngressName != "" { + return nil + } + + ns, err := kubernetes.NewNamespace(kubernetes.KubeClient) + if err != nil { + return err + } + + state.Namespace = ns + + ingress, err := kubernetes.IngressFromManifest(state.Namespace, spec.Content) + if err != nil { + return err + } + + err = kubernetes.DeploymentsFromIngress(kubernetes.KubeClient, ingress) + if err != nil { + return err + } + + err = kubernetes.NewIngress(kubernetes.KubeClient, state.Namespace, ingress) + if err != nil { + return err + } + + state.IngressName = ingress.GetName() + + return nil +} + +func theIngressStatusShowsTheIPAddressOrFQDNWhereItIsExposed() error { + if state.IPOrFQDN != nil { + return nil + } + + ingress, err := kubernetes.WaitForIngressAddress(kubernetes.KubeClient, state.Namespace, state.IngressName) + if err != nil { + return err + } + + state.IPOrFQDN = ingress + + time.Sleep(3 * time.Second) + + return err +} + +func iSendARequestTo(method string, rawURL string) error { + u, err := url.Parse(rawURL) + if err != nil { + return err + } + return state.CaptureRoundTrip(method, u.Scheme, u.Host, u.Path, nil) +} + +func theResponseStatuscodeMustBe(statusCode int) error { + return state.AssertStatusCode(statusCode) +} + +func theResponseMustBeServedByTheService(service string) error { + return state.AssertServedBy(service) +} diff --git a/test/e2e/steps/rules/host1/steps.go b/test/e2e/steps/rules/host1/steps.go new file mode 100644 index 00000000..2bc06c8c --- /dev/null +++ b/test/e2e/steps/rules/host1/steps.go @@ -0,0 +1,110 @@ +/* +Copyright 2020 The BFE Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package host1 + +import ( + "net/url" + "time" + + "github.com/cucumber/godog" + "github.com/cucumber/messages-go/v16" + + "github.com/bfenetworks/ingress-bfe/test/e2e/pkg/kubernetes" + tstate "github.com/bfenetworks/ingress-bfe/test/e2e/pkg/state" +) + +var ( + state *tstate.Scenario +) + +// IMPORTANT: Steps definitions are generated and should not be modified +// by hand but rather through make codegen. DO NOT EDIT. + +// InitializeScenario configures the Feature to test +func InitializeScenario(ctx *godog.ScenarioContext) { + ctx.Step(`^an Ingress resource in a new random namespace$`, anIngressResourceInANewRandomNamespace) + ctx.Step(`^The Ingress status shows the IP address or FQDN where it is exposed$`, theIngressStatusShowsTheIPAddressOrFQDNWhereItIsExposed) + ctx.Step(`^I send a "([^"]*)" request to "([^"]*)"$`, iSendARequestTo) + ctx.Step(`^the response status-code must be (\d+)$`, theResponseStatuscodeMustBe) + ctx.Step(`^the response must be served by the "([^"]*)" service$`, theResponseMustBeServedByTheService) + + ctx.BeforeScenario(func(*godog.Scenario) { + state = tstate.New() + }) + + ctx.AfterScenario(func(*messages.Pickle, error) { + // delete namespace an all the content + _ = kubernetes.DeleteNamespace(kubernetes.KubeClient, state.Namespace) + }) +} + +func anIngressResourceInANewRandomNamespace(spec *godog.DocString) error { + ns, err := kubernetes.NewNamespace(kubernetes.KubeClient) + if err != nil { + return err + } + + state.Namespace = ns + + ingress, err := kubernetes.IngressFromManifest(state.Namespace, spec.Content) + if err != nil { + return err + } + + err = kubernetes.DeploymentsFromIngress(kubernetes.KubeClient, ingress) + if err != nil { + return err + } + + err = kubernetes.NewIngress(kubernetes.KubeClient, state.Namespace, ingress) + if err != nil { + return err + } + + state.IngressName = ingress.GetName() + + return nil +} + +func theIngressStatusShowsTheIPAddressOrFQDNWhereItIsExposed() error { + ingress, err := kubernetes.WaitForIngressAddress(kubernetes.KubeClient, state.Namespace, state.IngressName) + if err != nil { + return err + } + + state.IPOrFQDN = ingress + + time.Sleep(3 * time.Second) + + return err +} + +func iSendARequestTo(method string, rawURL string) error { + u, err := url.Parse(rawURL) + if err != nil { + return err + } + return state.CaptureRoundTrip(method, u.Scheme, u.Host, u.Path, nil) +} + +func theResponseStatuscodeMustBe(statusCode int) error { + return state.AssertStatusCode(statusCode) +} + +func theResponseMustBeServedByTheService(service string) error { + return state.AssertServedBy(service) +} diff --git a/test/e2e/steps/rules/host2/steps.go b/test/e2e/steps/rules/host2/steps.go new file mode 100644 index 00000000..4a9682a1 --- /dev/null +++ b/test/e2e/steps/rules/host2/steps.go @@ -0,0 +1,110 @@ +/* +Copyright 2020 The BFE Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package host2 + +import ( + "net/url" + "time" + + "github.com/cucumber/godog" + "github.com/cucumber/messages-go/v16" + + "github.com/bfenetworks/ingress-bfe/test/e2e/pkg/kubernetes" + tstate "github.com/bfenetworks/ingress-bfe/test/e2e/pkg/state" +) + +var ( + state *tstate.Scenario +) + +// IMPORTANT: Steps definitions are generated and should not be modified +// by hand but rather through make codegen. DO NOT EDIT. + +// InitializeScenario configures the Feature to test +func InitializeScenario(ctx *godog.ScenarioContext) { + ctx.Step(`^an Ingress resource in a new random namespace$`, anIngressResourceInANewRandomNamespace) + ctx.Step(`^The Ingress status shows the IP address or FQDN where it is exposed$`, theIngressStatusShowsTheIPAddressOrFQDNWhereItIsExposed) + ctx.Step(`^I send a "([^"]*)" request to "([^"]*)"$`, iSendARequestTo) + ctx.Step(`^the response status-code must be (\d+)$`, theResponseStatuscodeMustBe) + ctx.Step(`^the response must be served by the "([^"]*)" service$`, theResponseMustBeServedByTheService) + + ctx.BeforeScenario(func(*godog.Scenario) { + state = tstate.New() + }) + + ctx.AfterScenario(func(*messages.Pickle, error) { + // delete namespace an all the content + _ = kubernetes.DeleteNamespace(kubernetes.KubeClient, state.Namespace) + }) +} + +func anIngressResourceInANewRandomNamespace(spec *godog.DocString) error { + ns, err := kubernetes.NewNamespace(kubernetes.KubeClient) + if err != nil { + return err + } + + state.Namespace = ns + + ingress, err := kubernetes.IngressFromManifest(state.Namespace, spec.Content) + if err != nil { + return err + } + + err = kubernetes.DeploymentsFromIngress(kubernetes.KubeClient, ingress) + if err != nil { + return err + } + + err = kubernetes.NewIngress(kubernetes.KubeClient, state.Namespace, ingress) + if err != nil { + return err + } + + state.IngressName = ingress.GetName() + + return nil +} + +func theIngressStatusShowsTheIPAddressOrFQDNWhereItIsExposed() error { + ingress, err := kubernetes.WaitForIngressAddress(kubernetes.KubeClient, state.Namespace, state.IngressName) + if err != nil { + return err + } + + state.IPOrFQDN = ingress + + time.Sleep(3 * time.Second) + + return err +} + +func iSendARequestTo(method string, rawURL string) error { + u, err := url.Parse(rawURL) + if err != nil { + return err + } + return state.CaptureRoundTrip(method, u.Scheme, u.Host, u.Path, nil) +} + +func theResponseStatuscodeMustBe(statusCode int) error { + return state.AssertStatusCode(statusCode) +} + +func theResponseMustBeServedByTheService(service string) error { + return state.AssertServedBy(service) +} diff --git a/test/e2e/steps/rules/multipleingress/steps.go b/test/e2e/steps/rules/multipleingress/steps.go new file mode 100644 index 00000000..22025b66 --- /dev/null +++ b/test/e2e/steps/rules/multipleingress/steps.go @@ -0,0 +1,132 @@ +/* +Copyright 2020 The BFE Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package multipleingress + +import ( + "net/url" + "time" + + "github.com/cucumber/godog" + "github.com/cucumber/messages-go/v16" + + "github.com/bfenetworks/ingress-bfe/test/e2e/pkg/kubernetes" + tstate "github.com/bfenetworks/ingress-bfe/test/e2e/pkg/state" +) + +var ( + state *tstate.Scenario +) + +// IMPORTANT: Steps definitions are generated and should not be modified +// by hand but rather through make codegen. DO NOT EDIT. + +// InitializeScenario configures the Feature to test +func InitializeScenario(ctx *godog.ScenarioContext) { + ctx.Step(`^an Ingress resource in a new random namespace$`, anIngressResourceInANewRandomNamespace) + ctx.Step(`^The Ingress status shows the IP address or FQDN where it is exposed$`, theIngressStatusShowsTheIPAddressOrFQDNWhereItIsExposed) + ctx.Step(`^an Ingress resource$`, anIngressResource) + ctx.Step(`^I send a "([^"]*)" request to "([^"]*)"$`, iSendARequestTo) + ctx.Step(`^the response status-code must be (\d+)$`, theResponseStatuscodeMustBe) + ctx.Step(`^the response must be served by the "([^"]*)" service$`, theResponseMustBeServedByTheService) + + ctx.BeforeScenario(func(*godog.Scenario) { + state = tstate.New() + }) + + ctx.AfterScenario(func(*messages.Pickle, error) { + // delete namespace an all the content + _ = kubernetes.DeleteNamespace(kubernetes.KubeClient, state.Namespace) + }) +} + +func anIngressResourceInANewRandomNamespace(arg1 *godog.DocString) error { + ns, err := kubernetes.NewNamespace(kubernetes.KubeClient) + if err != nil { + return err + } + + state.Namespace = ns + + ingress, err := kubernetes.IngressFromManifest(state.Namespace, arg1.Content) + if err != nil { + return err + } + + err = kubernetes.DeploymentsFromIngress(kubernetes.KubeClient, ingress) + if err != nil { + return err + } + + err = kubernetes.NewIngress(kubernetes.KubeClient, state.Namespace, ingress) + if err != nil { + return err + } + + state.IngressName = ingress.GetName() + + return nil +} + +func theIngressStatusShowsTheIPAddressOrFQDNWhereItIsExposed() error { + ingress, err := kubernetes.WaitForIngressAddress(kubernetes.KubeClient, state.Namespace, state.IngressName) + if err != nil { + return err + } + + state.IPOrFQDN = ingress + + time.Sleep(3 * time.Second) + + return err +} + +func anIngressResource(spec *godog.DocString) error { + ingress, err := kubernetes.IngressFromManifest(state.Namespace, spec.Content) + if err != nil { + return err + } + + err = kubernetes.DeploymentsFromIngress(kubernetes.KubeClient, ingress) + if err != nil { + return err + } + + err = kubernetes.NewIngress(kubernetes.KubeClient, state.Namespace, ingress) + if err != nil { + return err + } + + state.IngressName = ingress.GetName() + + return nil +} + +func iSendARequestTo(method string, rawURL string) error { + u, err := url.Parse(rawURL) + if err != nil { + return err + } + return state.CaptureRoundTrip(method, u.Scheme, u.Host, u.Path, nil) +} + +func theResponseStatuscodeMustBe(statusCode int) error { + return state.AssertStatusCode(statusCode) +} + +func theResponseMustBeServedByTheService(service string) error { + return state.AssertServedBy(service) +} diff --git a/test/e2e/steps/rules/path1/steps.go b/test/e2e/steps/rules/path1/steps.go new file mode 100644 index 00000000..0709448b --- /dev/null +++ b/test/e2e/steps/rules/path1/steps.go @@ -0,0 +1,120 @@ +/* +Copyright 2020 The BFE Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package path1 + +import ( + "net/url" + "time" + + "github.com/cucumber/godog" + + "github.com/bfenetworks/ingress-bfe/test/e2e/pkg/kubernetes" + tstate "github.com/bfenetworks/ingress-bfe/test/e2e/pkg/state" +) + +var ( + state *tstate.Scenario +) + +// IMPORTANT: Steps definitions are generated and should not be modified +// by hand but rather through make codegen. DO NOT EDIT. + +// InitializeScenario configures the Feature to test +func InitializeScenario(ctx *godog.ScenarioContext) { + ctx.Step(`^an Ingress resource in a new random namespace$`, anIngressResourceInANewRandomNamespace) + ctx.Step(`^The Ingress status shows the IP address or FQDN where it is exposed$`, theIngressStatusShowsTheIPAddressOrFQDNWhereItIsExposed) + ctx.Step(`^I send a "([^"]*)" request to "([^"]*)"$`, iSendARequestTo) + ctx.Step(`^the response status-code must be (\d+)$`, theResponseStatuscodeMustBe) + ctx.Step(`^the response must be served by the "([^"]*)" service$`, theResponseMustBeServedByTheService) +} + +func InitializeSuite(ctx *godog.TestSuiteContext) { + ctx.BeforeSuite(func() { + state = tstate.New() + }) + + ctx.AfterSuite(func() { + // delete namespace an all the content + _ = kubernetes.DeleteNamespace(kubernetes.KubeClient, state.Namespace) + }) + +} + +func anIngressResourceInANewRandomNamespace(arg1 *godog.DocString) error { + if state.Namespace != "" && state.IngressName != "" { + return nil + } + + ns, err := kubernetes.NewNamespace(kubernetes.KubeClient) + if err != nil { + return err + } + + state.Namespace = ns + + ingress, err := kubernetes.IngressFromManifest(state.Namespace, arg1.Content) + if err != nil { + return err + } + + err = kubernetes.DeploymentsFromIngress(kubernetes.KubeClient, ingress) + if err != nil { + return err + } + + err = kubernetes.NewIngress(kubernetes.KubeClient, state.Namespace, ingress) + if err != nil { + return err + } + + state.IngressName = ingress.GetName() + + return nil +} + +func theIngressStatusShowsTheIPAddressOrFQDNWhereItIsExposed() error { + if state.IPOrFQDN != nil { + return nil + } + + ingress, err := kubernetes.WaitForIngressAddress(kubernetes.KubeClient, state.Namespace, state.IngressName) + if err != nil { + return err + } + + state.IPOrFQDN = ingress + + time.Sleep(3 * time.Second) + + return err +} + +func iSendARequestTo(method string, rawURL string) error { + u, err := url.Parse(rawURL) + if err != nil { + return err + } + return state.CaptureRoundTrip(method, u.Scheme, u.Host, u.Path, nil) +} + +func theResponseStatuscodeMustBe(statusCode int) error { + return state.AssertStatusCode(statusCode) +} + +func theResponseMustBeServedByTheService(service string) error { + return state.AssertServedBy(service) +} diff --git a/test/e2e/steps/rules/path2/steps.go b/test/e2e/steps/rules/path2/steps.go new file mode 100644 index 00000000..2971c190 --- /dev/null +++ b/test/e2e/steps/rules/path2/steps.go @@ -0,0 +1,120 @@ +/* +Copyright 2020 The BFE Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package path2 + +import ( + "net/url" + "time" + + "github.com/cucumber/godog" + + "github.com/bfenetworks/ingress-bfe/test/e2e/pkg/kubernetes" + tstate "github.com/bfenetworks/ingress-bfe/test/e2e/pkg/state" +) + +var ( + state *tstate.Scenario +) + +// IMPORTANT: Steps definitions are generated and should not be modified +// by hand but rather through make codegen. DO NOT EDIT. + +// InitializeScenario configures the Feature to test +func InitializeScenario(ctx *godog.ScenarioContext) { + ctx.Step(`^an Ingress resource in a new random namespace$`, anIngressResourceInANewRandomNamespace) + ctx.Step(`^The Ingress status shows the IP address or FQDN where it is exposed$`, theIngressStatusShowsTheIPAddressOrFQDNWhereItIsExposed) + ctx.Step(`^I send a "([^"]*)" request to "([^"]*)"$`, iSendARequestTo) + ctx.Step(`^the response status-code must be (\d+)$`, theResponseStatuscodeMustBe) + ctx.Step(`^the response must be served by the "([^"]*)" service$`, theResponseMustBeServedByTheService) +} + +func InitializeSuite(ctx *godog.TestSuiteContext) { + ctx.BeforeSuite(func() { + state = tstate.New() + }) + + ctx.AfterSuite(func() { + // delete namespace an all the content + _ = kubernetes.DeleteNamespace(kubernetes.KubeClient, state.Namespace) + }) + +} + +func anIngressResourceInANewRandomNamespace(arg1 *godog.DocString) error { + if state.Namespace != "" && state.IngressName != "" { + return nil + } + + ns, err := kubernetes.NewNamespace(kubernetes.KubeClient) + if err != nil { + return err + } + + state.Namespace = ns + + ingress, err := kubernetes.IngressFromManifest(state.Namespace, arg1.Content) + if err != nil { + return err + } + + err = kubernetes.DeploymentsFromIngress(kubernetes.KubeClient, ingress) + if err != nil { + return err + } + + err = kubernetes.NewIngress(kubernetes.KubeClient, state.Namespace, ingress) + if err != nil { + return err + } + + state.IngressName = ingress.GetName() + + return nil +} + +func theIngressStatusShowsTheIPAddressOrFQDNWhereItIsExposed() error { + if state.IPOrFQDN != nil { + return nil + } + + ingress, err := kubernetes.WaitForIngressAddress(kubernetes.KubeClient, state.Namespace, state.IngressName) + if err != nil { + return err + } + + state.IPOrFQDN = ingress + + time.Sleep(3 * time.Second) + + return err +} + +func iSendARequestTo(method string, rawURL string) error { + u, err := url.Parse(rawURL) + if err != nil { + return err + } + return state.CaptureRoundTrip(method, u.Scheme, u.Host, u.Path, nil) +} + +func theResponseStatuscodeMustBe(statusCode int) error { + return state.AssertStatusCode(statusCode) +} + +func theResponseMustBeServedByTheService(service string) error { + return state.AssertServedBy(service) +} diff --git a/test/e2e/steps/rules/patherr/steps.go b/test/e2e/steps/rules/patherr/steps.go new file mode 100644 index 00000000..6276a489 --- /dev/null +++ b/test/e2e/steps/rules/patherr/steps.go @@ -0,0 +1,76 @@ +/* +Copyright 2020 The BFE Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package patherr + +import ( + "fmt" + + "github.com/cucumber/godog" + "github.com/cucumber/messages-go/v16" + + "github.com/bfenetworks/ingress-bfe/test/e2e/pkg/kubernetes" + tstate "github.com/bfenetworks/ingress-bfe/test/e2e/pkg/state" +) + +var ( + state *tstate.Scenario +) + +// IMPORTANT: Steps definitions are generated and should not be modified +// by hand but rather through make codegen. DO NOT EDIT. + +// InitializeScenario configures the Feature to test +func InitializeScenario(ctx *godog.ScenarioContext) { + ctx.Step(`^an Ingress resource in a new random namespace should not create$`, anIngressResourceInANewRandomNamespaceShouldNotCreate) + + ctx.BeforeScenario(func(*godog.Scenario) { + state = tstate.New() + }) + + ctx.AfterScenario(func(*messages.Pickle, error) { + // delete namespace an all the content + _ = kubernetes.DeleteNamespace(kubernetes.KubeClient, state.Namespace) + }) +} + +func anIngressResourceInANewRandomNamespaceShouldNotCreate(spec *godog.DocString) error { + ns, err := kubernetes.NewNamespace(kubernetes.KubeClient) + if err != nil { + return err + } + + state.Namespace = ns + + ingress, err := kubernetes.IngressFromManifest(state.Namespace, spec.Content) + if err != nil { + return err + } + + err = kubernetes.DeploymentsFromIngress(kubernetes.KubeClient, ingress) + if err != nil { + return err + } + + err = kubernetes.NewIngress(kubernetes.KubeClient, state.Namespace, ingress) + if err == nil { + return fmt.Errorf("create ingress should return error") + } + + state.IngressName = ingress.GetName() + + return nil +} diff --git a/test/script/controller-svc.yaml b/test/script/controller-svc.yaml new file mode 100644 index 00000000..c8e1f37e --- /dev/null +++ b/test/script/controller-svc.yaml @@ -0,0 +1,22 @@ +apiVersion: v1 +kind: Service +metadata: + name: bfe-controller-service + namespace: ingress-bfe + labels: + app.kubernetes.io/name: bfe-ingress-controller + app.kubernetes.io/instance: bfe-ingress-controller +spec: + type: NodePort + selector: + app.kubernetes.io/name: bfe-ingress-controller + app.kubernetes.io/instance: bfe-ingress-controller + ports: + - name: http + port: 8080 + targetPort: 8080 + nodePort: 30000 + - name: https + port: 8443 + targetPort: 8443 + nodePort: 30001 diff --git a/test/script/deploy-controller.sh b/test/script/deploy-controller.sh new file mode 100755 index 00000000..f70f8aa2 --- /dev/null +++ b/test/script/deploy-controller.sh @@ -0,0 +1,59 @@ +#!/usr/bin/env bash +# Copyright 2022 The BFE Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +set -e + +download_kubectl(){ + if [[ "$OSTYPE" == "linux-gnu"* ]]; then + # linux + echo "linux" + curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" + + elif [[ "$OSTYPE" == "darwin"* ]]; then + # Mac + if [[ $(arch) == 'arm64' ]]; then + curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/darwin/arm64/kubectl" + + else + curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/darwin/amd64/kubectl" + fi + else + echo "unsupported os type: " "$OSTYPE" + exit 1 + fi + + chmod +x ./kubectl + +} + +cd "$(dirname "$0")" +VERSION=$1 + +IMAGE="bfenetworks/bfe-ingress-controller:"$VERSION + +if [[ "$(docker images -q $IMAGE 2> /dev/null)" == "" ]]; then + echo "image does not exist:" "$IMAGE" + exit 1 +fi + +if [[ ! -f kubectl ]]; then + download_kubectl +fi + +# update yaml to version +sed "s#image: .*\$#image: $IMAGE#g" ../../examples/controller-all.yaml > controller-all.yaml + +./kubectl apply -f controller-all.yaml +./kubectl apply -f controller-svc.yaml -f ingressclass.yaml + diff --git a/test/script/ingressclass.yaml b/test/script/ingressclass.yaml new file mode 100644 index 00000000..4a7575e8 --- /dev/null +++ b/test/script/ingressclass.yaml @@ -0,0 +1,11 @@ +apiVersion: networking.k8s.io/v1 +kind: IngressClass +metadata: + labels: + app.kubernetes.io/component: controller + name: bfe + namespace: ingress-bfe + annotations: + ingressclass.kubernetes.io/is-default-class: "true" +spec: + controller: bfe-networks.com/ingress-controller diff --git a/test/script/kind-config.yaml b/test/script/kind-config.yaml new file mode 100644 index 00000000..2aff5f26 --- /dev/null +++ b/test/script/kind-config.yaml @@ -0,0 +1,23 @@ +apiVersion: kind.x-k8s.io/v1alpha4 +kind: Cluster +nodes: +- role: control-plane + kubeadmConfigPatches: + - | + kind: InitConfiguration + nodeRegistration: + kubeletExtraArgs: + node-labels: "ingress-ready=true" + controllerManager: + extraArgs: + namespace-sync-period: 10s + concurrent-deployment-syncs: "30" + deployment-controller-sync-period: 10s + extraPortMappings: + - containerPort: 30000 + hostPort: 30000 + protocol: TCP + - containerPort: 30001 + hostPort: 30001 + protocol: TCP +- role: worker diff --git a/test/script/kind-create-cluster.sh b/test/script/kind-create-cluster.sh new file mode 100755 index 00000000..8ad9e009 --- /dev/null +++ b/test/script/kind-create-cluster.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash +# Copyright 2022 The BFE Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +set -ex + + +download_kind(){ + if [[ "$OSTYPE" == "linux-gnu"* ]]; then + # linux + curl -Lo ./kind https://kind.sigs.k8s.io/dl/v0.11.1/kind-linux-amd64 + + elif [[ "$OSTYPE" == "darwin"* ]]; then + # Mac + if [[ $(arch) == 'arm64' ]]; then + curl -Lo ./kind https://kind.sigs.k8s.io/dl/v0.11.1/kind-darwin-arm64 + else + curl -Lo ./kind https://kind.sigs.k8s.io/dl/v0.11.1/kind-darwin-amd64 + fi + else + echo "unsupported os type: " "$OSTYPE" + exit 1 + fi + chmod +x ./kind +} + +cd "$(dirname "$0")" + +if [[ ! -f kind ]]; then + download_kind +fi + +# check if cluster exist +if ./kind get clusters | grep -Fxq "kind"; then + exit 0 +fi + +./kind create cluster --config=./kind-config.yaml + + diff --git a/test/script/kind-delete-cluster.sh b/test/script/kind-delete-cluster.sh new file mode 100755 index 00000000..bf73c5bc --- /dev/null +++ b/test/script/kind-delete-cluster.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +# Copyright 2022 The BFE Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +set -e + +cd "$(dirname "$0")" + +./kind delete cluster diff --git a/test/script/kind-load-images.sh b/test/script/kind-load-images.sh new file mode 100755 index 00000000..191a4124 --- /dev/null +++ b/test/script/kind-load-images.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash +# Copyright 2022 The BFE Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +set -e + +cd "$(dirname "$0")" + +# load bfe-ingress-controller image +VERSION=$1 +IMAGE="bfenetworks/bfe-ingress-controller:"$VERSION + +if [[ "$(docker images -q $IMAGE 2> /dev/null)" == "" ]]; then + echo "image does not exist:" "$IMAGE" + exit 1 +fi + +./kind load docker-image $IMAGE + +# build and load backend image (echoserver) +IMAGE="local/echoserver:0.0.1" + +if [[ "$(docker images -q $IMAGE 2> /dev/null)" == "" ]]; then + (cd ../e2e/images/echoserver; make build-image) +fi + +./kind load docker-image $IMAGE