test(RHIDP-460): update rhdh-setup and add search-catalog scenario

Signed-off-by: Pavel Macík <[email protected]>

.github/workflows/pr-checks.yaml

@@ -24,6 +24,3 @@ jobs:
 
       - name: Run linters
         run: make lint
-
-      - name: Run ShellCheck
-        uses: ludeeus/action-shellcheck@master

.gitignore

@@ -1,8 +1,8 @@
+**/.tmp/*
 .venv
 .vscode
-.rhdh-setup.git*
 benchmark-scenario
 benchmark-before
 benchmark-after
 load-test.log
-shellcheck
+shellcheck

Makefile

@@ -16,15 +16,16 @@ export DURATION ?= 1m
 # Used to set --spawn-rate option of locust CLI (Rate to spawn users at (users per second)).  See https://docs.locust.io/en/stable/configuration.html#command-line-options for details
 export SPAWN_RATE ?= 20
 
-# RHDH image to deploy
-export RHDH_IMAGE_REGISTRY ?= quay.io
-export RHDH_IMAGE_REPO ?= rhdh/rhdh-hub-rhel9
-export RHDH_IMAGE_TAG ?= 1.0-162
+# RHDH image to deploy. Uncomment and set to override RHDH image to deploy and test.
+export RHDH_IMAGE_REGISTRY ?=
+export RHDH_IMAGE_REPO ?=
+export RHDH_IMAGE_TAG ?=
 
 # RHDH Helm chart to deploy
 export RHDH_NAMESPACE ?= rhdh-performance
-export RHDH_HELM_REPO ?= https://gist.githubusercontent.com/nickboldt/a8483eb244f9c4286798e85accaa70af/raw/
+export RHDH_HELM_REPO ?= https://gist.githubusercontent.com/rhdh-bot/63cef5cb6285889527bd6a67c0e1c2a9/raw
 export RHDH_HELM_CHART ?= developer-hub
+export RHDH_HELM_CHART_VERSION ?=
 export RHDH_HELM_RELEASE_NAME ?= rhdh
 
 # RHDH horizontal scaling
@@ -35,6 +36,9 @@ export RHDH_KEYCLOAK_REPLICAS ?= 1
 # python's venv base dir relative to the root of the repository
 PYTHON_VENV=.venv
 
+# Local directory to store temporary files
+export TMP_DIR=$(shell readlink -m .tmp)
+
 # Name of the namespace to install locust operator as well as to run Pods of master and workers.
 LOCUST_NAMESPACE=locust-operator
 
@@ -64,12 +68,17 @@ namespace:
 ## Deploy RHDH
 .PHONY: deploy-rhdh
 deploy-rhdh:
-	./ci-scripts/setup.sh
+	cd ./ci-scripts/rhdh-setup/; ./deploy.sh -i
+
+## Create users, groups and objects such as components and APIs in RHDH
+.PHONY: populate-rhdh
+populate-rhdh:
+	cd ./ci-scripts/rhdh-setup/; ./deploy.sh -c
 
 ## Undeploy RHDH
 .PHONY: undeploy-rhdh
 undeploy-rhdh:
-	./ci-scripts/rhdh-setup/deploy.sh -d
+	cd ./ci-scripts/rhdh-setup/; ./deploy.sh -d
 
 ##	=== Locust Operator ===
 
@@ -93,18 +102,12 @@ undeploy-locust: clean
 	@kubectl delete namespace $(LOCUST_NAMESPACE) --wait
 	@helm repo remove $(LOCUST_OPERATOR_REPO)
 
-## Add docker.io token to default-dockercfg-* Secret to avoid pull rate limits from docker.io
-## Run `make add-dockerio DOCKERIO_TOKEN=...`
-.PHONY: add-dockerio
-add-dockerio: namespace
-	@TOKEN=$(DOCKERIO_TOKEN) NAMESPACE=$(LOCUST_NAMESPACE) ./add-dockercfg-docker.io.sh
-
 ##	=== Testing ===
 
 ## Remove test related resources from cluster
-## Run `make clean SCENARIO=...` to clean a specific scenario
-.PHONY: clean
-clean:
+## Run `make clean-test SCENARIO=...` to clean a specific scenario from cluster
+.PHONY: clean-test
+clean-test:
 	kubectl delete --namespace $(LOCUST_NAMESPACE) cm locust.$(SCENARIO) --ignore-not-found --wait
 	kubectl delete --namespace $(LOCUST_NAMESPACE) locusttests.locust.io $(SCENARIO).test --ignore-not-found --wait || true
 
@@ -132,12 +135,17 @@ shellcheck:
 ## Run all linters
 .PHONY: lint
 lint: shellcheck
+	shellcheck $$(find -name '*.sh')
 
 ##	=== CI ===
 
 ## Run the load test in CI end to end
 .PHONY: ci-run
-ci-run: setup-venv deploy-locust add-dockerio test
+ci-run: setup-venv deploy-locust test
+
+## Deploy and populate RHDH in CI end to end
+.PHONY: ci-deploy
+ci-deploy: clean namespace deploy-rhdh
 
 ##	=== Maintanence ===
 
@@ -149,6 +157,11 @@ update-locust-images:
 	skopeo copy --src-no-creds docker://docker.io/containersol/locust_exporter:latest docker://quay.io/backstage-performance/locust_exporter:latest
 	skopeo copy --src-no-creds docker://docker.io/lotest/locust-k8s-operator:latest docker://quay.io/backstage-performance/locust-k8s-operator:latest
 
+## Clean local resources
+.PHONY: clean
+clean:
+	rm -rvf *.log benchmark-* shellcheck ci-scripts/rhdh-setup/.tmp $(TMP_DIR)
+
 ##	=== Help ===
 
 ## Print help message for all Makefile targets

ci-scripts/collect-results.sh

@@ -4,19 +4,41 @@ set -o nounset
 set -o errexit
 set -o pipefail
 
+echo -e "\n === Collecting test results and metrics ===\n"
+
 ARTIFACT_DIR=${ARTIFACT_DIR:-artifacts}
+mkdir -p "${ARTIFACT_DIR}"
+
 monitoring_collection_data=$ARTIFACT_DIR/benchmark.json
 monitoring_collection_log=$ARTIFACT_DIR/monitoring-collection.log
 monitoring_collection_dir=$ARTIFACT_DIR/monitoring-collection-raw-data-dir
 mkdir -p "$monitoring_collection_dir"
 
-cp -vf load-test.log $ARTIFACT_DIR/load-test.log
+try_gather_file() {
+    if [ -f "$1" ]; then
+        cp -vf "$1" "${2:-$ARTIFACT_DIR}"
+    else
+        echo "WARNING: Tried to gather $1 but the file was not found!"
+    fi
+}
+
+try_gather_dir() {
+    if [ -d "$1" ]; then
+        cp -rvf "$1" "${2:-$ARTIFACT_DIR}"
+    else
+        echo "WARNING: Tried to gather $1 but the directory was not found!"
+    fi
+}
+
+try_gather_dir .tmp
+try_gather_file load-test.log
 
 PYTHON_VENV_DIR=.venv
 
 echo "$(date --utc -Ins) Setting up tool to collect monitoring data"
 python3 -m venv $PYTHON_VENV_DIR
 set +u
+# shellcheck disable=SC1090,SC1091
 source $PYTHON_VENV_DIR/bin/activate
 set -u
 python3 -m pip install --quiet -U pip
@@ -27,20 +49,21 @@ set -u
 
 echo "$(date --utc -Ins) Collecting monitoring data"
 set +u
+# shellcheck disable=SC1090,SC1091
 source $PYTHON_VENV_DIR/bin/activate
 set -u
 mstart=$(date --utc --date "$(cat benchmark-before)" --iso-8601=seconds)
 mend=$(date --utc --date "$(cat benchmark-after)" --iso-8601=seconds)
 mhost=$(kubectl -n openshift-monitoring get route -l app.kubernetes.io/name=thanos-query -o json | jq --raw-output '.items[0].spec.host')
-mversion=$(sed -n 's/^__version__ = "\(.*\)"/\1/p' scenarios/$(cat benchmark-scenario).py)
+mversion=$(sed -n 's/^__version__ = "\(.*\)"/\1/p' "scenarios/$(cat benchmark-scenario).py")
 status_data.py \
     --status-data-file "$monitoring_collection_data" \
     --set \
-        results.started="$(cat benchmark-before)" \
-        results.ended="$(cat benchmark-after)" \
-        name="RHDH load test $(cat benchmark-scenario)" \
-        metadata.scenario.name="$(cat benchmark-scenario)" \
-        metadata.scenario.version="$mversion" \
+    results.started="$(cat benchmark-before)" \
+    results.ended="$(cat benchmark-after)" \
+    name="RHDH load test $(cat benchmark-scenario)" \
+    metadata.scenario.name="$(cat benchmark-scenario)" \
+    metadata.scenario.version="$mversion" \
     -d &>"$monitoring_collection_log"
 status_data.py \
     --status-data-file "$monitoring_collection_data" \

ci-scripts/rhdh-setup/.gitignore

@@ -1,2 +1,3 @@
+nohup.out
 app-config.yaml
 config.yaml

ci-scripts/rhdh-setup/README.md

@@ -1,7 +1,14 @@
 # rhdh-setup
-Run `./deploy.sh` to setup backstage with keycloak.
-The script expects `QUAY_TOKEN` and `GITHUB_TOKEN` to be set.
+Run `./deploy.sh -i` to install backstage with keycloak.
+
+The script expects follwoing environmental variables to be set:
+* `QUAY_TOKEN`
+* `GITHUB_TOKEN`
+* `GITHUB_USER`
+* `GITHUB_REPO`
 
 Run with `-r` to delete backstage and redeploy again `./deploy.sh -r`. This is used to syc up users and groups from keycloak.
 
 Run with `-d` to delete backstage only.
+
+Run with `-c` to create objects (Users, Groups, Components and APIs).