validator: use shelltest docker image to run shelltests

To support this change the runner now copies the built cli from the builder stage of the main Dockerfile and the Dockerfile.shelltest was removed. To support running in both local environments and Docker the shelltests depend on the cli being in the path and the Docker image adds `/` to the PATH, where it copes the binary from the first stage, and the makefile run of the otel_config_validator on the examples will add the validator dir to the path at the end so it is only used if `/otel_config_validator` isn't in the PATH.
open-telemetry · Aug 22, 2024 · 9f210cb · 9f210cb
1 parent aacbdcf
commit 9f210cb
Show file tree

Hide file tree

Showing 16 changed files with 51 additions and 42 deletions.
diff --git a/.github/workflows/validator-tests.yaml b/.github/workflows/validator-tests.yaml
@@ -29,12 +29,7 @@ jobs:
 
     - name: Run ShellTests
       run: |
-        cd validator
-
-        sudo apt-get update
-        sudo apt-get install shelltestrunner
-
-        shelltest -c --diff --all shelltests/*.test
+        make validator-run-shelltests
 
     # TODO: Push this to registry on release
     - name: Build Docker Image

diff --git a/validator/Dockerfile b/validator/Dockerfile
@@ -14,10 +14,29 @@ COPY . .
 RUN --mount=type=cache,target=/root/.cache/go-build,id=cache-go-build --mount=type=cache,target=/root/.cache/go-mod,id=cache-go-mod \
     CGO_ENABLED=0 GOOS=linux go build -o /otel_config_validator
 
-FROM gcr.io/distroless/base-debian12
+FROM gcr.io/distroless/base-debian12 AS releaser
 
 WORKDIR /opt/otel_config_validator
 
 COPY --from=build /otel_config_validator /otel_config_validator
 
 ENTRYPOINT ["/otel_config_validator"]
+
+FROM ubuntu:22.04 AS shelltest
+
+RUN DEBIAN_FRONTEND=noninteractive \
+    apt-get update \
+    && apt-get install -y software-properties-common \
+    && apt-add-repository ppa:rmescandon/yq \
+    && apt-get update \
+    && apt-get install -y shelltestrunner jq yq make \
+    && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /root/validator
+
+COPY --from=build /otel_config_validator /otel_config_validator
+
+# add otel_config_validator to the path
+ENV PATH=/:$PATH
+
+ENTRYPOINT ["shelltest"]
diff --git a/validator/Dockerfile.shelltest b/validator/Dockerfile.shelltest
diff --git a/validator/Makefile b/validator/Makefile
@@ -1,8 +1,10 @@
-ROOT_DIR := $(realpath $(shell dirname $(lastword $(MAKEFILE_LIST))))
-SCHEMA_DIR := ${ROOT_DIR}/../schema
-CURRENT_GIT_REF := $(shell git rev-parse --short HEAD)
-DOCKER_IMAGE_TAG := ${CURRENT_GIT_REF}
-DOCKER_BUILD_ARGS := -f ${ROOT_DIR}/Dockerfile -t otel_config_validator:${DOCKER_IMAGE_TAG} -t otel_config_validator:current
+ROOT_DIR :=$(realpath $(shell dirname $(lastword $(MAKEFILE_LIST))))
+PARENT_DIR :=$(realpath ${ROOT_DIR}/../)
+SCHEMA_DIR :=${ROOT_DIR}/../schema
+CURRENT_GIT_REF :=$(shell git rev-parse --short HEAD)
+DOCKER_IMAGE_TAG :=${CURRENT_GIT_REF}
+DOCKER_BUILD_ARGS :=-f ${ROOT_DIR}/Dockerfile -t otel_config_validator:${DOCKER_IMAGE_TAG} -t otel_config_validator:current
+DOCKER_SHELLTEST_BUILD_ARGS :=-f ${ROOT_DIR}/Dockerfile --target shelltest -t shelltest:${CURRENT_GIT_REF} -t shelltest:current
 EXAMPLE_FILES := $(shell find ${ROOT_DIR}/../examples -name "*.yaml" -exec basename {} \; | sort)
 $(shell mkdir -p out)
 
@@ -13,12 +15,18 @@ validator: validator-copy-schema
 	go build -C ${ROOT_DIR} ${ROOT_DIR}
 
 validator-docker-image: validator-copy-schema
-	docker build ${DOCKER_BUILD_ARGS} ${ROOT_DIR}
+	docker build --target releaser ${DOCKER_BUILD_ARGS} ${ROOT_DIR}
 
 validator-validate-examples:
 	@for f in $(EXAMPLE_FILES); do \
 	    echo "Validating" $$f ; \
-	    ${ROOT_DIR}/otel_config_validator -o  ${ROOT_DIR}/out/$$f ${ROOT_DIR}/../examples/$$f ; \
+	    PATH=${PATH}:${ROOT_DIR}/ otel_config_validator -o  ${ROOT_DIR}/out/$$f ${ROOT_DIR}/../examples/$$f ; \
 	done
 
+validator-build-shelltest-image:
+	docker build ${DOCKER_SHELLTEST_BUILD_ARGS} ${ROOT_DIR}
+
+validator-run-shelltests: validator-build-shelltest-image
+	docker run -v ${PARENT_DIR}:/root shelltest:${CURRENT_GIT_REF} -- --plain /root/validator/shelltests
+
 .PHONY: validator-validate-examples validator-copy-schema validator validator-docker-image
diff --git a/validator/README.md b/validator/README.md
@@ -81,10 +81,12 @@ $ go test .
 
 Running the tests of the compiled CLI requires
 [shelltest](https://github.com/simonmichael/shelltestrunner),
-[jq](https://github.com/jqlang/jq/) and [yq](https://github.com/mikefarah/yq):
+[jq](https://github.com/jqlang/jq/) and [yq](https://github.com/mikefarah/yq)
+and setting the `$PATH` to include `otel_config_validator`, but you can just use
+the `Makefile` target `validator-run-shelltests` to run them in Docker:
 
 ```
-$ shelltest -c --diff --all shelltests/*.test
+$ make validator-run-shelltests
 ```
 
 ### Releasing

diff --git a/validator/shelltests/env_prefix.test b/validator/shelltests/env_prefix.test
@@ -1,3 +1,3 @@
-./otel_config_validator -o out/out.json shelltests/env_prefix.yaml
+otel_config_validator -o out/out.json shelltests/env_prefix.yaml
 >>>
 >>>= 0
diff --git a/validator/shelltests/help.test b/validator/shelltests/help.test
@@ -1,4 +1,4 @@
-./otel_config_validator --help
+otel_config_validator --help
 >>>
 NAME:
    otel_config_validator - Validate a configuration file against the OpenTelemetry Configuration Schema

diff --git a/validator/shelltests/hex_integer.test b/validator/shelltests/hex_integer.test
@@ -1,4 +1,4 @@
-OTEL_ATTRIBUTE_VALUE_LENGTH_LIMIT="0xdeadbeef" ./otel_config_validator -o out/hex_integer.json shelltests/hex_integer.yaml
+OTEL_ATTRIBUTE_VALUE_LENGTH_LIMIT="0xdeadbeef" otel_config_validator -o out/hex_integer.json shelltests/hex_integer.yaml
 >>>
 >>>= 0
 

diff --git a/validator/shelltests/json_out.test b/validator/shelltests/json_out.test
@@ -1,4 +1,4 @@
-FILE_FORMAT=0.1 ./otel_config_validator -o out/out.json shelltests/test.yaml
+FILE_FORMAT=0.1 otel_config_validator -o out/out.json shelltests/test.yaml
 >>>
 >>>= 0
 

diff --git a/validator/shelltests/missing_arg.test b/validator/shelltests/missing_arg.test
@@ -1,4 +1,4 @@
-./otel_config_validator -o out/out.json
+otel_config_validator -o out/out.json
 >>>2
 Error: Must pass a configuration filename
 >>>= 1
diff --git a/validator/shelltests/missing_ext.test b/validator/shelltests/missing_ext.test
@@ -1,4 +1,4 @@
-./otel_config_validator -o out shelltests/multiple_defaults.yaml
+otel_config_validator -o out shelltests/multiple_defaults.yaml
 >>>2
 Unknown extension on output file out
 >>>= 1
diff --git a/validator/shelltests/missing_required.test b/validator/shelltests/missing_required.test
@@ -1,4 +1,4 @@
-./otel_config_validator shelltests/missing_required.yaml
+otel_config_validator shelltests/missing_required.yaml
 >>>2
 jsonschema: '' does not validate with https://opentelemetry.io/otelconfig/opentelemetry_configuration.json#/required: missing properties: 'file_format'
 >>>= 1
diff --git a/validator/shelltests/no_expected_key.test b/validator/shelltests/no_expected_key.test
@@ -1,2 +1,2 @@
-FILE_FORMAT=0.1 ./otel_config_validator shelltests/no_expected_key.yaml
+FILE_FORMAT=0.1 otel_config_validator shelltests/no_expected_key.yaml
 >>>= 1
diff --git a/validator/shelltests/schema_option.test b/validator/shelltests/schema_option.test
@@ -1,4 +1,4 @@
-./otel_config_validator -s shelltests/false_schema shelltests/schema_option.yaml
+otel_config_validator -s shelltests/false_schema shelltests/schema_option.yaml
 >>>2
 jsonschema: '' does not validate with https://opentelemetry.io/otelconfig/opentelemetry_configuration.json#: not allowed
 >>>= 1
diff --git a/validator/shelltests/string_for_int.test b/validator/shelltests/string_for_int.test
@@ -1,4 +1,4 @@
-./otel_config_validator -o out/out.json shelltests/string_for_int.yaml
+otel_config_validator -o out/out.json shelltests/string_for_int.yaml
 >>>2
 jsonschema: '/attribute_limits/attribute_value_length_limit' does not validate with https://opentelemetry.io/otelconfig/opentelemetry_configuration.json#/properties/attribute_limits/$ref/properties/attribute_value_length_limit/type: expected integer or null, but got string
 >>>= 1
diff --git a/validator/shelltests/yaml_out.test b/validator/shelltests/yaml_out.test
@@ -1,4 +1,4 @@
-FILE_FORMAT=0.1 ./otel_config_validator -o out/out.yaml shelltests/test.yaml
+FILE_FORMAT=0.1 otel_config_validator -o out/out.yaml shelltests/test.yaml
 >>>
 >>>= 0