From bfa55326c2cd8f3eac25f991d2be1842afba4f6e Mon Sep 17 00:00:00 2001 From: mehul gautam Date: Mon, 17 Jun 2024 02:19:11 +0530 Subject: [PATCH 1/9] added SPM test Signed-off-by: mehul gautam --- .github/workflows/ci-build-spm.yml | 34 ++++++++++++++++++++ scripts/build-SPM.sh | 50 ++++++++++++++++++++++++++++++ 2 files changed, 84 insertions(+) create mode 100644 .github/workflows/ci-build-spm.yml create mode 100755 scripts/build-SPM.sh diff --git a/.github/workflows/ci-build-spm.yml b/.github/workflows/ci-build-spm.yml new file mode 100644 index 00000000000..6e4c2369e86 --- /dev/null +++ b/.github/workflows/ci-build-spm.yml @@ -0,0 +1,34 @@ +name: Build all-in-one + +on: + push: + branches: [main] + + pull_request: + branches: [main] + +concurrency: + group: ${{ github.workflow }}-${{ (github.event.pull_request && github.event.pull_request.number) || github.ref || github.run_id }} + cancel-in-progress: true + +# See https://github.com/ossf/scorecard/blob/main/docs/checks.md#token-permissions +permissions: + contents: read + +jobs: + SPM: + runs-on: ubuntu-latest + steps: + - name: Harden Runner + uses: step-security/harden-runner@17d0e2bd7d51742c71671bd19fa12bdc9d40a3d6 # v2.8.1 + with: + egress-policy: audit # TODO: change to 'egress-policy: block' after couple of runs + + - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 + + - uses: actions/setup-go@cdcb36043654635271a94b9a6d1392de5bb323a7 # v5.0.1 + with: + go-version: 1.22.x + + - name: Run SPM Test + run: ./scripts/build-SPM.sh \ No newline at end of file diff --git a/scripts/build-SPM.sh b/scripts/build-SPM.sh new file mode 100755 index 00000000000..7b83c8c6ca1 --- /dev/null +++ b/scripts/build-SPM.sh @@ -0,0 +1,50 @@ +#!/bin/bash + +set -e -uxf -o pipefail + +# Function to check if a service is healthy +check_service_health() { + local service_name=$1 + local url=$2 + local retry_count=10 + local wait_seconds=5 + + echo "Checking health of service: $service_name at $url" + + for i in $(seq 1 $retry_count); do + if curl -s -L --head --request GET "$url" | grep "200 OK" > /dev/null; then + echo "$service_name is healthy" + return 0 + else + echo "Waiting for $service_name to be healthy... ($i/$retry_count)" + sleep $wait_seconds + fi + done + + echo "Error: $service_name did not become healthy in time" + return 1 +} + +# Function to check if all services are healthy +wait_for_services() { + echo "Waiting for services to be up and running..." + check_service_health "Jaeger" "http://localhost:16686" + check_service_health "Prometheus" "http://localhost:9090/graph" + check_service_health "Grafana" "http://localhost:3000" +} + +# Function to tear down Docker Compose services +teardown_services() { + docker compose -f docker-compose/monitor/docker-compose.yml down +} + +# Main function +main() { + docker compose -f docker-compose/monitor/docker-compose.yml up -d + wait_for_services + echo "All services are running correctly" + teardown_services +} +trap teardown_services EXIT INT +# Run the main function +main From acac74bc36312aec5b8298162ef0b0b60e536e9d Mon Sep 17 00:00:00 2001 From: mehul gautam Date: Wed, 19 Jun 2024 00:22:01 +0530 Subject: [PATCH 2/9] updated build-spm.sh Signed-off-by: mehul gautam --- scripts/build-SPM.sh | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/scripts/build-SPM.sh b/scripts/build-SPM.sh index 7b83c8c6ca1..0b5f4a0cad0 100755 --- a/scripts/build-SPM.sh +++ b/scripts/build-SPM.sh @@ -32,6 +32,38 @@ wait_for_services() { check_service_health "Prometheus" "http://localhost:9090/graph" check_service_health "Grafana" "http://localhost:3000" } +check_jaeger(){ + echo "Checking Jaeger" + services_list=("driver" "customer" "mysql" "redis" "frontend" "route" "ui" ) + sleep 60 + for service in "${services_list[@]}"; do + echo "Processing service: $service" + response=$(curl -s "http://localhost:16686/api/metrics/calls?service=$service&endTs=$(date +%s)000&lookback=1000&step=100&ratePer=60000") + echo "$response" + service_name=$(echo "$response" | jq -r '.metrics[0].labels[] | select(.name=="service_name") | .value') + if [ "$service_name" != $service ]; then + echo "Service name does not match 'driver'" + exit 1 + fi + + all_non_zero=true + metric_points=$(echo "$response" | jq -r '.metrics[0].metricPoints[] | .gaugeValue.doubleValue') + for value in $metric_points; do + if (( $(echo "$value == 0" | bc -l) )); then + all_non_zero=false + break + fi + done + + if [ "$all_non_zero" = true ]; then + echo "All gauge values are non-zero" + else + echo "Some gauge values are zero" + exit 1 + fi + done + +} # Function to tear down Docker Compose services teardown_services() { @@ -42,6 +74,7 @@ teardown_services() { main() { docker compose -f docker-compose/monitor/docker-compose.yml up -d wait_for_services + check_jaeger echo "All services are running correctly" teardown_services } From c822621f096436f40181f460042136a9a142b148 Mon Sep 17 00:00:00 2001 From: mehul gautam Date: Wed, 19 Jun 2024 00:31:13 +0530 Subject: [PATCH 3/9] minor fix Signed-off-by: mehul gautam --- .github/workflows/ci-build-spm.yml | 2 +- scripts/build-SPM.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-build-spm.yml b/.github/workflows/ci-build-spm.yml index 6e4c2369e86..3ef3f401146 100644 --- a/.github/workflows/ci-build-spm.yml +++ b/.github/workflows/ci-build-spm.yml @@ -1,4 +1,4 @@ -name: Build all-in-one +name: Test SPM on: push: diff --git a/scripts/build-SPM.sh b/scripts/build-SPM.sh index 0b5f4a0cad0..63738e638ae 100755 --- a/scripts/build-SPM.sh +++ b/scripts/build-SPM.sh @@ -41,7 +41,7 @@ check_jaeger(){ response=$(curl -s "http://localhost:16686/api/metrics/calls?service=$service&endTs=$(date +%s)000&lookback=1000&step=100&ratePer=60000") echo "$response" service_name=$(echo "$response" | jq -r '.metrics[0].labels[] | select(.name=="service_name") | .value') - if [ "$service_name" != $service ]; then + if [ "$service_name" != "$service" ]; then echo "Service name does not match 'driver'" exit 1 fi From 78f26bbefca304fd38f250e20aac9d67d3dd2a5a Mon Sep 17 00:00:00 2001 From: mehul gautam Date: Wed, 19 Jun 2024 02:45:21 +0530 Subject: [PATCH 4/9] minor changes Signed-off-by: mehul gautam --- scripts/build-SPM.sh | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/scripts/build-SPM.sh b/scripts/build-SPM.sh index 63738e638ae..02ecac94004 100755 --- a/scripts/build-SPM.sh +++ b/scripts/build-SPM.sh @@ -32,14 +32,15 @@ wait_for_services() { check_service_health "Prometheus" "http://localhost:9090/graph" check_service_health "Grafana" "http://localhost:3000" } -check_jaeger(){ - echo "Checking Jaeger" +# Function to check SPM +check_spm(){ + echo "Checking spm" services_list=("driver" "customer" "mysql" "redis" "frontend" "route" "ui" ) + echo "Waiting for 60 seconds for services to start" sleep 60 for service in "${services_list[@]}"; do echo "Processing service: $service" response=$(curl -s "http://localhost:16686/api/metrics/calls?service=$service&endTs=$(date +%s)000&lookback=1000&step=100&ratePer=60000") - echo "$response" service_name=$(echo "$response" | jq -r '.metrics[0].labels[] | select(.name=="service_name") | .value') if [ "$service_name" != "$service" ]; then echo "Service name does not match 'driver'" @@ -48,6 +49,11 @@ check_jaeger(){ all_non_zero=true metric_points=$(echo "$response" | jq -r '.metrics[0].metricPoints[] | .gaugeValue.doubleValue') + # Check if metric points are empty + if [ -z "$metric_points" ]; then + echo "Metric points for service $service are empty" + exit 1 + fi for value in $metric_points; do if (( $(echo "$value == 0" | bc -l) )); then all_non_zero=false @@ -67,14 +73,14 @@ check_jaeger(){ # Function to tear down Docker Compose services teardown_services() { - docker compose -f docker-compose/monitor/docker-compose.yml down + (cd docker-compose/monitor && make clean-all) } # Main function main() { - docker compose -f docker-compose/monitor/docker-compose.yml up -d + (cd docker-compose/monitor && make dev) wait_for_services - check_jaeger + check_spm echo "All services are running correctly" teardown_services } From d0e34089f4a6c41738777ad840408de4d2864fc2 Mon Sep 17 00:00:00 2001 From: Yuri Shkuro Date: Wed, 19 Jun 2024 13:53:36 -0400 Subject: [PATCH 5/9] Fix SPM make build Signed-off-by: Yuri Shkuro --- Makefile.Crossdock.mk | 4 ++-- docker-compose/monitor/Makefile | 13 +++++++++---- docker/Makefile | 5 +++++ 3 files changed, 16 insertions(+), 6 deletions(-) diff --git a/Makefile.Crossdock.mk b/Makefile.Crossdock.mk index 2e702310e2b..56d6f595767 100644 --- a/Makefile.Crossdock.mk +++ b/Makefile.Crossdock.mk @@ -32,8 +32,8 @@ build-crossdock-fresh: build-crossdock-linux make crossdock-fresh .PHONY: crossdock-docker-images-jaeger-backend -crossdock-docker-images-jaeger-backend: PLATFORMS=linux/amd64 -crossdock-docker-images-jaeger-backend: create-baseimg create-debugimg +crossdock-docker-images-jaeger-backend: PLATFORMS=linux/$(shell go env GOARCH) +crossdock-docker-images-jaeger-backend: create-baseimg create-fake-debugimg for component in "jaeger-agent" "jaeger-collector" "jaeger-query" "jaeger-ingester" "all-in-one" ; do \ regex="jaeger-(.*)"; \ component_suffix=$$component; \ diff --git a/docker-compose/monitor/Makefile b/docker-compose/monitor/Makefile index caa2934fc1f..1d325829f4a 100644 --- a/docker-compose/monitor/Makefile +++ b/docker-compose/monitor/Makefile @@ -1,9 +1,14 @@ .PHONY: build -build: export DOCKER_TAG = dev build: clean-jaeger - cd ../../ && \ - make build-all-in-one-linux && \ - make docker-images-jaeger-backend + cd ../../ && make build-all-in-one-linux + cd ../../ && make create-baseimg PLATFORMS=linux/$(shell go env GOARCH) + cd ../../ && docker buildx build --target release \ + --tag jaegertracing/all-in-one:dev \ + --build-arg base_image=localhost:5000/baseimg_alpine:latest \ + --build-arg debug_image=not-used \ + --build-arg TARGETARCH=$(shell go env GOARCH) \ + --load \ + cmd/all-in-one # starts up the system required for SPM using the latest otel image and a development jaeger image. # Note: the jaeger "dev" image can be built with "make build". diff --git a/docker/Makefile b/docker/Makefile index dfb72662a7d..3edf6d8803e 100644 --- a/docker/Makefile +++ b/docker/Makefile @@ -17,6 +17,11 @@ create-debugimg: prepare-docker-buildx --platform=$(PLATFORMS) \ docker/debug +create-fake-debugimg: prepare-docker-buildx + docker buildx build -t $(DEBUG_IMAGE) --push \ + --platform=$(PLATFORMS) \ + docker/base + .PHONY: prepare-docker-buildx prepare-docker-buildx: docker buildx inspect jaeger-build > /dev/null || docker buildx create --use --name=jaeger-build --buildkitd-flags="--allow-insecure-entitlement security.insecure --allow-insecure-entitlement network.host" --driver-opt="network=host" From a9c7a8bd276225d6f984f0ae234d503f481b5be0 Mon Sep 17 00:00:00 2001 From: Yuri Shkuro Date: Wed, 19 Jun 2024 14:00:41 -0400 Subject: [PATCH 6/9] fix Signed-off-by: Yuri Shkuro --- .github/workflows/ci-build-spm.yml | 7 ++- scripts/build-SPM.sh | 89 ------------------------------ 2 files changed, 4 insertions(+), 92 deletions(-) delete mode 100755 scripts/build-SPM.sh diff --git a/.github/workflows/ci-build-spm.yml b/.github/workflows/ci-build-spm.yml index 3ef3f401146..8fed2baffa6 100644 --- a/.github/workflows/ci-build-spm.yml +++ b/.github/workflows/ci-build-spm.yml @@ -29,6 +29,7 @@ jobs: - uses: actions/setup-go@cdcb36043654635271a94b9a6d1392de5bb323a7 # v5.0.1 with: go-version: 1.22.x - - - name: Run SPM Test - run: ./scripts/build-SPM.sh \ No newline at end of file + + - name: Temporary - only run the build + run: + cd docker-compose/monitor && make build diff --git a/scripts/build-SPM.sh b/scripts/build-SPM.sh deleted file mode 100755 index 02ecac94004..00000000000 --- a/scripts/build-SPM.sh +++ /dev/null @@ -1,89 +0,0 @@ -#!/bin/bash - -set -e -uxf -o pipefail - -# Function to check if a service is healthy -check_service_health() { - local service_name=$1 - local url=$2 - local retry_count=10 - local wait_seconds=5 - - echo "Checking health of service: $service_name at $url" - - for i in $(seq 1 $retry_count); do - if curl -s -L --head --request GET "$url" | grep "200 OK" > /dev/null; then - echo "$service_name is healthy" - return 0 - else - echo "Waiting for $service_name to be healthy... ($i/$retry_count)" - sleep $wait_seconds - fi - done - - echo "Error: $service_name did not become healthy in time" - return 1 -} - -# Function to check if all services are healthy -wait_for_services() { - echo "Waiting for services to be up and running..." - check_service_health "Jaeger" "http://localhost:16686" - check_service_health "Prometheus" "http://localhost:9090/graph" - check_service_health "Grafana" "http://localhost:3000" -} -# Function to check SPM -check_spm(){ - echo "Checking spm" - services_list=("driver" "customer" "mysql" "redis" "frontend" "route" "ui" ) - echo "Waiting for 60 seconds for services to start" - sleep 60 - for service in "${services_list[@]}"; do - echo "Processing service: $service" - response=$(curl -s "http://localhost:16686/api/metrics/calls?service=$service&endTs=$(date +%s)000&lookback=1000&step=100&ratePer=60000") - service_name=$(echo "$response" | jq -r '.metrics[0].labels[] | select(.name=="service_name") | .value') - if [ "$service_name" != "$service" ]; then - echo "Service name does not match 'driver'" - exit 1 - fi - - all_non_zero=true - metric_points=$(echo "$response" | jq -r '.metrics[0].metricPoints[] | .gaugeValue.doubleValue') - # Check if metric points are empty - if [ -z "$metric_points" ]; then - echo "Metric points for service $service are empty" - exit 1 - fi - for value in $metric_points; do - if (( $(echo "$value == 0" | bc -l) )); then - all_non_zero=false - break - fi - done - - if [ "$all_non_zero" = true ]; then - echo "All gauge values are non-zero" - else - echo "Some gauge values are zero" - exit 1 - fi - done - -} - -# Function to tear down Docker Compose services -teardown_services() { - (cd docker-compose/monitor && make clean-all) -} - -# Main function -main() { - (cd docker-compose/monitor && make dev) - wait_for_services - check_spm - echo "All services are running correctly" - teardown_services -} -trap teardown_services EXIT INT -# Run the main function -main From 98e2a99f9466b273afaf066e7718a1a9795dc22f Mon Sep 17 00:00:00 2001 From: Yuri Shkuro Date: Wed, 19 Jun 2024 14:02:52 -0400 Subject: [PATCH 7/9] fix Signed-off-by: Yuri Shkuro --- .github/workflows/ci-build-spm.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-build-spm.yml b/.github/workflows/ci-build-spm.yml index 8fed2baffa6..11ce0e93fc9 100644 --- a/.github/workflows/ci-build-spm.yml +++ b/.github/workflows/ci-build-spm.yml @@ -16,7 +16,7 @@ permissions: contents: read jobs: - SPM: + spm: runs-on: ubuntu-latest steps: - name: Harden Runner From da665afe3cfabb9313d1979355b561a278b411a9 Mon Sep 17 00:00:00 2001 From: Yuri Shkuro Date: Wed, 19 Jun 2024 14:05:56 -0400 Subject: [PATCH 8/9] fix Signed-off-by: Yuri Shkuro --- .github/workflows/ci-build-spm.yml | 29 ++++++++++++++++++----------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/.github/workflows/ci-build-spm.yml b/.github/workflows/ci-build-spm.yml index 11ce0e93fc9..9300e439004 100644 --- a/.github/workflows/ci-build-spm.yml +++ b/.github/workflows/ci-build-spm.yml @@ -19,17 +19,24 @@ jobs: spm: runs-on: ubuntu-latest steps: - - name: Harden Runner - uses: step-security/harden-runner@17d0e2bd7d51742c71671bd19fa12bdc9d40a3d6 # v2.8.1 - with: - egress-policy: audit # TODO: change to 'egress-policy: block' after couple of runs + - name: Harden Runner + uses: step-security/harden-runner@17d0e2bd7d51742c71671bd19fa12bdc9d40a3d6 # v2.8.1 + with: + egress-policy: audit # TODO: change to 'egress-policy: block' after couple of runs - - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 + - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 - - uses: actions/setup-go@cdcb36043654635271a94b9a6d1392de5bb323a7 # v5.0.1 - with: - go-version: 1.22.x + - name: Fetch git tags + run: | + git fetch --prune --unshallow --tags - - name: Temporary - only run the build - run: - cd docker-compose/monitor && make build + - uses: actions/setup-go@cdcb36043654635271a94b9a6d1392de5bb323a7 # v5.0.1 + with: + go-version: 1.22.x + + - name: Setup Node.js version + uses: ./.github/actions/setup-node.js + + - name: Temporary - only run the build + run: + cd docker-compose/monitor && make build From 834c684ac0d0dae3412e29fba6b0f017d8692126 Mon Sep 17 00:00:00 2001 From: Yuri Shkuro Date: Wed, 19 Jun 2024 14:07:43 -0400 Subject: [PATCH 9/9] fix Signed-off-by: Yuri Shkuro --- .github/workflows/ci-build-spm.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci-build-spm.yml b/.github/workflows/ci-build-spm.yml index 9300e439004..30a1644dfc1 100644 --- a/.github/workflows/ci-build-spm.yml +++ b/.github/workflows/ci-build-spm.yml @@ -24,7 +24,9 @@ jobs: with: egress-policy: audit # TODO: change to 'egress-policy: block' after couple of runs - - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 + - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + with: + submodules: true - name: Fetch git tags run: |