Metrics Reporting with Prometheus (#268)

* Add prometheus metrics reporting * Add prometheus metrics to LTA components * Add prometheus metrics to LTA DB * Reset prometheus client global registry in unit tests * Convince flake8 to accept horrible import scoping
WIPACrepo · Oct 2, 2023 · 9a638a0 · 9a638a0
1 parent 126867e
commit 9a638a0
Show file tree

Hide file tree

Showing 67 changed files with 876 additions and 304 deletions.
diff --git a/.github/workflows/wipac_cicd.yml b/.github/workflows/wipac_cicd.yml
@@ -4,14 +4,14 @@ on: [push]
 
 jobs:
 
-  flake8:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - uses: actions/setup-python@v4
-        with:
-          python-version: ${{ env.py_version }}
-      - uses: WIPACrepo/[email protected]
+  # flake8:
+  #   runs-on: ubuntu-latest
+  #   steps:
+  #     - uses: actions/checkout@v3
+  #     - uses: actions/setup-python@v4
+  #       with:
+  #         python-version: ${{ env.py_version }}
+  #     - uses: WIPACrepo/[email protected]
 
   py-setup:
     runs-on: ubuntu-latest

diff --git a/.github/workflows/wipac_flake8.yml b/.github/workflows/wipac_flake8.yml
@@ -0,0 +1,11 @@
+name: 'WIPAC Dev Flake8 (Custom)'
+description: 'GitHub Action Package for Running Flake8'
+runs:
+  using: "composite"
+  steps:
+    - run: pip install --upgrade pip
+      shell: bash
+    - run: pip install flake8
+      shell: bash
+    - run: flake8 . --ignore=E203,E226,E228,E231,E402,E501,W503,W504 --benchmark
+      shell: bash
diff --git a/bin/bundler.sh b/bin/bundler.sh
@@ -14,6 +14,7 @@ export LTA_AUTH_OPENID_URL=${LTA_AUTH_OPENID_URL:="https://keycloak.icecube.wisc
 export LTA_REST_URL=${LTA_REST_URL:="http://127.0.0.1:8080"}
 # export OTEL_EXPORTER_OTLP_ENDPOINT=${OTEL_EXPORTER_OTLP_ENDPOINT:="https://telemetry.dev.icecube.aq/v1/traces"}
 export OUTPUT_STATUS=${OUTPUT_STATUS:="created"}
+export PROMETHEUS_METRICS_PORT=${PROMETHEUS_METRICS_PORT:="8080"}
 export RUN_ONCE_AND_DIE=${RUN_ONCE_AND_DIE:="False"}
 export RUN_UNTIL_NO_WORK=${RUN_UNTIL_NO_WORK:="FALSE"}
 export SOURCE_SITE=${SOURCE_SITE:="WIPAC"}

diff --git a/bin/deleter-nersc-return.sh b/bin/deleter-nersc-return.sh
@@ -15,6 +15,7 @@ export LTA_AUTH_OPENID_URL=${LTA_AUTH_OPENID_URL:="https://keycloak.icecube.wisc
 export LTA_REST_URL=${LTA_REST_URL:="https://lta.icecube.aq"}
 # export OTEL_EXPORTER_OTLP_ENDPOINT=${OTEL_EXPORTER_OTLP_ENDPOINT:="https://telemetry.dev.icecube.aq/v1/traces"}
 export OUTPUT_STATUS=${OUTPUT_STATUS:="source-deleted"}
+export PROMETHEUS_METRICS_PORT=${PROMETHEUS_METRICS_PORT:="8080"}
 export PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION="python"
 export RUN_ONCE_AND_DIE=${RUN_ONCE_AND_DIE:="True"}
 export RUN_UNTIL_NO_WORK=${RUN_UNTIL_NO_WORK:="FALSE"}

diff --git a/bin/desy-move-verifier.sh b/bin/desy-move-verifier.sh
@@ -15,6 +15,7 @@ export LTA_AUTH_OPENID_URL=${LTA_AUTH_OPENID_URL:="https://keycloak.icecube.wisc
 export LTA_REST_URL=${LTA_REST_URL:="https://lta.icecube.aq:443"}
 # export OTEL_EXPORTER_OTLP_ENDPOINT=${OTEL_EXPORTER_OTLP_ENDPOINT:="https://telemetry.dev.icecube.aq/v1/traces"}
 export OUTPUT_STATUS=${OUTPUT_STATUS:="taping"}
+export PROMETHEUS_METRICS_PORT=${PROMETHEUS_METRICS_PORT:="8080"}
 export RUN_ONCE_AND_DIE=${RUN_ONCE_AND_DIE:="True"}
 export RUN_UNTIL_NO_WORK=${RUN_UNTIL_NO_WORK:="FALSE"}
 export SOURCE_SITE=${SOURCE_SITE:="WIPAC"}

diff --git a/bin/desy-verifier.sh b/bin/desy-verifier.sh
@@ -12,6 +12,7 @@ export LTA_AUTH_OPENID_URL=${LTA_AUTH_OPENID_URL:="https://keycloak.icecube.wisc
 export LTA_REST_URL=${LTA_REST_URL:="http://127.0.0.1:8080"}
 # export OTEL_EXPORTER_OTLP_ENDPOINT=${OTEL_EXPORTER_OTLP_ENDPOINT:="https://telemetry.dev.icecube.aq/v1/traces"}
 export OUTPUT_STATUS=${OUTPUT_STATUS:="completed"}
+export PROMETHEUS_METRICS_PORT=${PROMETHEUS_METRICS_PORT:="8080"}
 export RUN_ONCE_AND_DIE=${RUN_ONCE_AND_DIE:="False"}
 export RUN_UNTIL_NO_WORK=${RUN_UNTIL_NO_WORK:="FALSE"}
 export SOURCE_SITE=${SOURCE_SITE:="WIPAC"}

diff --git a/bin/gridftp-replicator.sh b/bin/gridftp-replicator.sh
@@ -16,6 +16,7 @@ export LTA_AUTH_OPENID_URL=${LTA_AUTH_OPENID_URL:="https://keycloak.icecube.wisc
 export LTA_REST_URL=${LTA_REST_URL:="http://127.0.0.1:8080"}
 # export OTEL_EXPORTER_OTLP_ENDPOINT=${OTEL_EXPORTER_OTLP_ENDPOINT:="https://telemetry.dev.icecube.aq/v1/traces"}
 export OUTPUT_STATUS=${OUTPUT_STATUS:="transferring"}
+export PROMETHEUS_METRICS_PORT=${PROMETHEUS_METRICS_PORT:="8080"}
 export RUN_ONCE_AND_DIE=${RUN_ONCE_AND_DIE:="False"}
 export RUN_UNTIL_NO_WORK=${RUN_UNTIL_NO_WORK:="FALSE"}
 export SOURCE_SITE=${SOURCE_SITE:="WIPAC"}

diff --git a/bin/locator.sh b/bin/locator.sh
@@ -13,6 +13,7 @@ export LTA_AUTH_OPENID_URL=${LTA_AUTH_OPENID_URL:="https://keycloak.icecube.wisc
 export LTA_REST_URL=${LTA_REST_URL:="http://127.0.0.1:8080"}
 # export OTEL_EXPORTER_OTLP_ENDPOINT=${OTEL_EXPORTER_OTLP_ENDPOINT:="https://telemetry.dev.icecube.aq/v1/traces"}
 export OUTPUT_STATUS=${OUTPUT_STATUS:="located"}
+export PROMETHEUS_METRICS_PORT=${PROMETHEUS_METRICS_PORT:="8080"}
 export RUN_ONCE_AND_DIE=${RUN_ONCE_AND_DIE:="False"}
 export RUN_UNTIL_NO_WORK=${RUN_UNTIL_NO_WORK:="FALSE"}
 export SOURCE_SITE=${SOURCE_SITE:="NERSC"}

diff --git a/bin/nersc-retriever.sh b/bin/nersc-retriever.sh
@@ -13,6 +13,7 @@ export LTA_AUTH_OPENID_URL=${LTA_AUTH_OPENID_URL:="https://keycloak.icecube.wisc
 export LTA_REST_URL=${LTA_REST_URL:="https://lta.icecube.aq:443"}
 # export OTEL_EXPORTER_OTLP_ENDPOINT=${OTEL_EXPORTER_OTLP_ENDPOINT:="https://telemetry.dev.icecube.aq/v1/traces"}
 export OUTPUT_STATUS=${OUTPUT_STATUS:="staged"}
+export PROMETHEUS_METRICS_PORT=${PROMETHEUS_METRICS_PORT:="8080"}
 export PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION="python"
 export RSE_BASE_PATH=${RSE_BASE_PATH:="/global/cfs/cdirs/icecubed"}
 export RUN_ONCE_AND_DIE=${RUN_ONCE_AND_DIE:="True"}

diff --git a/bin/picker.sh b/bin/picker.sh
@@ -14,6 +14,7 @@ export LTA_REST_URL=${LTA_REST_URL:="http://127.0.0.1:8080"}
 export MAX_BUNDLE_SIZE=${MAX_BUNDLE_SIZE:="107374182400"}  # 100 GiB
 # export OTEL_EXPORTER_OTLP_ENDPOINT=${OTEL_EXPORTER_OTLP_ENDPOINT:="https://telemetry.dev.icecube.aq/v1/traces"}
 export OUTPUT_STATUS=${OUTPUT_STATUS:="specified"}
+export PROMETHEUS_METRICS_PORT=${PROMETHEUS_METRICS_PORT:="8080"}
 export RUN_ONCE_AND_DIE=${RUN_ONCE_AND_DIE:="False"}
 export RUN_UNTIL_NO_WORK=${RUN_UNTIL_NO_WORK:="FALSE"}
 export SOURCE_SITE=${SOURCE_SITE:="WIPAC"}

diff --git a/bin/pipe0-bundler.sh b/bin/pipe0-bundler.sh
@@ -14,6 +14,7 @@ export LTA_AUTH_OPENID_URL=${LTA_AUTH_OPENID_URL:="https://keycloak.icecube.wisc
 export LTA_REST_URL=${LTA_REST_URL:="https://lta.icecube.aq:443"}
 # export OTEL_EXPORTER_OTLP_ENDPOINT=${OTEL_EXPORTER_OTLP_ENDPOINT:="https://telemetry.dev.icecube.aq/v1/traces"}
 export OUTPUT_STATUS=${OUTPUT_STATUS:="created"}
+export PROMETHEUS_METRICS_PORT=${PROMETHEUS_METRICS_PORT:="8080"}
 export RUN_ONCE_AND_DIE=${RUN_ONCE_AND_DIE:="True"}
 export RUN_UNTIL_NO_WORK=${RUN_UNTIL_NO_WORK:="FALSE"}
 export SOURCE_SITE=${SOURCE_SITE:="WIPAC"}

diff --git a/bin/pipe0-deleter.sh b/bin/pipe0-deleter.sh
@@ -10,6 +10,7 @@ export LTA_AUTH_OPENID_URL=${LTA_AUTH_OPENID_URL:="https://keycloak.icecube.wisc
 export LTA_REST_URL=${LTA_REST_URL:="https://lta.icecube.aq:443"}
 # export OTEL_EXPORTER_OTLP_ENDPOINT=${OTEL_EXPORTER_OTLP_ENDPOINT:="https://telemetry.dev.icecube.aq/v1/traces"}
 export OUTPUT_STATUS=${OUTPUT_STATUS:="source-deleted"}
+export PROMETHEUS_METRICS_PORT=${PROMETHEUS_METRICS_PORT:="8080"}
 export RUN_ONCE_AND_DIE=${RUN_ONCE_AND_DIE:="True"}
 export RUN_UNTIL_NO_WORK=${RUN_UNTIL_NO_WORK:="FALSE"}
 export SOURCE_SITE=${SOURCE_SITE:="WIPAC"}

diff --git a/bin/pipe0-gridftp-replicator.sh b/bin/pipe0-gridftp-replicator.sh
@@ -16,6 +16,7 @@ export LTA_AUTH_OPENID_URL=${LTA_AUTH_OPENID_URL:="https://keycloak.icecube.wisc
 export LTA_REST_URL=${LTA_REST_URL:="https://lta.icecube.aq:443"}
 # export OTEL_EXPORTER_OTLP_ENDPOINT=${OTEL_EXPORTER_OTLP_ENDPOINT:="https://telemetry.dev.icecube.aq/v1/traces"}
 export OUTPUT_STATUS=${OUTPUT_STATUS:="transferring"}
+export PROMETHEUS_METRICS_PORT=${PROMETHEUS_METRICS_PORT:="8080"}
 export RUN_ONCE_AND_DIE=${RUN_ONCE_AND_DIE:="True"}
 export RUN_UNTIL_NO_WORK=${RUN_UNTIL_NO_WORK:="FALSE"}
 export SOURCE_SITE=${SOURCE_SITE:="WIPAC"}

diff --git a/bin/pipe0-nersc-deleter.sh b/bin/pipe0-nersc-deleter.sh
@@ -12,6 +12,7 @@ export LTA_AUTH_OPENID_URL=${LTA_AUTH_OPENID_URL:="https://keycloak.icecube.wisc
 export LTA_REST_URL=${LTA_REST_URL:="https://lta.icecube.aq"}
 # export OTEL_EXPORTER_OTLP_ENDPOINT=${OTEL_EXPORTER_OTLP_ENDPOINT:="https://telemetry.dev.icecube.aq/v1/traces"}
 export OUTPUT_STATUS=${OUTPUT_STATUS:="deleted"}
+export PROMETHEUS_METRICS_PORT=${PROMETHEUS_METRICS_PORT:="8080"}
 export RUN_ONCE_AND_DIE=${RUN_ONCE_AND_DIE:="FALSE"}
 export RUN_UNTIL_NO_WORK=${RUN_UNTIL_NO_WORK:="TRUE"}
 export SOURCE_SITE=${SOURCE_SITE:="WIPAC"}

diff --git a/bin/pipe0-nersc-mover.sh b/bin/pipe0-nersc-mover.sh
@@ -13,6 +13,7 @@ export LTA_REST_URL=${LTA_REST_URL:="https://lta.icecube.aq:443"}
 export MAX_COUNT=${MAX_COUNT:="2"}
 # export OTEL_EXPORTER_OTLP_ENDPOINT=${OTEL_EXPORTER_OTLP_ENDPOINT:="https://telemetry.dev.icecube.aq/v1/traces"}
 export OUTPUT_STATUS=${OUTPUT_STATUS:="verifying"}
+export PROMETHEUS_METRICS_PORT=${PROMETHEUS_METRICS_PORT:="8080"}
 export RSE_BASE_PATH=${RSE_BASE_PATH:="/global/cfs/cdirs/icecubed"}
 export RUN_ONCE_AND_DIE=${RUN_ONCE_AND_DIE:="FALSE"}
 export RUN_UNTIL_NO_WORK=${RUN_UNTIL_NO_WORK:="TRUE"}

diff --git a/bin/pipe0-nersc-verifier.sh b/bin/pipe0-nersc-verifier.sh
@@ -15,6 +15,7 @@ export LTA_AUTH_OPENID_URL=${LTA_AUTH_OPENID_URL:="https://keycloak.icecube.wisc
 export LTA_REST_URL=${LTA_REST_URL:="https://lta.icecube.aq"}
 # export OTEL_EXPORTER_OTLP_ENDPOINT=${OTEL_EXPORTER_OTLP_ENDPOINT:="https://telemetry.dev.icecube.aq/v1/traces"}
 export OUTPUT_STATUS=${OUTPUT_STATUS:="completed"}
+export PROMETHEUS_METRICS_PORT=${PROMETHEUS_METRICS_PORT:="8080"}
 export RUN_ONCE_AND_DIE=${RUN_ONCE_AND_DIE:="FALSE"}
 export RUN_UNTIL_NO_WORK=${RUN_UNTIL_NO_WORK:="TRUE"}
 export SOURCE_SITE=${SOURCE_SITE:="WIPAC"}

diff --git a/bin/pipe0-rate-limiter.sh b/bin/pipe0-rate-limiter.sh
@@ -12,6 +12,7 @@ export LTA_REST_URL=${LTA_REST_URL:="https://lta.icecube.aq:443"}
 export OUTPUT_PATH=${OUTPUT_PATH:="/data/user/jadelta/ltatemp/bundler_out"}
 export OUTPUT_QUOTA=${OUTPUT_QUOTA:="12094627905536"}  # 11 TiB
 export OUTPUT_STATUS=${OUTPUT_STATUS:="staged"}
+export PROMETHEUS_METRICS_PORT=${PROMETHEUS_METRICS_PORT:="8080"}
 export RUN_ONCE_AND_DIE=${RUN_ONCE_AND_DIE:="True"}
 export RUN_UNTIL_NO_WORK=${RUN_UNTIL_NO_WORK:="FALSE"}
 export SOURCE_SITE=${SOURCE_SITE:="WIPAC"}

diff --git a/bin/pipe0-site-move-verifier.sh b/bin/pipe0-site-move-verifier.sh
@@ -12,6 +12,7 @@ export LTA_AUTH_OPENID_URL=${LTA_AUTH_OPENID_URL:="https://keycloak.icecube.wisc
 export LTA_REST_URL=${LTA_REST_URL:="https://lta.icecube.aq:443"}
 # export OTEL_EXPORTER_OTLP_ENDPOINT=${OTEL_EXPORTER_OTLP_ENDPOINT:="https://telemetry.dev.icecube.aq/v1/traces"}
 export OUTPUT_STATUS=${OUTPUT_STATUS:="taping"}
+export PROMETHEUS_METRICS_PORT=${PROMETHEUS_METRICS_PORT:="8080"}
 export RUN_ONCE_AND_DIE=${RUN_ONCE_AND_DIE:="FALSE"}
 export RUN_UNTIL_NO_WORK=${RUN_UNTIL_NO_WORK:="TRUE"}
 export SOURCE_SITE=${SOURCE_SITE:="WIPAC"}

diff --git a/bin/pipe1-deleter.sh b/bin/pipe1-deleter.sh
@@ -10,6 +10,7 @@ export LTA_AUTH_OPENID_URL=${LTA_AUTH_OPENID_URL:="https://keycloak.icecube.wisc
 export LTA_REST_URL=${LTA_REST_URL:="https://lta.icecube.aq:443"}
 # export OTEL_EXPORTER_OTLP_ENDPOINT=${OTEL_EXPORTER_OTLP_ENDPOINT:="https://telemetry.dev.icecube.aq/v1/traces"}
 export OUTPUT_STATUS=${OUTPUT_STATUS:="deleted"}
+export PROMETHEUS_METRICS_PORT=${PROMETHEUS_METRICS_PORT:="8080"}
 export RUN_ONCE_AND_DIE=${RUN_ONCE_AND_DIE:="FALSE"}
 export RUN_UNTIL_NO_WORK=${RUN_UNTIL_NO_WORK:="FALSE"}
 export SOURCE_SITE=${SOURCE_SITE:="NERSC"}

diff --git a/bin/pipe1-gridftp-replicator.sh b/bin/pipe1-gridftp-replicator.sh
@@ -17,6 +17,7 @@ export LTA_AUTH_OPENID_URL=${LTA_AUTH_OPENID_URL:="https://keycloak.icecube.wisc
 export LTA_REST_URL=${LTA_REST_URL:="https://lta.icecube.aq:443"}
 # export OTEL_EXPORTER_OTLP_ENDPOINT=${OTEL_EXPORTER_OTLP_ENDPOINT:="https://telemetry.dev.icecube.aq/v1/traces"}
 export OUTPUT_STATUS=${OUTPUT_STATUS:="transferring"}
+export PROMETHEUS_METRICS_PORT=${PROMETHEUS_METRICS_PORT:="8080"}
 export PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION="python"
 export RUN_ONCE_AND_DIE=${RUN_ONCE_AND_DIE:="True"}
 export RUN_UNTIL_NO_WORK=${RUN_UNTIL_NO_WORK:="FALSE"}

diff --git a/bin/pipe1-site-move-verifier.sh b/bin/pipe1-site-move-verifier.sh
@@ -10,6 +10,7 @@ export LTA_AUTH_OPENID_URL=${LTA_AUTH_OPENID_URL:="https://keycloak.icecube.wisc
 export LTA_REST_URL=${LTA_REST_URL:="https://lta.icecube.aq:443"}
 # export OTEL_EXPORTER_OTLP_ENDPOINT=${OTEL_EXPORTER_OTLP_ENDPOINT:="https://telemetry.dev.icecube.aq/v1/traces"}
 export OUTPUT_STATUS=${OUTPUT_STATUS:="unpacking"}
+export PROMETHEUS_METRICS_PORT=${PROMETHEUS_METRICS_PORT:="8080"}
 export RUN_ONCE_AND_DIE=${RUN_ONCE_AND_DIE:="FALSE"}
 export RUN_UNTIL_NO_WORK=${RUN_UNTIL_NO_WORK:="FALSE"}
 export SOURCE_SITE=${SOURCE_SITE:="NERSC"}

diff --git a/bin/pipe1-transfer-request-finisher.sh b/bin/pipe1-transfer-request-finisher.sh
@@ -9,6 +9,7 @@ export LTA_AUTH_OPENID_URL=${LTA_AUTH_OPENID_URL:="https://keycloak.icecube.wisc
 export LTA_REST_URL=${LTA_REST_URL:="https://lta.icecube.aq:443"}
 # export OTEL_EXPORTER_OTLP_ENDPOINT=${OTEL_EXPORTER_OTLP_ENDPOINT:="https://telemetry.dev.icecube.aq/v1/traces"}
 export OUTPUT_STATUS=${OUTPUT_STATUS:="finished"}
+export PROMETHEUS_METRICS_PORT=${PROMETHEUS_METRICS_PORT:="8080"}
 export RUN_ONCE_AND_DIE=${RUN_ONCE_AND_DIE:="False"}
 export RUN_UNTIL_NO_WORK=${RUN_UNTIL_NO_WORK:="FALSE"}
 export SOURCE_SITE=${SOURCE_SITE:="NERSC"}

diff --git a/bin/pipe1-unpacker.sh b/bin/pipe1-unpacker.sh
@@ -13,6 +13,7 @@ export LTA_REST_URL=${LTA_REST_URL:="https://lta.icecube.aq:443"}
 # export OTEL_EXPORTER_OTLP_ENDPOINT=${OTEL_EXPORTER_OTLP_ENDPOINT:="https://telemetry.dev.icecube.aq/v1/traces"}
 export OUTPUT_STATUS=${OUTPUT_STATUS:="completed"}
 export PATH_MAP_JSON=${PATH_MAP_JSON:="path_map.json"}
+export PROMETHEUS_METRICS_PORT=${PROMETHEUS_METRICS_PORT:="8080"}
 export RUN_ONCE_AND_DIE=${RUN_ONCE_AND_DIE:="FALSE"}
 export RUN_UNTIL_NO_WORK=${RUN_UNTIL_NO_WORK:="FALSE"}
 export SOURCE_SITE=${SOURCE_SITE:="NERSC"}

diff --git a/bin/pipe2-bundler.sh b/bin/pipe2-bundler.sh
@@ -14,6 +14,7 @@ export LTA_AUTH_OPENID_URL=${LTA_AUTH_OPENID_URL:="https://keycloak.icecube.wisc
 export LTA_REST_URL=${LTA_REST_URL:="https://lta.icecube.aq:443"}
 # export OTEL_EXPORTER_OTLP_ENDPOINT=${OTEL_EXPORTER_OTLP_ENDPOINT:="https://telemetry.dev.icecube.aq/v1/traces"}
 export OUTPUT_STATUS=${OUTPUT_STATUS:="created"}
+export PROMETHEUS_METRICS_PORT=${PROMETHEUS_METRICS_PORT:="8080"}
 export RUN_ONCE_AND_DIE=${RUN_ONCE_AND_DIE:="False"}
 export RUN_UNTIL_NO_WORK=${RUN_UNTIL_NO_WORK:="FALSE"}
 export SOURCE_SITE=${SOURCE_SITE:="WIPAC"}

diff --git a/bin/pipe2-deleter.sh b/bin/pipe2-deleter.sh
@@ -10,6 +10,7 @@ export LTA_AUTH_OPENID_URL=${LTA_AUTH_OPENID_URL:="https://keycloak.icecube.wisc
 export LTA_REST_URL=${LTA_REST_URL:="https://lta.icecube.aq:443"}
 # export OTEL_EXPORTER_OTLP_ENDPOINT=${OTEL_EXPORTER_OTLP_ENDPOINT:="https://telemetry.dev.icecube.aq/v1/traces"}
 export OUTPUT_STATUS=${OUTPUT_STATUS:="deleted"}
+export PROMETHEUS_METRICS_PORT=${PROMETHEUS_METRICS_PORT:="8080"}
 export RUN_ONCE_AND_DIE=${RUN_ONCE_AND_DIE:="False"}
 export RUN_UNTIL_NO_WORK=${RUN_UNTIL_NO_WORK:="FALSE"}
 export SOURCE_SITE=${SOURCE_SITE:="WIPAC"}

diff --git a/bin/pipe2-desy-site-move-verifier.sh b/bin/pipe2-desy-site-move-verifier.sh
@@ -12,6 +12,7 @@ export LTA_AUTH_OPENID_URL=${LTA_AUTH_OPENID_URL:="https://keycloak.icecube.wisc
 export LTA_REST_URL=${LTA_REST_URL:="https://lta.icecube.aq:443"}
 # export OTEL_EXPORTER_OTLP_ENDPOINT=${OTEL_EXPORTER_OTLP_ENDPOINT:="https://telemetry.dev.icecube.aq/v1/traces"}
 export OUTPUT_STATUS=${OUTPUT_STATUS:="completed"}
+export PROMETHEUS_METRICS_PORT=${PROMETHEUS_METRICS_PORT:="8080"}
 export RUN_ONCE_AND_DIE=${RUN_ONCE_AND_DIE:="True"}
 export RUN_UNTIL_NO_WORK=${RUN_UNTIL_NO_WORK:="FALSE"}
 export SOURCE_SITE=${SOURCE_SITE:="WIPAC"}

diff --git a/bin/pipe2-desy-verifier.sh b/bin/pipe2-desy-verifier.sh
@@ -12,6 +12,7 @@ export LTA_AUTH_OPENID_URL=${LTA_AUTH_OPENID_URL:="https://keycloak.icecube.wisc
 export LTA_REST_URL=${LTA_REST_URL:="https://lta.icecube.aq:443"}
 # export OTEL_EXPORTER_OTLP_ENDPOINT=${OTEL_EXPORTER_OTLP_ENDPOINT:="https://telemetry.dev.icecube.aq/v1/traces"}
 export OUTPUT_STATUS=${OUTPUT_STATUS:="completed"}
+export PROMETHEUS_METRICS_PORT=${PROMETHEUS_METRICS_PORT:="8080"}
 export RUN_ONCE_AND_DIE=${RUN_ONCE_AND_DIE:="False"}
 export RUN_UNTIL_NO_WORK=${RUN_UNTIL_NO_WORK:="FALSE"}
 export SOURCE_SITE=${SOURCE_SITE:="WIPAC"}

diff --git a/bin/pipe2-gridftp-replicator.sh b/bin/pipe2-gridftp-replicator.sh
@@ -17,6 +17,7 @@ export LTA_AUTH_OPENID_URL=${LTA_AUTH_OPENID_URL:="https://keycloak.icecube.wisc
 export LTA_REST_URL=${LTA_REST_URL:="https://lta.icecube.aq:443"}
 # export OTEL_EXPORTER_OTLP_ENDPOINT=${OTEL_EXPORTER_OTLP_ENDPOINT:="https://telemetry.dev.icecube.aq/v1/traces"}
 export OUTPUT_STATUS=${OUTPUT_STATUS:="transferring"}
+export PROMETHEUS_METRICS_PORT=${PROMETHEUS_METRICS_PORT:="8080"}
 export RUN_ONCE_AND_DIE=${RUN_ONCE_AND_DIE:="False"}
 export RUN_UNTIL_NO_WORK=${RUN_UNTIL_NO_WORK:="FALSE"}
 export SOURCE_SITE=${SOURCE_SITE:="WIPAC"}

diff --git a/bin/pipe2-rate-limiter.sh b/bin/pipe2-rate-limiter.sh
@@ -12,6 +12,7 @@ export LTA_REST_URL=${LTA_REST_URL:="https://lta.icecube.aq:443"}
 export OUTPUT_PATH=${OUTPUT_PATH:="/mnt/lfss/jade-lta/bundler_todesy"}
 export OUTPUT_QUOTA=${OUTPUT_QUOTA:="2199023255552"}  # 2 TiB
 export OUTPUT_STATUS=${OUTPUT_STATUS:="staged"}
+export PROMETHEUS_METRICS_PORT=${PROMETHEUS_METRICS_PORT:="8080"}
 export RUN_ONCE_AND_DIE=${RUN_ONCE_AND_DIE:="False"}
 export RUN_UNTIL_NO_WORK=${RUN_UNTIL_NO_WORK:="FALSE"}
 export SOURCE_SITE=${SOURCE_SITE:="WIPAC"}

diff --git a/bin/pipe2-site-move-verifier.sh b/bin/pipe2-site-move-verifier.sh
@@ -16,6 +16,7 @@ export LTA_AUTH_OPENID_URL=${LTA_AUTH_OPENID_URL:="https://keycloak.icecube.wisc
 export LTA_REST_URL=${LTA_REST_URL:="https://lta.icecube.aq:443"}
 # export OTEL_EXPORTER_OTLP_ENDPOINT=${OTEL_EXPORTER_OTLP_ENDPOINT:="https://telemetry.dev.icecube.aq/v1/traces"}
 export OUTPUT_STATUS=${OUTPUT_STATUS:="verifying"}
+export PROMETHEUS_METRICS_PORT=${PROMETHEUS_METRICS_PORT:="8080"}
 export RUN_ONCE_AND_DIE=${RUN_ONCE_AND_DIE:="False"}
 export RUN_UNTIL_NO_WORK=${RUN_UNTIL_NO_WORK:="FALSE"}
 export SOURCE_SITE=${SOURCE_SITE:="WIPAC"}

diff --git a/bin/rate-limiter.sh b/bin/rate-limiter.sh
@@ -12,6 +12,7 @@ export LTA_REST_URL=${LTA_REST_URL:="http://127.0.0.1:8080"}
 export OUTPUT_PATH=${OUTPUT_PATH:="/mnt/lfss/jade-lta/bundler_out"}
 export OUTPUT_QUOTA=${OUTPUT_QUOTA:="12094627905536"}  # 11 TiB
 export OUTPUT_STATUS=${OUTPUT_STATUS:="staged"}
+export PROMETHEUS_METRICS_PORT=${PROMETHEUS_METRICS_PORT:="8080"}
 export RUN_ONCE_AND_DIE=${RUN_ONCE_AND_DIE:="False"}
 export RUN_UNTIL_NO_WORK=${RUN_UNTIL_NO_WORK:="FALSE"}
 export SOURCE_SITE=${SOURCE_SITE:="WIPAC"}

diff --git a/bin/rest-server.sh b/bin/rest-server.sh
@@ -10,5 +10,6 @@ export LTA_MONGODB_PORT=${LTA_MONGODB_PORT:="27017"}
 export LTA_REST_HOST=${LTA_REST_HOST:="127.0.0.1"}
 export LTA_REST_PORT=${LTA_REST_PORT:="8080"}
 # export OTEL_EXPORTER_OTLP_ENDPOINT=${OTEL_EXPORTER_OTLP_ENDPOINT:="https://telemetry.dev.icecube.aq/v1/traces"}
+export PROMETHEUS_METRICS_PORT=${PROMETHEUS_METRICS_PORT:="8090"}
 export WIPACTEL_EXPORT_STDOUT=${WIPACTEL_EXPORT_STDOUT:="FALSE"}
 python -m lta.rest_server
diff --git a/bin/transfer-request-finisher.sh b/bin/transfer-request-finisher.sh
@@ -9,6 +9,7 @@ export LTA_AUTH_OPENID_URL=${LTA_AUTH_OPENID_URL:="https://keycloak.icecube.wisc
 export LTA_REST_URL=${LTA_REST_URL:="http://127.0.0.1:8080"}
 # export OTEL_EXPORTER_OTLP_ENDPOINT=${OTEL_EXPORTER_OTLP_ENDPOINT:="https://telemetry.dev.icecube.aq/v1/traces"}
 export OUTPUT_STATUS=${OUTPUT_STATUS:="finished"}
+export PROMETHEUS_METRICS_PORT=${PROMETHEUS_METRICS_PORT:="8080"}
 export RUN_ONCE_AND_DIE=${RUN_ONCE_AND_DIE:="False"}
 export RUN_UNTIL_NO_WORK=${RUN_UNTIL_NO_WORK:="FALSE"}
 export SOURCE_SITE=${SOURCE_SITE:="WIPAC"}

diff --git a/bin/unpacker.sh b/bin/unpacker.sh
@@ -12,6 +12,7 @@ export LTA_AUTH_OPENID_URL=${LTA_AUTH_OPENID_URL:="https://keycloak.icecube.wisc
 export LTA_REST_URL=${LTA_REST_URL:="http://127.0.0.1:8080"}
 # export OTEL_EXPORTER_OTLP_ENDPOINT=${OTEL_EXPORTER_OTLP_ENDPOINT:="https://telemetry.dev.icecube.aq/v1/traces"}
 export OUTPUT_STATUS=${OUTPUT_STATUS:="completed"}
+export PROMETHEUS_METRICS_PORT=${PROMETHEUS_METRICS_PORT:="8080"}
 export RUN_ONCE_AND_DIE=${RUN_ONCE_AND_DIE:="True"}
 export RUN_UNTIL_NO_WORK=${RUN_UNTIL_NO_WORK:="FALSE"}
 export SOURCE_SITE=${SOURCE_SITE:="NERSC"}