✨ add metrics endpoints to fastapi apps for promethius scraping (IT…

…ISFoundation#5217)

packages/service-library/requirements/_aiohttp.txt

@@ -105,7 +105,7 @@ parse==1.19.1
     # via openapi-core
 pathable==0.4.3
     # via jsonschema-spec
-prometheus-client==0.18.0
+prometheus-client==0.19.0
     # via -r requirements/_aiohttp.in
 psycopg2-binary==2.9.9
     # via

packages/service-library/requirements/_fastapi.in

@@ -8,4 +8,6 @@
 
 fastapi
 httpx
+prometheus-client
+prometheus-fastapi-instrumentator
 uvicorn

packages/service-library/requirements/_fastapi.txt

@@ -39,6 +39,7 @@ fastapi==0.99.1
     #   -c requirements/./../../../packages/settings-library/requirements/../../../requirements/constraints.txt
     #   -c requirements/./../../../requirements/constraints.txt
     #   -r requirements/_fastapi.in
+    #   prometheus-fastapi-instrumentator
 h11==0.14.0
     # via
     #   httpcore
@@ -66,6 +67,12 @@ mdurl==0.1.2
     # via markdown-it-py
 orjson==3.9.10
     # via -r requirements/./../../../packages/models-library/requirements/_base.in
+prometheus-client==0.19.0
+    # via
+    #   -r requirements/_fastapi.in
+    #   prometheus-fastapi-instrumentator
+prometheus-fastapi-instrumentator==6.1.0
+    # via -r requirements/_fastapi.in
 pydantic==1.10.13
     # via
     #   -c requirements/./../../../packages/models-library/requirements/../../../requirements/constraints.txt

packages/service-library/src/servicelib/fastapi/prometheus_instrumentation.py

@@ -0,0 +1,23 @@
+# pylint: disable=protected-access
+
+
+from fastapi import FastAPI
+from prometheus_fastapi_instrumentator import Instrumentator
+
+
+def setup_prometheus_instrumentation(app: FastAPI):
+
+    instrumentator = (
+        Instrumentator(
+            should_instrument_requests_inprogress=True, inprogress_labels=False
+        )
+        .instrument(app)
+        .expose(app, include_in_schema=False)
+    )
+
+    def _unregister():
+        for collector in list(instrumentator.registry._collector_to_names.keys()):
+            instrumentator.registry.unregister(collector)
+
+    # avoid registering collectors multiple times when running unittests consecutively (https://stackoverflow.com/a/62489287)
+    app.add_event_handler("shutdown", _unregister)

services/agent/requirements/_base.in

@@ -8,7 +8,7 @@
 # intra-repo required dependencies
 --requirement ../../../packages/models-library/requirements/_base.in
 --requirement ../../../packages/settings-library/requirements/_base.in
---requirement ../../../packages/service-library/requirements/_base.in
+--requirement ../../../packages/service-library/requirements/_fastapi.in
 
 aiodocker
 fastapi

services/agent/requirements/_base.txt

@@ -4,44 +4,46 @@
 #
 #    pip-compile --output-file=requirements/_base.txt --strip-extras requirements/_base.in
 #
-aio-pika==9.1.2
-    # via -r requirements/../../../packages/service-library/requirements/_base.in
-aiodebug==2.3.0
-    # via -r requirements/../../../packages/service-library/requirements/_base.in
 aiodocker==0.21.0
     # via
-    #   -r requirements/../../../packages/service-library/requirements/_base.in
+    #   -c requirements/../../../packages/service-library/requirements/./_base.in
     #   -r requirements/_base.in
-aiofiles==22.1.0
-    # via -r requirements/../../../packages/service-library/requirements/_base.in
 aiohttp==3.8.5
     # via
     #   -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt
-    #   -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt
-    #   -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt
-    #   -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt
+    #   -c requirements/../../../packages/service-library/requirements/./../../../packages/models-library/requirements/../../../requirements/constraints.txt
+    #   -c requirements/../../../packages/service-library/requirements/./../../../packages/settings-library/requirements/../../../requirements/constraints.txt
+    #   -c requirements/../../../packages/service-library/requirements/./../../../requirements/constraints.txt
     #   -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt
     #   -c requirements/../../../requirements/constraints.txt
     #   aiodocker
-aiormq==6.7.6
-    # via aio-pika
 aiosignal==1.2.0
     # via aiohttp
 anyio==3.6.2
-    # via starlette
+    # via
+    #   httpx
+    #   starlette
 arrow==1.2.3
     # via
+    #   -c requirements/../../../packages/service-library/requirements/./_base.in
     #   -r requirements/../../../packages/models-library/requirements/_base.in
-    #   -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in
-    #   -r requirements/../../../packages/service-library/requirements/_base.in
+    #   -r requirements/../../../packages/service-library/requirements/./../../../packages/models-library/requirements/_base.in
 async-timeout==4.0.2
-    # via
-    #   aiohttp
-    #   redis
+    # via aiohttp
 attrs==21.4.0
     # via
     #   aiohttp
     #   jsonschema
+certifi==2023.11.17
+    # via
+    #   -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt
+    #   -c requirements/../../../packages/service-library/requirements/./../../../packages/models-library/requirements/../../../requirements/constraints.txt
+    #   -c requirements/../../../packages/service-library/requirements/./../../../packages/settings-library/requirements/../../../requirements/constraints.txt
+    #   -c requirements/../../../packages/service-library/requirements/./../../../requirements/constraints.txt
+    #   -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt
+    #   -c requirements/../../../requirements/constraints.txt
+    #   httpcore
+    #   httpx
 charset-normalizer==2.1.1
     # via aiohttp
 click==8.1.3
@@ -55,27 +57,43 @@ email-validator==1.3.0
 fastapi==0.96.0
     # via
     #   -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt
-    #   -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt
-    #   -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt
-    #   -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt
+    #   -c requirements/../../../packages/service-library/requirements/./../../../packages/models-library/requirements/../../../requirements/constraints.txt
+    #   -c requirements/../../../packages/service-library/requirements/./../../../packages/settings-library/requirements/../../../requirements/constraints.txt
+    #   -c requirements/../../../packages/service-library/requirements/./../../../requirements/constraints.txt
     #   -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt
     #   -c requirements/../../../requirements/constraints.txt
+    #   -r requirements/../../../packages/service-library/requirements/_fastapi.in
     #   -r requirements/_base.in
+    #   prometheus-fastapi-instrumentator
 frozenlist==1.3.1
     # via
     #   aiohttp
     #   aiosignal
 h11==0.14.0
-    # via uvicorn
+    # via
+    #   httpcore
+    #   uvicorn
+httpcore==1.0.2
+    # via httpx
+httpx==0.26.0
+    # via
+    #   -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt
+    #   -c requirements/../../../packages/service-library/requirements/./../../../packages/models-library/requirements/../../../requirements/constraints.txt
+    #   -c requirements/../../../packages/service-library/requirements/./../../../packages/settings-library/requirements/../../../requirements/constraints.txt
+    #   -c requirements/../../../packages/service-library/requirements/./../../../requirements/constraints.txt
+    #   -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt
+    #   -c requirements/../../../requirements/constraints.txt
+    #   -r requirements/../../../packages/service-library/requirements/_fastapi.in
 idna==3.4
     # via
     #   anyio
     #   email-validator
+    #   httpx
     #   yarl
 jsonschema==3.2.0
     # via
     #   -r requirements/../../../packages/models-library/requirements/_base.in
-    #   -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in
+    #   -r requirements/../../../packages/service-library/requirements/./../../../packages/models-library/requirements/_base.in
 markdown-it-py==3.0.0
     # via rich
 mdurl==0.1.2
@@ -87,95 +105,71 @@ multidict==6.0.2
 orjson==3.9.7
     # via
     #   -r requirements/../../../packages/models-library/requirements/_base.in
-    #   -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in
+    #   -r requirements/../../../packages/service-library/requirements/./../../../packages/models-library/requirements/_base.in
 packaging==23.1
     # via -r requirements/_base.in
-pamqp==3.2.1
-    # via aiormq
+prometheus-client==0.19.0
+    # via prometheus-fastapi-instrumentator
+prometheus-fastapi-instrumentator==6.1.0
+    # via -r requirements/../../../packages/service-library/requirements/_fastapi.in
 pydantic==1.10.2
     # via
     #   -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt
-    #   -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt
-    #   -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt
-    #   -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt
+    #   -c requirements/../../../packages/service-library/requirements/./../../../packages/models-library/requirements/../../../requirements/constraints.txt
+    #   -c requirements/../../../packages/service-library/requirements/./../../../packages/settings-library/requirements/../../../requirements/constraints.txt
+    #   -c requirements/../../../packages/service-library/requirements/./../../../requirements/constraints.txt
+    #   -c requirements/../../../packages/service-library/requirements/./_base.in
     #   -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt
     #   -c requirements/../../../requirements/constraints.txt
     #   -r requirements/../../../packages/models-library/requirements/_base.in
-    #   -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in
-    #   -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in
-    #   -r requirements/../../../packages/service-library/requirements/_base.in
+    #   -r requirements/../../../packages/service-library/requirements/./../../../packages/models-library/requirements/_base.in
+    #   -r requirements/../../../packages/service-library/requirements/./../../../packages/settings-library/requirements/_base.in
     #   -r requirements/../../../packages/settings-library/requirements/_base.in
     #   -r requirements/_base.in
     #   fastapi
 pygments==2.15.1
     # via rich
-pyinstrument==4.3.0
-    # via -r requirements/../../../packages/service-library/requirements/_base.in
 pyrsistent==0.19.2
     # via jsonschema
 python-dateutil==2.8.2
     # via arrow
 python-dotenv==1.0.0
     # via -r requirements/_base.in
-pyyaml==6.0.1
-    # via
-    #   -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt
-    #   -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt
-    #   -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt
-    #   -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt
-    #   -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt
-    #   -c requirements/../../../requirements/constraints.txt
-    #   -r requirements/../../../packages/service-library/requirements/_base.in
-redis==4.5.4
-    # via
-    #   -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt
-    #   -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt
-    #   -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt
-    #   -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt
-    #   -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt
-    #   -c requirements/../../../requirements/constraints.txt
-    #   -r requirements/../../../packages/service-library/requirements/_base.in
 rich==13.4.2
     # via
-    #   -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in
+    #   -r requirements/../../../packages/service-library/requirements/./../../../packages/settings-library/requirements/_base.in
     #   -r requirements/../../../packages/settings-library/requirements/_base.in
 six==1.16.0
     # via
     #   jsonschema
     #   python-dateutil
 sniffio==1.3.0
-    # via anyio
+    # via
+    #   anyio
+    #   httpx
 starlette==0.27.0
     # via
     #   -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt
-    #   -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt
-    #   -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt
-    #   -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt
+    #   -c requirements/../../../packages/service-library/requirements/./../../../packages/models-library/requirements/../../../requirements/constraints.txt
+    #   -c requirements/../../../packages/service-library/requirements/./../../../packages/settings-library/requirements/../../../requirements/constraints.txt
+    #   -c requirements/../../../packages/service-library/requirements/./../../../requirements/constraints.txt
     #   -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt
     #   -c requirements/../../../requirements/constraints.txt
     #   fastapi
-tenacity==8.1.0
-    # via -r requirements/../../../packages/service-library/requirements/_base.in
-toolz==0.12.0
-    # via -r requirements/../../../packages/service-library/requirements/_base.in
-tqdm==4.64.1
-    # via -r requirements/../../../packages/service-library/requirements/_base.in
 typer==0.6.1
     # via
-    #   -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in
+    #   -r requirements/../../../packages/service-library/requirements/./../../../packages/settings-library/requirements/_base.in
     #   -r requirements/../../../packages/settings-library/requirements/_base.in
 typing-extensions==4.4.0
     # via
-    #   aiodebug
     #   aiodocker
     #   pydantic
 uvicorn==0.19.0
-    # via -r requirements/_base.in
-yarl==1.9.2
     # via
-    #   aio-pika
-    #   aiohttp
-    #   aiormq
+    #   -r requirements/../../../packages/service-library/requirements/_fastapi.in
+    #   -r requirements/_base.in
+yarl==1.9.2
+    # via aiohttp
 
 # The following packages are considered to be unsafe in a requirements file:
 # setuptools

services/agent/requirements/_test.txt

@@ -26,7 +26,7 @@ aiosignal==1.2.0
 anyio==3.6.2
     # via
     #   -c requirements/_base.txt
-    #   httpcore
+    #   httpx
 async-timeout==4.0.2
     # via
     #   -c requirements/_base.txt
@@ -57,9 +57,10 @@ botocore==1.24.21
     #   boto3
     #   moto
     #   s3transfer
-certifi==2023.7.22
+certifi==2023.11.17
     # via
     #   -c requirements/../../../requirements/constraints.txt
+    #   -c requirements/_base.txt
     #   httpcore
     #   httpx
     #   requests
@@ -114,11 +115,14 @@ h11==0.14.0
     # via
     #   -c requirements/_base.txt
     #   httpcore
-httpcore==0.18.0
-    # via httpx
-httpx==0.25.0
+httpcore==1.0.2
+    # via
+    #   -c requirements/_base.txt
+    #   httpx
+httpx==0.26.0
     # via
     #   -c requirements/../../../requirements/constraints.txt
+    #   -c requirements/_base.txt
     #   -r requirements/_test.in
 idna==3.4
     # via
@@ -237,7 +241,6 @@ python-jose==3.3.0
 pyyaml==6.0.1
     # via
     #   -c requirements/../../../requirements/constraints.txt
-    #   -c requirements/_base.txt
     #   cfn-lint
     #   moto
     #   openapi-spec-validator
@@ -270,7 +273,6 @@ sniffio==1.3.0
     # via
     #   -c requirements/_base.txt
     #   anyio
-    #   httpcore
     #   httpx
 sshpubkeys==3.3.1
     # via moto

services/agent/requirements/_tools.txt

@@ -108,7 +108,6 @@ pyproject-hooks==1.0.0
 pyyaml==6.0.1
     # via
     #   -c requirements/../../../requirements/constraints.txt
-    #   -c requirements/_base.txt
     #   -c requirements/_test.txt
     #   pre-commit
     #   watchdog

services/agent/src/simcore_service_agent/core/application.py

@@ -5,6 +5,9 @@
     get_common_oas_options,
     override_fastapi_openapi_method,
 )
+from servicelib.fastapi.prometheus_instrumentation import (
+    setup_prometheus_instrumentation,
+)
 from servicelib.logging_utils import config_all_loggers
 
 from .._meta import (
@@ -47,6 +50,9 @@ def create_app() -> FastAPI:
     override_fastapi_openapi_method(app)
     app.state.settings = settings
 
+    if app.state.settings.AGENT_PROMETHEUS_INSTRUMENTATION_ENABLED:
+        setup_prometheus_instrumentation(app)
+
     # ROUTERS
     app.include_router(router)
 

services/agent/src/simcore_service_agent/core/settings.py

@@ -46,6 +46,7 @@ class ApplicationSettings(BaseCustomSettings, MixinLoggingSettings):
     AGENT_VOLUMES_CLEANUP_INTERVAL_S: NonNegativeInt = Field(
         60 * _MINUTE, description="interval at which to repeat volumes cleanup"
     )
+    AGENT_PROMETHEUS_INSTRUMENTATION_ENABLED: bool = True
 
     @validator("AGENT_VOLUMES_CLEANUP_S3_ENDPOINT", pre=True)
     @classmethod