🎨 Tracing: Add more autoinstrumentation, enhance setup_tracing() - …

…DON'T DELETE PR BRANCH (#6561) Signed-off-by: dependabot[bot] <[email protected]> Co-authored-by: Dustin Kaiser <[email protected]> Co-authored-by: Andrei Neagu <[email protected]> Co-authored-by: Andrei Neagu <[email protected]> Co-authored-by: Odei Maiz <[email protected]> Co-authored-by: Sylvain <[email protected]> Co-authored-by: Pedro Crespo-Valero <[email protected]> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: matusdrobuliak66 <[email protected]> Co-authored-by: Julian Querido <[email protected]> Co-authored-by: Odei Maiz <[email protected]>
ITISFoundation · Nov 5, 2024 · cd20a2c · cd20a2c
1 parent d8f1720
commit cd20a2c
Show file tree

Hide file tree

Showing 47 changed files with 826 additions and 128 deletions.
diff --git a/packages/aws-library/requirements/_base.in b/packages/aws-library/requirements/_base.in
@@ -11,4 +11,5 @@ aiocache
 arrow
 pydantic[email]
 types-aiobotocore[ec2,s3,ssm]
+opentelemetry-instrumentation-botocore
 sh
diff --git a/packages/aws-library/requirements/_base.txt b/packages/aws-library/requirements/_base.txt
@@ -44,6 +44,8 @@ arrow==1.3.0
     #   -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in
     #   -r requirements/../../../packages/service-library/requirements/_base.in
     #   -r requirements/_base.in
+async-timeout==4.0.3
+    # via redis
 attrs==24.2.0
     # via
     #   aiohttp
@@ -127,7 +129,10 @@ opentelemetry-api==1.27.0
     #   opentelemetry-exporter-otlp-proto-grpc
     #   opentelemetry-exporter-otlp-proto-http
     #   opentelemetry-instrumentation
+    #   opentelemetry-instrumentation-botocore
+    #   opentelemetry-instrumentation-redis
     #   opentelemetry-instrumentation-requests
+    #   opentelemetry-propagator-aws-xray
     #   opentelemetry-sdk
     #   opentelemetry-semantic-conventions
 opentelemetry-exporter-otlp==1.27.0
@@ -141,9 +146,18 @@ opentelemetry-exporter-otlp-proto-grpc==1.27.0
 opentelemetry-exporter-otlp-proto-http==1.27.0
     # via opentelemetry-exporter-otlp
 opentelemetry-instrumentation==0.48b0
-    # via opentelemetry-instrumentation-requests
+    # via
+    #   opentelemetry-instrumentation-botocore
+    #   opentelemetry-instrumentation-redis
+    #   opentelemetry-instrumentation-requests
+opentelemetry-instrumentation-botocore==0.48b0
+    # via -r requirements/_base.in
+opentelemetry-instrumentation-redis==0.48b0
+    # via -r requirements/../../../packages/service-library/requirements/_base.in
 opentelemetry-instrumentation-requests==0.48b0
     # via -r requirements/../../../packages/service-library/requirements/_base.in
+opentelemetry-propagator-aws-xray==1.0.2
+    # via opentelemetry-instrumentation-botocore
 opentelemetry-proto==1.27.0
     # via
     #   opentelemetry-exporter-otlp-proto-common
@@ -156,6 +170,8 @@ opentelemetry-sdk==1.27.0
     #   opentelemetry-exporter-otlp-proto-http
 opentelemetry-semantic-conventions==0.48b0
     # via
+    #   opentelemetry-instrumentation-botocore
+    #   opentelemetry-instrumentation-redis
     #   opentelemetry-instrumentation-requests
     #   opentelemetry-sdk
 opentelemetry-util-http==0.48b0
@@ -297,6 +313,7 @@ wrapt==1.16.0
     #   aiobotocore
     #   deprecated
     #   opentelemetry-instrumentation
+    #   opentelemetry-instrumentation-redis
 yarl==1.12.1
     # via
     #   aio-pika

diff --git a/packages/postgres-database/requirements/_base.in b/packages/postgres-database/requirements/_base.in
@@ -7,4 +7,5 @@
 alembic
 pydantic
 sqlalchemy[postgresql_psycopg2binary,postgresql_asyncpg] # SEE extras in https://github.com/sqlalchemy/sqlalchemy/blob/main/setup.cfg#L43
+opentelemetry-instrumentation-asyncpg
 yarl
diff --git a/packages/postgres-database/requirements/_base.txt b/packages/postgres-database/requirements/_base.txt
@@ -4,10 +4,16 @@ async-timeout==4.0.3
     # via asyncpg
 asyncpg==0.29.0
     # via sqlalchemy
+deprecated==1.2.14
+    # via
+    #   opentelemetry-api
+    #   opentelemetry-semantic-conventions
 greenlet==3.1.1
     # via sqlalchemy
 idna==3.10
     # via yarl
+importlib-metadata==8.4.0
+    # via opentelemetry-api
 mako==1.3.5
     # via
     #   -c requirements/../../../requirements/constraints.txt
@@ -16,12 +22,25 @@ markupsafe==2.1.5
     # via mako
 multidict==6.1.0
     # via yarl
+opentelemetry-api==1.27.0
+    # via
+    #   opentelemetry-instrumentation
+    #   opentelemetry-instrumentation-asyncpg
+    #   opentelemetry-semantic-conventions
+opentelemetry-instrumentation==0.48b0
+    # via opentelemetry-instrumentation-asyncpg
+opentelemetry-instrumentation-asyncpg==0.48b0
+    # via -r requirements/_base.in
+opentelemetry-semantic-conventions==0.48b0
+    # via opentelemetry-instrumentation-asyncpg
 psycopg2-binary==2.9.9
     # via sqlalchemy
 pydantic==1.10.18
     # via
     #   -c requirements/../../../requirements/constraints.txt
     #   -r requirements/_base.in
+setuptools==75.2.0
+    # via opentelemetry-instrumentation
 sqlalchemy==1.4.54
     # via
     #   -c requirements/../../../requirements/constraints.txt
@@ -31,5 +50,11 @@ typing-extensions==4.12.2
     # via
     #   alembic
     #   pydantic
+wrapt==1.16.0
+    # via
+    #   deprecated
+    #   opentelemetry-instrumentation
 yarl==1.12.1
     # via -r requirements/_base.in
+zipp==3.20.2
+    # via importlib-metadata
diff --git a/packages/postgres-database/requirements/_tools.txt b/packages/postgres-database/requirements/_tools.txt
@@ -68,8 +68,10 @@ pyyaml==6.0.2
     #   pre-commit
 ruff==0.6.7
     # via -r requirements/../../../requirements/devenv.txt
-setuptools==75.1.0
-    # via pip-tools
+setuptools==75.2.0
+    # via
+    #   -c requirements/_base.txt
+    #   pip-tools
 tomlkit==0.13.2
     # via pylint
 typing-extensions==4.12.2

diff --git a/packages/service-library/requirements/_base.in b/packages/service-library/requirements/_base.in
@@ -18,6 +18,7 @@ faststream
 opentelemetry-api
 opentelemetry-exporter-otlp
 opentelemetry-instrumentation-requests
+opentelemetry-instrumentation-redis
 opentelemetry-sdk
 psutil
 pydantic

diff --git a/packages/service-library/requirements/_base.txt b/packages/service-library/requirements/_base.txt
@@ -28,6 +28,8 @@ arrow==1.3.0
     # via
     #   -r requirements/../../../packages/models-library/requirements/_base.in
     #   -r requirements/_base.in
+async-timeout==4.0.3
+    # via redis
 attrs==24.2.0
     # via
     #   aiohttp
@@ -93,6 +95,7 @@ opentelemetry-api==1.27.0
     #   opentelemetry-exporter-otlp-proto-grpc
     #   opentelemetry-exporter-otlp-proto-http
     #   opentelemetry-instrumentation
+    #   opentelemetry-instrumentation-redis
     #   opentelemetry-instrumentation-requests
     #   opentelemetry-sdk
     #   opentelemetry-semantic-conventions
@@ -107,7 +110,11 @@ opentelemetry-exporter-otlp-proto-grpc==1.27.0
 opentelemetry-exporter-otlp-proto-http==1.27.0
     # via opentelemetry-exporter-otlp
 opentelemetry-instrumentation==0.48b0
-    # via opentelemetry-instrumentation-requests
+    # via
+    #   opentelemetry-instrumentation-redis
+    #   opentelemetry-instrumentation-requests
+opentelemetry-instrumentation-redis==0.48b0
+    # via -r requirements/_base.in
 opentelemetry-instrumentation-requests==0.48b0
     # via -r requirements/_base.in
 opentelemetry-proto==1.27.0
@@ -122,6 +129,7 @@ opentelemetry-sdk==1.27.0
     #   opentelemetry-exporter-otlp-proto-http
 opentelemetry-semantic-conventions==0.48b0
     # via
+    #   opentelemetry-instrumentation-redis
     #   opentelemetry-instrumentation-requests
     #   opentelemetry-sdk
 opentelemetry-util-http==0.48b0
@@ -219,6 +227,7 @@ wrapt==1.16.0
     # via
     #   deprecated
     #   opentelemetry-instrumentation
+    #   opentelemetry-instrumentation-redis
 yarl==1.12.1
     # via
     #   aio-pika

diff --git a/packages/service-library/requirements/_test.in b/packages/service-library/requirements/_test.in
@@ -13,13 +13,15 @@
 # testing
 
 asgi_lifespan
+botocore
 coverage
 docker
 faker
 flaky
 numpy
 openapi-spec-validator
 pillow
+pip
 pytest
 pytest-aiohttp
 pytest-asyncio

diff --git a/packages/service-library/requirements/_test.txt b/packages/service-library/requirements/_test.txt
@@ -29,6 +29,8 @@ attrs==24.2.0
     #   jsonschema
     #   pytest-docker
     #   referencing
+botocore==1.35.50
+    # via -r requirements/_test.in
 certifi==2024.8.30
     # via
     #   -c requirements/../../../requirements/constraints.txt
@@ -91,6 +93,8 @@ idna==3.10
     #   yarl
 iniconfig==2.0.0
     # via pytest
+jmespath==1.0.1
+    # via botocore
 jsonschema==4.23.0
     # via
     #   -c requirements/_aiohttp.txt
@@ -141,6 +145,8 @@ pathable==0.4.3
     #   jsonschema-path
 pillow==10.4.0
     # via -r requirements/_test.in
+pip==24.3.1
+    # via -r requirements/_test.in
 pluggy==1.5.0
     # via pytest
 pprintpp==0.4.0
@@ -188,6 +194,7 @@ pytest-xdist==3.6.1
 python-dateutil==2.9.0.post0
     # via
     #   -c requirements/_base.txt
+    #   botocore
     #   faker
 python-dotenv==1.0.1
     # via -r requirements/_test.in
@@ -263,6 +270,7 @@ urllib3==2.2.3
     #   -c requirements/../../../requirements/constraints.txt
     #   -c requirements/_aiohttp.txt
     #   -c requirements/_base.txt
+    #   botocore
     #   docker
     #   requests
 yarl==1.12.1

diff --git a/packages/service-library/requirements/_tools.txt b/packages/service-library/requirements/_tools.txt
@@ -45,8 +45,10 @@ packaging==24.1
     #   build
 pathspec==0.12.1
     # via black
-pip==24.2
-    # via pip-tools
+pip==24.3.1
+    # via
+    #   -c requirements/_test.txt
+    #   pip-tools
 pip-tools==7.4.1
     # via -r requirements/../../../requirements/devenv.txt
 platformdirs==4.3.6

diff --git a/packages/service-library/src/servicelib/aiohttp/tracing.py b/packages/service-library/src/servicelib/aiohttp/tracing.py
@@ -15,23 +15,39 @@
 from opentelemetry.instrumentation.aiohttp_server import (
     middleware as aiohttp_server_opentelemetry_middleware,  # pylint:disable=no-name-in-module
 )
-from opentelemetry.instrumentation.aiopg import (  # pylint:disable=no-name-in-module
-    AiopgInstrumentor,
-)
-from opentelemetry.instrumentation.requests import RequestsInstrumentor
 from opentelemetry.sdk.resources import Resource
 from opentelemetry.sdk.trace import TracerProvider
 from opentelemetry.sdk.trace.export import BatchSpanProcessor
+from servicelib.logging_utils import log_context
 from settings_library.tracing import TracingSettings
 
 _logger = logging.getLogger(__name__)
+try:
+    from opentelemetry.instrumentation.botocore import (  # type: ignore[import-not-found]
+        BotocoreInstrumentor,
+    )
+
+    HAS_BOTOCORE = True
+except ImportError:
+    HAS_BOTOCORE = False
+try:
+    from opentelemetry.instrumentation.aiopg import AiopgInstrumentor
+
+    HAS_AIOPG = True
+except ImportError:
+    HAS_AIOPG = False
+try:
+    from opentelemetry.instrumentation.requests import RequestsInstrumentor
+
+    HAS_REQUESTS = True
+except ImportError:
+    HAS_REQUESTS = False
 
 
 def setup_tracing(
     app: web.Application,
     tracing_settings: TracingSettings,
     service_name: str,
-    instrument_aiopg: bool = False,  # noqa: FBT001, FBT002
 ) -> None:
     """
     Sets up this service for a distributed tracing system (opentelemetry)
@@ -91,6 +107,24 @@ def setup_tracing(
 
     # Instrument aiohttp client
     AioHttpClientInstrumentor().instrument()
-    if instrument_aiopg:
-        AiopgInstrumentor().instrument()
-    RequestsInstrumentor().instrument()
+    if HAS_AIOPG:
+        with log_context(
+            _logger,
+            logging.INFO,
+            msg="Attempting to add aio-pg opentelemetry autoinstrumentation...",
+        ):
+            AiopgInstrumentor().instrument()
+    if HAS_BOTOCORE:
+        with log_context(
+            _logger,
+            logging.INFO,
+            msg="Attempting to add botocore opentelemetry autoinstrumentation...",
+        ):
+            BotocoreInstrumentor().instrument()
+    if HAS_REQUESTS:
+        with log_context(
+            _logger,
+            logging.INFO,
+            msg="Attempting to add requests opentelemetry autoinstrumentation...",
+        ):
+            RequestsInstrumentor().instrument()