From 9eff2745f2af01a7543431f606c4d3c919ff400a Mon Sep 17 00:00:00 2001 From: Mads Bisgaard Date: Thu, 21 Nov 2024 13:48:09 +0100 Subject: [PATCH 1/3] add tracing for all fcns in api-server --- .../src/servicelib/fastapi/tracing.py | 66 +++++++++++++++++++ .../core/application.py | 10 ++- 2 files changed, 74 insertions(+), 2 deletions(-) diff --git a/packages/service-library/src/servicelib/fastapi/tracing.py b/packages/service-library/src/servicelib/fastapi/tracing.py index 36e9b06fa12..be90fbfc7b1 100644 --- a/packages/service-library/src/servicelib/fastapi/tracing.py +++ b/packages/service-library/src/servicelib/fastapi/tracing.py @@ -2,7 +2,16 @@ """ +import importlib +import importlib.machinery +import inspect import logging +import sys +from functools import wraps +from importlib.abc import Loader, MetaPathFinder +from importlib.machinery import ModuleSpec +from types import ModuleType +from typing import Callable, Sequence from fastapi import FastAPI from httpx import AsyncClient, Client @@ -127,3 +136,60 @@ def setup_tracing( def setup_httpx_client_tracing(client: AsyncClient | Client): HTTPXClientInstrumentor.instrument_client(client) + + +def _create_opentelemetry_function_span(func: Callable): + """Decorator that wraps a function call in an OpenTelemetry span.""" + tracer = trace.get_tracer(__name__) + + @wraps(func) + def wrapper(*args, **kwargs): + with tracer.start_as_current_span(f"{func.__module__}.{func.__name__}"): + return func(*args, **kwargs) + + @wraps(func) + async def async_wrapper(*args, **kwargs): + with tracer.start_as_current_span(f"{func.__module__}.{func.__name__}"): + return await func(*args, **kwargs) + + if inspect.iscoroutinefunction(func): + return async_wrapper + else: + return wrapper + + +class _AddTracingSpansLoader(Loader): + def __init__(self, loader: Loader): + self.loader = loader + + def exec_module(self, module: ModuleType): + # Execute the module normally + self.loader.exec_module(module) + for name, func in inspect.getmembers(module, inspect.isfunction): + if name in module.__dict__: + setattr(module, name, _create_opentelemetry_function_span(func)) + + +class _AddTracingSpansFinder(MetaPathFinder): + def find_spec( + self, + fullname: str, + path: Sequence[str] | None, + target: ModuleType | None = None, + ) -> ModuleSpec | None: + if fullname.startswith("simcore_service"): + # Find the original spec + spec = importlib.machinery.PathFinder.find_spec( + fullname=fullname, path=path + ) + # spec = find_spec(fullname, path) + if spec and spec.loader: + # Wrap the loader with our DecoratingLoader + spec.loader = _AddTracingSpansLoader(spec.loader) + return spec + + return None + + +def setup_tracing_spans_for_simcore_service_functions(): + sys.meta_path.insert(0, _AddTracingSpansFinder()) diff --git a/services/api-server/src/simcore_service_api_server/core/application.py b/services/api-server/src/simcore_service_api_server/core/application.py index 3d67746deb7..95183ba8b97 100644 --- a/services/api-server/src/simcore_service_api_server/core/application.py +++ b/services/api-server/src/simcore_service_api_server/core/application.py @@ -5,10 +5,16 @@ from models_library.basic_types import BootModeEnum from packaging.version import Version from servicelib.fastapi.profiler_middleware import ProfilerMiddleware -from servicelib.fastapi.tracing import setup_tracing +from servicelib.fastapi.tracing import ( + setup_tracing, + setup_tracing_spans_for_simcore_service_functions, +) from servicelib.logging_utils import config_all_loggers +from simcore_service_api_server import exceptions + +setup_tracing_spans_for_simcore_service_functions() +# isort: split -from .. import exceptions from .._meta import API_VERSION, API_VTAG, APP_NAME from ..api.root import create_router from ..api.routes.health import router as health_router From b0ec0f9672f69ff647981ab92cc916130f877ba2 Mon Sep 17 00:00:00 2001 From: Mads Bisgaard Date: Thu, 21 Nov 2024 14:10:31 +0100 Subject: [PATCH 2/3] add spans to class methods --- .../src/servicelib/fastapi/tracing.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/packages/service-library/src/servicelib/fastapi/tracing.py b/packages/service-library/src/servicelib/fastapi/tracing.py index be90fbfc7b1..9bef4a23edd 100644 --- a/packages/service-library/src/servicelib/fastapi/tracing.py +++ b/packages/service-library/src/servicelib/fastapi/tracing.py @@ -138,7 +138,7 @@ def setup_httpx_client_tracing(client: AsyncClient | Client): HTTPXClientInstrumentor.instrument_client(client) -def _create_opentelemetry_function_span(func: Callable): +def _opentelemetry_function_span(func: Callable): """Decorator that wraps a function call in an OpenTelemetry span.""" tracer = trace.get_tracer(__name__) @@ -158,6 +158,15 @@ async def async_wrapper(*args, **kwargs): return wrapper +def _opentelemetry_method_span(cls): + for name, value in cls.__dict__.items(): + if callable(value) and not name.startswith("_"): + setattr( + cls, name, _opentelemetry_function_span(value) + ) # Apply the decorator + return cls + + class _AddTracingSpansLoader(Loader): def __init__(self, loader: Loader): self.loader = loader @@ -167,7 +176,10 @@ def exec_module(self, module: ModuleType): self.loader.exec_module(module) for name, func in inspect.getmembers(module, inspect.isfunction): if name in module.__dict__: - setattr(module, name, _create_opentelemetry_function_span(func)) + setattr(module, name, _opentelemetry_function_span(func)) + for name, cls in inspect.getmembers(module, inspect.isclass): + if name in module.__dict__ and cls.__module__ == module.__name__: + setattr(module, name, _opentelemetry_method_span(cls)) class _AddTracingSpansFinder(MetaPathFinder): From ef21fd68b3dd5a852f8f553438e7a1be2d923494 Mon Sep 17 00:00:00 2001 From: Mads Bisgaard Date: Thu, 21 Nov 2024 14:12:00 +0100 Subject: [PATCH 3/3] minor cleanup --- .../service-library/src/servicelib/fastapi/tracing.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/packages/service-library/src/servicelib/fastapi/tracing.py b/packages/service-library/src/servicelib/fastapi/tracing.py index 9bef4a23edd..fddcdbfb2a0 100644 --- a/packages/service-library/src/servicelib/fastapi/tracing.py +++ b/packages/service-library/src/servicelib/fastapi/tracing.py @@ -161,9 +161,7 @@ async def async_wrapper(*args, **kwargs): def _opentelemetry_method_span(cls): for name, value in cls.__dict__.items(): if callable(value) and not name.startswith("_"): - setattr( - cls, name, _opentelemetry_function_span(value) - ) # Apply the decorator + setattr(cls, name, _opentelemetry_function_span(value)) return cls @@ -172,7 +170,6 @@ def __init__(self, loader: Loader): self.loader = loader def exec_module(self, module: ModuleType): - # Execute the module normally self.loader.exec_module(module) for name, func in inspect.getmembers(module, inspect.isfunction): if name in module.__dict__: @@ -190,13 +187,10 @@ def find_spec( target: ModuleType | None = None, ) -> ModuleSpec | None: if fullname.startswith("simcore_service"): - # Find the original spec spec = importlib.machinery.PathFinder.find_spec( fullname=fullname, path=path ) - # spec = find_spec(fullname, path) if spec and spec.loader: - # Wrap the loader with our DecoratingLoader spec.loader = _AddTracingSpansLoader(spec.loader) return spec