✨(backend) Adding /prometheus logging endpoint numerique-gouv#455

implements various metrics for users, documents and
the actual django application

Signed-off-by: lindenb1 <[email protected]>

CHANGELOG.md

@@ -9,6 +9,9 @@ and this project adheres to
 
 ## [Unreleased]
 
+## Added
+
+✨(backend) Adding /prometheus metrics endpoint #455
 
 ## [1.8.2] - 2024-11-28
 

docker-compose.yml

@@ -50,6 +50,8 @@ services:
     environment:
       - PYLINTHOME=/app/.pylint.d
       - DJANGO_CONFIGURATION=Development
+      - PROMETHEUS_EXPORTER=true
+      - PROMETHEUS_ALLOWED_CIDR_RANGES="172.23.0.0/16" # separate by comma
     env_file:
       - env.d/development/common
       - env.d/development/postgresql

src/backend/core/api/__init__.py

@@ -37,4 +37,4 @@ def get_frontend_configuration(request):
         "LANGUAGE_CODE": settings.LANGUAGE_CODE,
     }
     frontend_configuration.update(settings.FRONTEND_CONFIGURATION)
-    return Response(frontend_configuration)
+    return Response(frontend_configuration)

src/backend/core/api/custom_metrics_exporter.py

@@ -0,0 +1,100 @@
+from prometheus_client.core import GaugeMetricFamily
+from django.utils.timezone import now
+from django.db.models import Count, Min, Max, Q, F
+from datetime import timedelta
+from core import models
+from django.conf import settings
+
+
+class CustomMetricsExporter:
+    """
+    Custom Prometheus metrics collector for user and document statistics.
+    """
+
+    def collect(self):
+        namespace = getattr(settings, "PROMETHEUS_METRIC_NAMESPACE", "")
+
+        def prefixed_metric_name(name):
+            return f"{namespace}_{name}" if namespace else name
+
+        now_time = now()
+        today_start_utc = now_time.replace(hour=0, minute=0, second=0, microsecond=0)
+        one_week_ago = today_start_utc - timedelta(days=7)
+        one_month_ago = today_start_utc - timedelta(days=30)
+
+        user_count = models.User.objects.count()
+        active_users_today = models.User.objects.filter(
+            Q(documentaccess__updated_at__gte=today_start_utc) |
+            Q(link_traces__created_at__gte=today_start_utc) |
+            Q(last_login__gte=today_start_utc)
+        ).distinct().count()
+        active_users_7_days = models.User.objects.filter(
+            Q(documentaccess__updated_at__gte=one_week_ago) |
+            Q(link_traces__created_at__gte=one_week_ago) |
+            Q(last_login__gte=one_week_ago)
+        ).distinct().count()
+        active_users_30_days = models.User.objects.filter(
+            Q(documentaccess__updated_at__gte=one_month_ago) |
+            Q(link_traces__created_at__gte=one_month_ago) |
+            Q(last_login__gte=one_month_ago)
+        ).distinct().count()
+
+        total_documents = models.Document.objects.count()
+        shared_docs_count = models.Document.objects.annotate(
+            access_count=Count("accesses")
+        ).filter(access_count__gt=1).count()
+        active_docs_today = models.Document.objects.filter(
+            updated_at__gte=today_start_utc,
+            updated_at__lt=today_start_utc + timedelta(days=1),
+        ).count()
+        active_docs_last_7_days = models.Document.objects.filter(
+            updated_at__gte=one_week_ago
+        ).count()
+        active_docs_last_30_days = models.Document.objects.filter(
+            updated_at__gte=one_month_ago
+        ).count()
+
+        oldest_doc_date = models.Document.objects.aggregate(
+            oldest=Min("created_at")
+        )["oldest"]
+        newest_doc_date = models.Document.objects.aggregate(
+            newest=Max("created_at")
+        )["newest"]
+
+        user_doc_counts = models.DocumentAccess.objects.values("user_id").annotate(
+            doc_count=Count("document_id"),
+            admin_email=F("user__admin_email")
+        )
+
+        metrics = []
+        metrics.append(GaugeMetricFamily(prefixed_metric_name("total_users"), "Total number of users", value=user_count))
+        metrics.append(GaugeMetricFamily(prefixed_metric_name("active_users_today"), "Number of active users today", value=active_users_today))
+        metrics.append(GaugeMetricFamily(prefixed_metric_name("active_users_7_days"), "Number of active users in the last 7 days", value=active_users_7_days))
+        metrics.append(GaugeMetricFamily(prefixed_metric_name("active_users_30_days"), "Number of active users in the last 30 days", value=active_users_30_days))
+        metrics.append(GaugeMetricFamily(prefixed_metric_name("total_documents"), "Total number of documents", value=total_documents))
+        metrics.append(GaugeMetricFamily(prefixed_metric_name("shared_documents"), "Number of shared documents", value=shared_docs_count))
+        metrics.append(GaugeMetricFamily(prefixed_metric_name("active_documents_today"), "Number of active documents today", value=active_docs_today))
+        metrics.append(GaugeMetricFamily(prefixed_metric_name("active_documents_7_days"), "Number of active documents in the last 7 days", value=active_docs_last_7_days))
+        metrics.append(GaugeMetricFamily(prefixed_metric_name("active_documents_30_days"), "Number of active documents in the last 30 days", value=active_docs_last_30_days))
+
+        if oldest_doc_date:
+            metrics.append(GaugeMetricFamily(
+                prefixed_metric_name("oldest_document_date"), "Timestamp of the oldest document creation date",
+                value=oldest_doc_date.timestamp()
+            ))
+        if newest_doc_date:
+            metrics.append(GaugeMetricFamily(
+                prefixed_metric_name("newest_document_date"), "Timestamp of the newest document creation date",
+                value=newest_doc_date.timestamp()
+            ))
+
+        user_distribution_metric = GaugeMetricFamily(
+            prefixed_metric_name("user_document_distribution"), "Document counts per user", labels=["user_email"]
+        )
+        for user in user_doc_counts:
+            if user["admin_email"]:  # Validate email existence
+                user_distribution_metric.add_metric([user["admin_email"]], user["doc_count"])
+        metrics.append(user_distribution_metric)
+
+        for metric in metrics:
+            yield metric

src/backend/core/api/decorators.py

@@ -0,0 +1,40 @@
+import os
+from ipaddress import ip_network, ip_address
+from django.http import HttpResponseForbidden
+
+
+def cidr_protected_view(view):
+    """
+    Decorator to protect a view with a CIDR filter.
+    CIDR ranges are fetched from the environment variable `PROMETHEUS_ALLOWED_CIDR_RANGES`.
+    If not set, access is denied by default.
+    """
+    # Fetch allowed CIDR ranges from the environment variable
+    cidr_env = os.environ.get("PROMETHEUS_ALLOWED_CIDR_RANGES", "")
+
+    # validate CIDR ranges
+    try:
+        allowed_cidr_ranges = [
+            ip_network(cidr.strip().strip('"').strip("'"))
+            for cidr in cidr_env.split(",")
+            if cidr.strip()
+        ]
+    except ValueError as e:
+        raise ValueError(f"Invalid CIDR range in PROMETHEUS_ALLOWED_CIDR_RANGES: {e}")
+
+    def wrapped_view(request, *args, **kwargs):
+        # Get the client's IP address from the request
+        client_ip = request.META.get("REMOTE_ADDR")
+
+        # If no CIDR ranges are configured, deny access
+        if not allowed_cidr_ranges:
+            return HttpResponseForbidden("Access denied: No allowed CIDR ranges configured.")
+
+        # Check if the client's IP is in the allowed CIDR ranges
+        if not any(ip_address(client_ip) in cidr for cidr in allowed_cidr_ranges):
+            return HttpResponseForbidden("Access denied: Your IP is not allowed.")
+
+        # Proceed to the original view
+        return view(request, *args, **kwargs)
+
+    return wrapped_view

src/backend/impress/settings.py

@@ -23,6 +23,7 @@
 # Build paths inside the project like this: BASE_DIR / 'subdir'.
 BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 DATA_DIR = os.path.join("/", "data")
+PROMETHEUS_EXPORTER = os.getenv("PROMETHEUS_EXPORTER", "False").lower() == "true"
 
 
 def get_release():
@@ -282,6 +283,14 @@ class Base(Configuration):
         "dockerflow.django.middleware.DockerflowMiddleware",
     ]
 
+    if PROMETHEUS_EXPORTER:
+        MIDDLEWARE.insert(0, "django_prometheus.middleware.PrometheusBeforeMiddleware")
+        MIDDLEWARE.append("django_prometheus.middleware.PrometheusAfterMiddleware")
+        PROMETHEUS_METRIC_NAMESPACE = "impress"
+        PROMETHEUS_LATENCY_BUCKETS = (
+            .05, .1, .25, .5, .75, 1.0, 1.5, 2.5, 5.0, 10.0, 15.0, 30.0, float("inf")
+        )
+
     AUTHENTICATION_BACKENDS = [
         "django.contrib.auth.backends.ModelBackend",
         "core.authentication.backends.OIDCAuthenticationBackend",
@@ -295,6 +304,7 @@ class Base(Configuration):
         "drf_spectacular",
         # Third party apps
         "corsheaders",
+        "django_prometheus",
         "dockerflow.django",
         "rest_framework",
         "parler",
@@ -314,7 +324,10 @@ class Base(Configuration):
 
     # Cache
     CACHES = {
-        "default": {"BACKEND": "django.core.cache.backends.locmem.LocMemCache"},
+        "default": {
+            "BACKEND": "django.core.cache.backends.locmem.LocMemCache" if not PROMETHEUS_EXPORTER
+            else "django_prometheus.cache.backends.locmem.LocMemCache",
+        },
     }
 
     REST_FRAMEWORK = {

src/backend/impress/urls.py

@@ -1,5 +1,7 @@
 """URL configuration for the impress project"""
 
+import os
+from django_prometheus import exports
 from django.conf import settings
 from django.conf.urls.static import static
 from django.contrib import admin
@@ -12,11 +14,20 @@
     SpectacularSwaggerView,
 )
 
+from core.api.decorators import cidr_protected_view
+
 urlpatterns = [
     path("admin/", admin.site.urls),
     path("", include("core.urls")),
 ]
 
+if os.environ.get("PROMETHEUS_EXPORTER", "False").lower() == "true":
+    # Protect the Prometheus view with the CIDR decorator
+    protected_export_view = cidr_protected_view(exports.ExportToDjangoView)
+    urlpatterns.append(
+        path("prometheus/", protected_export_view, name="prometheus-django-metrics"),
+    )
+
 if settings.DEBUG:
     urlpatterns = (
         urlpatterns

src/backend/impress/wsgi.py

@@ -11,7 +11,26 @@
 
 from configurations.wsgi import get_wsgi_application
 
+# Prometheus Metrics Registration
+from prometheus_client import REGISTRY
+from core.api.custom_metrics_exporter import CustomMetricsExporter
+
+
+def register_prometheus_exporter():
+    """
+    Register custom Prometheus metrics collector.
+    """
+    if not any(isinstance(cme, CustomMetricsExporter) for cme in REGISTRY._collector_to_names):
+        REGISTRY.register(CustomMetricsExporter())
+        print("Custom Prometheus metrics registered successfully.")
+    else:
+        print("Custom Prometheus metrics already registered.")
+
 os.environ.setdefault("DJANGO_SETTINGS_MODULE", "impress.settings")
 os.environ.setdefault("DJANGO_CONFIGURATION", "Development")
 
+# Call register_prometheus_exporter to register Prometheus metrics if enabled
+if os.environ.get("PROMETHEUS_EXPORTER", "False").lower() == "true":
+    register_prometheus_exporter()
+
 application = get_wsgi_application()

src/backend/pyproject.toml

@@ -33,6 +33,7 @@ dependencies = [
     "django-countries==7.6.1",
     "django-filter==24.3",
     "django-parler==2.3",
+    "django-prometheus==2.3.1",
     "redis==5.1.1",
     "django-redis==5.4.0",
     "django-storages[s3]==1.14.4",