✨(backend) Adding /prometheus logging endpoint suitenumerique#455

implements various metrics for users, documents and
the actual django application

Signed-off-by: lindenb1 <[email protected]>

CHANGELOG.md

@@ -9,6 +9,9 @@ and this project adheres to
 
 ## [Unreleased]
 
+## Added
+
+✨(backend) Adding /prometheus logging endpoint #455
 
 ## [1.8.2] - 2024-11-28
 

docker-compose.yml

@@ -50,6 +50,7 @@ services:
     environment:
       - PYLINTHOME=/app/.pylint.d
       - DJANGO_CONFIGURATION=Development
+      - PROMETHEUS_METRICS=true
     env_file:
       - env.d/development/common
       - env.d/development/postgresql

src/backend/core/api/__init__.py

@@ -1,5 +1,7 @@
 """Impress core API endpoints"""
 
+import os
+
 from django.conf import settings
 from django.core.exceptions import ValidationError
 
@@ -8,6 +10,14 @@
 from rest_framework.decorators import api_view
 from rest_framework.response import Response
 
+from prometheus_client import REGISTRY
+from .custom_metrics_collector import CustomMetricsCollector
+
+# Register the custom Prometheus metric collector during API initialization if it has been enabled
+if os.environ.get("PROMETHEUS_METRICS", "False").lower() == "true":
+    if not any(isinstance(cmc, CustomMetricsCollector) for cmc in REGISTRY._collector_to_names):
+        REGISTRY.register(CustomMetricsCollector())
+
 
 def exception_handler(exc, context):
     """Handle Django ValidationError as an accepted exception.

src/backend/core/api/custom_metrics_collector.py

@@ -0,0 +1,100 @@
+from prometheus_client.core import GaugeMetricFamily
+from django.utils.timezone import now
+from django.db.models import Count, Min, Max, Q, F
+from datetime import timedelta
+from core import models
+from django.conf import settings
+
+
+class CustomMetricsCollector:
+    """
+    Custom Prometheus metrics collector for user and document statistics.
+    """
+
+    def collect(self):
+        namespace = getattr(settings, "PROMETHEUS_METRIC_NAMESPACE", "")
+
+        def prefixed_metric_name(name):
+            return f"{namespace}_{name}" if namespace else name
+
+        now_time = now()
+        today_start_utc = now_time.replace(hour=0, minute=0, second=0, microsecond=0)
+        one_week_ago = today_start_utc - timedelta(days=7)
+        one_month_ago = today_start_utc - timedelta(days=30)
+
+        user_count = models.User.objects.count()
+        active_users_today = models.User.objects.filter(
+            Q(documentaccess__updated_at__gte=today_start_utc) |
+            Q(link_traces__created_at__gte=today_start_utc) |
+            Q(last_login__gte=today_start_utc)
+        ).distinct().count()
+        active_users_7_days = models.User.objects.filter(
+            Q(documentaccess__updated_at__gte=one_week_ago) |
+            Q(link_traces__created_at__gte=one_week_ago) |
+            Q(last_login__gte=one_week_ago)
+        ).distinct().count()
+        active_users_30_days = models.User.objects.filter(
+            Q(documentaccess__updated_at__gte=one_month_ago) |
+            Q(link_traces__created_at__gte=one_month_ago) |
+            Q(last_login__gte=one_month_ago)
+        ).distinct().count()
+
+        total_documents = models.Document.objects.count()
+        shared_docs_count = models.Document.objects.annotate(
+            access_count=Count("accesses")
+        ).filter(access_count__gt=1).count()
+        active_docs_today = models.Document.objects.filter(
+            updated_at__gte=today_start_utc,
+            updated_at__lt=today_start_utc + timedelta(days=1),
+        ).count()
+        active_docs_last_7_days = models.Document.objects.filter(
+            updated_at__gte=one_week_ago
+        ).count()
+        active_docs_last_30_days = models.Document.objects.filter(
+            updated_at__gte=one_month_ago
+        ).count()
+
+        oldest_doc_date = models.Document.objects.aggregate(
+            oldest=Min("created_at")
+        )["oldest"]
+        newest_doc_date = models.Document.objects.aggregate(
+            newest=Max("created_at")
+        )["newest"]
+
+        user_doc_counts = models.DocumentAccess.objects.values("user_id").annotate(
+            doc_count=Count("document_id"),
+            admin_email=F("user__admin_email")
+        )
+
+        metrics = []
+        metrics.append(GaugeMetricFamily(prefixed_metric_name("total_users"), "Total number of users", value=user_count))
+        metrics.append(GaugeMetricFamily(prefixed_metric_name("active_users_today"), "Number of active users today", value=active_users_today))
+        metrics.append(GaugeMetricFamily(prefixed_metric_name("active_users_7_days"), "Number of active users in the last 7 days", value=active_users_7_days))
+        metrics.append(GaugeMetricFamily(prefixed_metric_name("active_users_30_days"), "Number of active users in the last 30 days", value=active_users_30_days))
+        metrics.append(GaugeMetricFamily(prefixed_metric_name("total_documents"), "Total number of documents", value=total_documents))
+        metrics.append(GaugeMetricFamily(prefixed_metric_name("shared_documents"), "Number of shared documents", value=shared_docs_count))
+        metrics.append(GaugeMetricFamily(prefixed_metric_name("active_documents_today"), "Number of active documents today", value=active_docs_today))
+        metrics.append(GaugeMetricFamily(prefixed_metric_name("active_documents_7_days"), "Number of active documents in the last 7 days", value=active_docs_last_7_days))
+        metrics.append(GaugeMetricFamily(prefixed_metric_name("active_documents_30_days"), "Number of active documents in the last 30 days", value=active_docs_last_30_days))
+
+        if oldest_doc_date:
+            metrics.append(GaugeMetricFamily(
+                prefixed_metric_name("oldest_document_date"), "Timestamp of the oldest document creation date",
+                value=oldest_doc_date.timestamp()
+            ))
+        if newest_doc_date:
+            metrics.append(GaugeMetricFamily(
+                prefixed_metric_name("newest_document_date"), "Timestamp of the newest document creation date",
+                value=newest_doc_date.timestamp()
+            ))
+
+        user_distribution_metric = GaugeMetricFamily(
+            prefixed_metric_name("user_document_distribution"), "Document counts per user", labels=["user_email"]
+        )
+        for user in user_doc_counts:
+            if user["admin_email"]:  # Validate email existence
+                user_distribution_metric.add_metric([user["admin_email"]], user["doc_count"])
+        metrics.append(user_distribution_metric)
+
+        for metric in metrics:
+            yield metric

src/backend/impress/settings.py

@@ -23,6 +23,7 @@
 # Build paths inside the project like this: BASE_DIR / 'subdir'.
 BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 DATA_DIR = os.path.join("/", "data")
+PROMETHEUS_METRICS = os.getenv("PROMETHEUS_METRICS", "False").lower() == "true"
 
 
 def get_release():
@@ -74,7 +75,9 @@ class Base(Configuration):
     DATABASES = {
         "default": {
             "ENGINE": values.Value(
-                "django.db.backends.postgresql_psycopg2",
+                "django.db.backends.postgresql_psycopg2"
+                if not PROMETHEUS_METRICS
+                else "django_prometheus.db.backends.postgresql",
                 environ_name="DB_ENGINE",
                 environ_prefix=None,
             ),
@@ -282,6 +285,14 @@ class Base(Configuration):
         "dockerflow.django.middleware.DockerflowMiddleware",
     ]
 
+    if PROMETHEUS_METRICS:
+        MIDDLEWARE.insert(0, "django_prometheus.middleware.PrometheusBeforeMiddleware")
+        MIDDLEWARE.append("django_prometheus.middleware.PrometheusAfterMiddleware")
+        PROMETHEUS_METRIC_NAMESPACE = "impress"
+        PROMETHEUS_LATENCY_BUCKETS = (
+            .05, .1, .25, .5, .75, 1.0, 1.5, 2.5, 5.0, 10.0, 15.0, 30.0, float("inf")
+        )
+
     AUTHENTICATION_BACKENDS = [
         "django.contrib.auth.backends.ModelBackend",
         "core.authentication.backends.OIDCAuthenticationBackend",
@@ -295,6 +306,7 @@ class Base(Configuration):
         "drf_spectacular",
         # Third party apps
         "corsheaders",
+        "django_prometheus",
         "dockerflow.django",
         "rest_framework",
         "parler",
@@ -314,7 +326,10 @@ class Base(Configuration):
 
     # Cache
     CACHES = {
-        "default": {"BACKEND": "django.core.cache.backends.locmem.LocMemCache"},
+        "default": {
+            "BACKEND": "django.core.cache.backends.locmem.LocMemCache" if not PROMETHEUS_METRICS
+            else "django_prometheus.cache.backends.locmem.LocMemCache",
+        },
     }
 
     REST_FRAMEWORK = {

src/backend/impress/urls.py

@@ -1,5 +1,7 @@
 """URL configuration for the impress project"""
 
+import os
+from django_prometheus import exports
 from django.conf import settings
 from django.conf.urls.static import static
 from django.contrib import admin
@@ -17,6 +19,11 @@
     path("", include("core.urls")),
 ]
 
+if os.environ.get("PROMETHEUS_METRICS", "False").lower() == "true":
+    urlpatterns.append(
+        path("prometheus/", exports.ExportToDjangoView, name="prometheus-django-metrics")
+    )
+
 if settings.DEBUG:
     urlpatterns = (
         urlpatterns

src/backend/pyproject.toml

@@ -33,6 +33,7 @@ dependencies = [
     "django-countries==7.6.1",
     "django-filter==24.3",
     "django-parler==2.3",
+    "django-prometheus==2.3.1",
     "redis==5.1.1",
     "django-redis==5.4.0",
     "django-storages[s3]==1.14.4",