setup api_metrics app and celery task to update them

memberportal/api_general/urls.py

@@ -42,10 +42,5 @@
         views.UserSiteSession.as_view(),
         name="api_user_site_session",
     ),
-    path(
-        "api/statistics/",
-        views.Statistics.as_view(),
-        name="api_statistics",
-    ),
     path("api/kiosks/<int:id>/", views.Kiosks.as_view(), name="api_kiosks"),
 ]

memberportal/api_general/views.py

@@ -627,23 +627,6 @@ def get(self, request):
         return Response(status=status.HTTP_401_UNAUTHORIZED)
 
 
-class Statistics(APIView):
-    """
-    get: gets site statistics.
-    """
-
-    def get(self, request):
-        members = SiteSession.objects.filter(signout_date=None).order_by("-signin_date")
-        member_list = []
-
-        for member in members:
-            member_list.append(member.user.profile.get_full_name())
-
-        statistics = {"onSite": {"members": member_list, "count": members.count()}}
-
-        return Response(statistics)
-
-
 class Register(APIView):
     """
     post: registers a new member.

memberportal/api_metrics/admin.py

@@ -0,0 +1,7 @@
+from django.contrib import admin
+from api_metrics.models import *
+
+
+@admin.register(Metric)
+class Metric(admin.ModelAdmin):
+    pass

memberportal/api_general/metrics.py → memberportal/api_metrics/metrics.py

@@ -9,5 +9,5 @@
 subscription_count_total = Gauge(
     "mm_subscription_count_total",
     "Number of subscriptions in the system",
-    ["status"],
+    ["state"],
 )

memberportal/api_metrics/migrations/0001_initial.py

@@ -0,0 +1,42 @@
+# Generated by Django 3.2.25 on 2024-05-31 03:39
+
+from django.db import migrations, models
+import django.utils.timezone
+import django_prometheus.models
+
+
+class Migration(migrations.Migration):
+
+    initial = True
+
+    dependencies = []
+
+    operations = [
+        migrations.CreateModel(
+            name="Metric",
+            fields=[
+                ("id", models.AutoField(primary_key=True, serialize=False)),
+                (
+                    "creation_date",
+                    models.DateTimeField(default=django.utils.timezone.now),
+                ),
+                ("data", models.JSONField(verbose_name="Data")),
+                (
+                    "name",
+                    models.CharField(
+                        choices=[
+                            ("member_count_total", "Member Count Total"),
+                            ("subscription_count_total", "Subscription Count Total"),
+                        ],
+                        default=None,
+                        max_length=250,
+                        verbose_name="Metric Name",
+                    ),
+                ),
+            ],
+            bases=(
+                django_prometheus.models.ExportModelOperationsMixin("metric"),
+                models.Model,
+            ),
+        ),
+    ]

memberportal/api_metrics/models.py

@@ -0,0 +1,30 @@
+from django.db import models
+from django.utils import timezone
+import pytz
+from django_prometheus.models import ExportModelOperationsMixin
+
+utc = pytz.UTC
+
+
+class Metric(ExportModelOperationsMixin("metric"), models.Model):
+    """Stores a single instance of a metric value."""
+
+    class MetricName(models.TextChoices):
+        MEMBER_COUNT_TOTAL = "member_count_total", "Member Count Total"
+        SUBSCRIPTION_COUNT_TOTAL = (
+            "subscription_count_total",
+            "Subscription Count Total",
+        )
+
+    id = models.AutoField(primary_key=True)
+    creation_date = models.DateTimeField(default=timezone.now)
+    data = models.JSONField("Data")
+    name = models.CharField(
+        "Metric Name",
+        max_length=250,
+        choices=MetricName.choices,
+        default=None,
+    )
+
+    def __str__(self):
+        return f"{self.name} - {self.creation_date}"

memberportal/api_general/tasks.py → memberportal/api_metrics/tasks.py

@@ -1,12 +1,15 @@
-import api_general.metrics as metrics
+from api_metrics.models import Metric
 from profile.models import Profile
 from membermatters.celeryapp import app
 
+import requests
 from django.db.models import Count
 from constance import config
 import logging
+from channels.layers import get_channel_layer
+from asgiref.sync import async_to_sync
 
-logger = logging.getLogger("celery:api_general")
+logger = logging.getLogger("celery:api_metrics")
 
 
 @app.on_after_finalize.connect
@@ -35,13 +38,9 @@ def calculate_metrics():
         .annotate(total=Count("state"))
         .order_by("total")
     )
-    # TODO: store the last X months in the database too
-    for state in profile_states:
-        logger.debug(f"State: {state['state']} - Total: {state['total']}")
-        metrics.member_count_total.labels(state=state["state"]).set(state["total"])
-        metric_results["member_count"].append(
-            {"state": state["state"], "total": state["total"]}
-        )
+    Metric.objects.create(
+        name=Metric.MetricName.MEMBER_COUNT_TOTAL, data=profile_states.values()
+    ).full_clean()
 
     # get the count of all the different subscription states
     logger.debug("Calculating subscription count total")
@@ -51,16 +50,18 @@ def calculate_metrics():
         .annotate(total=Count("subscription_status"))
         .order_by("total")
     )
-    # TODO: store the last X months in the database too
-    for status in subscription_states:
-        logger.debug(
-            f"State: {status['subscription_status']} - Total: {status['total']}"
-        )
-        metrics.subscription_count_total.labels(
-            status=status["subscription_status"]
-        ).set(status["total"])
-        metric_results["subscription_count"].append(
-            {"status": status["subscription_status"], "total": status["total"]}
-        )
+    Metric.objects.create(
+        name=Metric.MetricName.SUBSCRIPTION_COUNT_TOTAL,
+        data=subscription_states.values(),
+    ).full_clean()
+
+    channel_layer = get_channel_layer()
+    async_to_sync(channel_layer.group_send)("api_update_prom_metrics")
+
+    try:
+        requests.post(config.SITE_URL + "/api/metrics/update_prom_metrics/")
+
+    except Exception as e:
+        logger.error(f"Failed to update Prometheus metrics: {e}")
 
     return metric_results

memberportal/api_metrics/urls.py

@@ -0,0 +1,16 @@
+from django.urls import path
+from rest_framework_simplejwt import views as jwt_views
+from . import views
+
+urlpatterns = [
+    path(
+        "api/statistics/",
+        views.Statistics.as_view(),
+        name="api_statistics",
+    ),
+    path(
+        "api/update-prom-metrics/",
+        views.UpdatePromMetrics.as_view(),
+        name="api_update_prom_metrics",
+    ),
+]

memberportal/api_metrics/views.py

@@ -0,0 +1,58 @@
+import logging
+import json
+
+from rest_framework import permissions
+
+import api_metrics.metrics
+from api_metrics.models import Metric
+from api_general.models import SiteSession
+
+from rest_framework.response import Response
+from rest_framework.views import APIView
+
+logger = logging.getLogger("metrics")
+
+
+class Statistics(APIView):
+    """
+    get: gets site statistics.
+    """
+
+    def get(self, request):
+        members = SiteSession.objects.filter(signout_date=None).order_by("-signin_date")
+        member_list = []
+
+        for member in members:
+            member_list.append(member.user.profile.get_full_name())
+
+        statistics = {"onSite": {"members": member_list, "count": members.count()}}
+
+        return Response(statistics)
+
+
+class UpdatePromMetrics(APIView):
+    """
+    post: triggers Django to update the Prometheus site metrics from the database.
+    """
+
+    permission_classes = (permissions.AllowAny,)
+
+    def post(self, request):
+        # get the latest distinct metric
+        metrics = Metric.objects.order_by("creation_date").distinct("metric_name")
+
+        for metric in metrics:
+            if metric.metric_name in [
+                Metric.MetricName.MEMBER_COUNT_TOTAL,
+                Metric.MetricName.SUBSCRIPTION_COUNT_TOTAL,
+            ]:
+                prom_metric = getattr(api_metrics.metrics, metric.name)
+
+                if not prom_metric:
+                    logger.error(f"Prometheus metric {metric.name} not found.")
+                    continue
+
+                for state in metric.data:
+                    prom_metric.labels(state=state["state"]).set(state["total"])
+
+        return Response()

memberportal/membermatters/celeryapp.py

@@ -1,7 +1,5 @@
 import os
 from celery import Celery
-from celery.signals import worker_init
-from prometheus_client import CollectorRegistry, multiprocess, start_http_server
 import logging
 
 logger = logging.getLogger("celery:celeryapp")
@@ -24,18 +22,3 @@
 @app.task(bind=True)
 def debug_task(self):
     logger.debug(f"Request: {self.request!r}")
-    print(f"Request: {self.request!r}")
-
-
-@worker_init.connect
-def celery_prom_server(sender=None, conf=None, **kwargs):
-    print("Starting CollectorRegistry() and multiprocess.MultiProcessCollector()...")
-    logger.info(
-        "Starting CollectorRegistry() and multiprocess.MultiProcessCollector()..."
-    )
-    registry = CollectorRegistry()
-    multiprocess.MultiProcessCollector(registry)
-
-    logger.info("Starting Prometheus metrics server on port 8000...")
-    start_http_server(8000, registry=registry)
-    logger.info("Prometheus metrics server started on port 8000!")

memberportal/membermatters/settings.py

@@ -68,6 +68,7 @@
     "api_meeting",
     "api_admin_tools",
     "api_billing",
+    "api_metrics",
     "corsheaders",
     "rest_framework",
     "rest_framework_api_key",

memberportal/membermatters/urls.py

@@ -40,6 +40,7 @@ def safe_constance_get(fld: str):
 
 urlpatterns = [
     path("api/openid/", include("oidc_provider.urls", namespace="oidc_provider")),
+    path("", include("api_metrics.urls")),
     path("", include("api_spacedirectory.urls")),
     path("", include("api_general.urls")),
     path("", include("api_access.urls")),