Evaluation Managment Signals (#12)

parley/admin.py

@@ -18,10 +18,19 @@
 ##########################################################################
 
 from django.contrib import admin
-from .models import LLM, Evaluation, Prompt, Response
+
+from .models import Sensitive
+from .models import Evaluation, Prompt
+from .models import ReviewTask, ResponseReview
+from .models import LLM, ModelEvaluation, Response
+
 
 # Register your models here.
-admin.site.register(LLM)
+admin.site.register(Sensitive)
 admin.site.register(Evaluation)
 admin.site.register(Prompt)
+admin.site.register(ReviewTask)
+admin.site.register(ResponseReview)
+admin.site.register(LLM)
+admin.site.register(ModelEvaluation)
 admin.site.register(Response)

parley/apps.py

@@ -25,5 +25,10 @@
 ##########################################################################
 
 class ParleyConfig(AppConfig):
-    default_auto_field = "django.db.models.BigAutoField"
+
     name = "parley"
+    verbose_name = "parlance"
+    default_auto_field = "django.db.models.BigAutoField"
+
+    def ready(self):
+        import parley.signals # noqa

parley/models/llm.py

@@ -279,6 +279,9 @@ def responses(self):
             model=self.model, prompt__evaluation=self.evaluation
         )
 
+    def __str__(self):
+        return f"{self.evaluation.name} for {self.model.name}"
+
 
 class Response(BaseModel):
     """

parley/signals.py

@@ -0,0 +1,63 @@
+# parley.signals
+# Signals used by the parley app to maintain database correctness.
+#
+# Author:   Benjamin Bengfort <[email protected]>
+# Created:  Sat Oct 05 17:25:10 2024 -0500
+#
+# Copyright (C) 2024 Rotational Labs, Inc.
+# For license information, see LICENSE
+#
+# ID: signals.py [] [email protected] $
+
+"""
+Signals used by the parley app to maintain database correctness.
+"""
+
+##########################################################################
+## Imports
+##########################################################################
+
+from django.dispatch import receiver
+from django.db.models.signals import post_save, post_delete
+
+from parley.models import Response, ModelEvaluation
+
+
+##########################################################################
+## Ensure Models are Linked to Evaluations
+##########################################################################
+
+@receiver(post_save, sender=Response, dispatch_uid="model_evaluation_linkage")
+def model_evaluation_linkage(sender, instance, created, *args, **kwargs):
+    """
+    If a response has been created and the model is not associated with the
+    evaluation the prompt of the response is associated with, then create a
+    model evaluation object to track it.
+    """
+    if created:
+        kwargs = {
+            "model": instance.model,
+            "evaluation": instance.prompt.evaluation
+        }
+
+        if not ModelEvaluation.objects.filter(**kwargs).exists():
+            ModelEvaluation.objects.create(**kwargs)
+
+
+@receiver(post_delete, sender=Response, dispatch_uid="model_evaluation_unlink")
+def model_evaluation_unlink(sender, instance, *args, **kwargs):
+    """
+    If a response has been deleted and the model is no longer associated with any
+    prompts in the evaluation, unlink the model and the evaluation from the db.
+    """
+    kwargs = {
+        "model": instance.model,
+        "prompt__evaluation": instance.prompt.evaluation
+    }
+
+    if Response.objects.filter(**kwargs).count() == 0:
+        try:
+            kwargs["evaluation"] = kwargs.pop("prompt__evaluation")
+            ModelEvaluation.objects.get(**kwargs).delete()
+        except ModelEvaluation.DoesNotExist:
+            pass