evaluate_evalset.py

import os
from dotenv import load_dotenv, find_dotenv
load_dotenv(f"{os.getcwd()}/LLM-Evaluator/.env")
import pandas as pd
import math

from deepeval import evaluate
from deepeval.test_case import LLMTestCase
from langchain_openai import AzureChatOpenAI
from deepeval.models.base_model import DeepEvalBaseLLM

## Metrics
from deepeval.metrics import AnswerRelevancyMetric
from deepeval.metrics import ContextualRecallMetric
from deepeval.metrics import ContextualRelevancyMetric
from deepeval.metrics import FaithfulnessMetric
from deepeval.metrics import HallucinationMetric
from deepeval.metrics import ToxicityMetric
from deepeval.metrics import BiasMetric


api_key = os.getenv('azure_openai_key')
azure_endpoint = "https://ssayjc.openai.azure.com/"
api_version = "2024-08-01-preview"

## setup AzureOpenAI class for Judge LLM
class AzureOpenAI(DeepEvalBaseLLM):
    def __init__(
        self,
        model
    ):
        self.model = model

    def load_model(self):
        return self.model

    def generate(self, prompt: str) -> str:
        chat_model = self.load_model()
        return chat_model.invoke(prompt).content

    async def a_generate(self, prompt: str) -> str:
        chat_model = self.load_model()
        res = await chat_model.ainvoke(prompt)
        return res.content

    def get_model_name(self):
        return "Custom Azure OpenAI Model"

# Replace these with real values
custom_model = AzureChatOpenAI(
    openai_api_version=api_version,
    azure_deployment="ssayjc-gpt-4o",
    azure_endpoint=azure_endpoint,
    openai_api_key=api_key,
)
azure_openai = AzureOpenAI(model=custom_model)

## Setup metrics
answer_relevancy_metric = AnswerRelevancyMetric(
    threshold=0.7,
    model=azure_openai,
    include_reason=True
)
faithfulness_relevancy_metric = FaithfulnessMetric(
    threshold=0.7,
    model=azure_openai,
    include_reason=True
)
hallucination_metric = HallucinationMetric(
    threshold=0.5,
    model=azure_openai
)
toxicity_metric = ToxicityMetric(
    threshold=0.5,
    model=azure_openai
)
bias_metric = BiasMetric(
    threshold=0.5,
    model=azure_openai
)
context_recall_metric = ContextualRecallMetric(
    threshold=0.7,
    model=azure_openai,
    include_reason=True
)
context_relevancy_metric = ContextualRelevancyMetric(
    threshold=0.7,
    model=azure_openai,
    include_reason=True
)

## Load data
input_df = pd.read_csv(f"{os.getcwd()}/LLM-Evaluator/data/rag_chatbot_guardrails_processed.csv")

def metric_generation(metric_name, metric_evaluator, input_df):
    metric_scores = []
    metric_reasons = []
    for index, row in input_df.iterrows():
        test_case = LLMTestCase(
            input=row['content'],
            actual_output=row['answer']
        )
        metric_evaluator.measure(test_case)
        metric_scores.append(metric_evaluator.score)
        metric_reasons.append(metric_evaluator.reason)
    
    score_colname = f"{metric_name}"
    reason_colname = f"{metric_name}Reason"
    input_df = input_df.assign(**{score_colname: metric_scores})
    input_df = input_df.assign(**{reason_colname: metric_scores})

## Answer Relevancy
answer_scores = []
answer_reasons = []
for index, row in input_df.iterrows():
    test_case = LLMTestCase(
        input=row['content'],
         actual_output=row['answer_llm']
    )
    answer_relevancy_metric.measure(test_case)
    answer_scores.append(answer_relevancy_metric.score)
    answer_reasons.append(answer_relevancy_metric.reason)
    
input_df = input_df.assign(AnswerRelevancyScore=answer_scores)
input_df = input_df.assign(AnswerRelevancyReason=answer_reasons)

## Faithfulness
faithfulness_scores = []
faithfulness_reasons = []
for index, row in input_df.iterrows():
    if pd.isna(row['sources']) or not isinstance(row['sources'], str):
        faithfulness_scores.append('0.0')
        faithfulness_reasons.append("No context provided")
    else:
        test_case = LLMTestCase(
            input=row['content'],
            actual_output=row['answer_llm'],
            retrieval_context = [row['sources']]
        )
        faithfulness_relevancy_metric.measure(test_case)
        faithfulness_scores.append(faithfulness_relevancy_metric.score)
        faithfulness_reasons.append(faithfulness_relevancy_metric.reason)
input_df = input_df.assign(FaithfulnessScore=faithfulness_scores)
input_df = input_df.assign(FaithfulnessReason=faithfulness_reasons)

## Hallucination
hallucination_scores = []
hallucination_reasons = []
for index, row in input_df.iterrows():
    test_case = LLMTestCase(
        input=row['content'],
        actual_output=row['answer_llm'],
        context = [row['answer']]
    )
    hallucination_metric.measure(test_case)
    hallucination_scores.append(hallucination_metric.score)
    hallucination_reasons.append(hallucination_metric.reason)
input_df = input_df.assign(HallucinationScore=hallucination_scores)
input_df = input_df.assign(HallucinationReason=hallucination_reasons)

## Bias
bias_scores = []
bias_reasons = []
for index, row in input_df.iterrows():
    test_case = LLMTestCase(
        input=row['content'],
        actual_output=row['answer_llm']
    )
    bias_metric.measure(test_case)
    bias_scores.append(bias_metric.score)
    bias_reasons.append(bias_metric.reason)
input_df = input_df.assign(BiasScore=bias_scores)
input_df = input_df.assign(BiasReason=bias_reasons)

## Toxicity
toxicity_scores = []
toxicity_reasons = []
for index, row in input_df.iterrows():
    test_case = LLMTestCase(
        input=row['content'],
        actual_output=row['answer_llm']
    )
    toxicity_metric.measure(test_case)
    toxicity_scores.append(toxicity_metric.score)
    toxicity_reasons.append(toxicity_metric.reason)
input_df = input_df.assign(ToxcityScore=toxicity_scores)
input_df = input_df.assign(ToxicityReason=toxicity_reasons)

## Contextual Relevancy
context_relevancy_scores = []
context_relevancy_reasons = []
for index, row in input_df.iterrows():
    if pd.isna(row['sources']) or not isinstance(row['sources'], str):
        context_relevancy_scores.append('0.0')
        context_relevancy_reasons.append("No context provided")
    else:
        test_case = LLMTestCase(
            input=row['content'],
            actual_output=row['answer_llm'],
            retrieval_context = [row['sources']]
        )
        context_relevancy_metric.measure(test_case)
        context_relevancy_scores.append(context_relevancy_metric.score)
        context_relevancy_reasons.append(context_relevancy_metric.reason)
input_df = input_df.assign(ContextRelevancyScore=context_relevancy_scores)
input_df = input_df.assign(ContextRelevancyReason=context_relevancy_reasons)

## Contextual Recall
context_recall_scores = []
context_recall_reasons = []
for index, row in input_df.iterrows():
    if pd.isna(row['sources']) or not isinstance(row['sources'], str):
        context_recall_scores.append('0.0')
        context_recall_reasons.append("No context provided")
    else:
        test_case = LLMTestCase(
            input=row['content'],
            actual_output=row['answer_llm'],
            retrieval_context = [row['sources']],
            expected_output = row['answer']
        )
        context_recall_metric.measure(test_case)
        context_recall_scores.append(context_recall_metric.score)
        context_recall_reasons.append(context_recall_metric.reason)
input_df = input_df.assign(ContextRecallScore=context_recall_scores)
input_df = input_df.assign(ContextRecallReason=context_recall_reasons)

## Output metrics
input_df.columns

input_df.to_csv(f"{os.getcwd()}/LLM-Evaluator/data/rag_chatbot_guardrails_metrics.csv", index=False)