SAQUET.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 81,
   "id": "b3658995-00c0-4902-a90e-70b9d97d9a61",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Import the required libraries, lots of these are required for the LLMs we utilize for three criteria. \n",
    "import Levenshtein\n",
    "import spacy\n",
    "import string\n",
    "import pandas as pd\n",
    "from transformers import pipeline, AutoTokenizer, AutoModel, AutoModelForSequenceClassification\n",
    "import torch\n",
    "from sentence_transformers import SentenceTransformer, util\n",
    "from lexicalrichness import LexicalRichness\n",
    "import re\n",
    "import math\n",
    "import random\n",
    "import numpy as np\n",
    "nlp = spacy.load('en_core_web_lg')\n",
    "import time\n",
    "from datetime import datetime\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "\n",
    "#NLTK Imports\n",
    "import nltk\n",
    "nltk.download('stopwords') #Needed for query wellformedness\n",
    "nltk.download('punkt') #Needed for query wellformedness\n",
    "nltk.download('averaged_perceptron_tagger') #Needed for query wellformedness\n",
    "nltk.download('wordnet') #Another one\n",
    "from nltk.corpus import wordnet as wn\n",
    "from nltk.stem import WordNetLemmatizer\n",
    "from nltk.corpus import stopwords\n",
    "stop_words = set(stopwords.words('english'))\n",
    "from nltk.tokenize import word_tokenize, sent_tokenize\n",
    "\n",
    "#OpenAI, but could be replaced with Gemini, Claude, etc.\n",
    "import openai\n",
    "openai.api_key = 'YOUR_KEY_HERE'\n",
    "model_engine = \"gpt-4\" #For all popular LLMS, GPT-4 has proven the best for this process \n",
    "\n",
    "#Libraries for Perplexity, Diversity, Grammatical Error, Complexity, Answerability\n",
    "from evaluate import load\n",
    "from transformers import OpenAIGPTTokenizer, OpenAIGPTLMHeadModel, GPT2LMHeadModel, GPT2TokenizerFast\n",
    "from nltk import trigrams\n",
    "from nltk import ngrams\n",
    "import language_tool_python\n",
    "\n",
    "#Our class used to represent multiple-choice questions\n",
    "nl = '\\n'\n",
    "class MultipleChoiceQuestion:\n",
    "    def __init__(self, stem, options, correct_option, qid = None, quality = None):\n",
    "        self.stem = stem\n",
    "        self.options = options\n",
    "        self.correct_option = correct_option\n",
    "        self.qid = qid\n",
    "        self.quality = quality\n",
    "        \n",
    "    def __str__(self):\n",
    "        return f\"Question: {self.stem}\\n {nl.join(self.options)}\\nCorrect option: {self.correct_option}\\nQuality: {self.quality}\""
   ]
  },
  {
   "cell_type": "markdown",
   "id": "581b3cd5-19c2-4af5-b973-57448e8f9902",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true
   },
   "source": [
    "## Implausible Distractors\n",
    "Make all distractors plausible as good items depend on having effective distractors\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "id": "ee242951-010a-4fc7-aa24-a973ad3fd260",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Uses NER, so if the score is too low, if they're matching entities (i.e. people) then we can ignore this case and say True\n",
    "def implausible_distractors(question):\n",
    "    #MiniLM from: https://huggingface.co/sentence-transformers/all-MiniLM-L6-v2\n",
    "    model = SentenceTransformer('all-MiniLM-L6-v2')\n",
    "    correct = question.correct_option\n",
    "    options = question.options.copy()\n",
    "\n",
    "    for opt in question.correct_option.split('[SEP]'):\n",
    "        try:\n",
    "            options.remove(opt)\n",
    "        except:\n",
    "            print('error trying to remove an option, there might be an incorrect space present: ', opt)\n",
    "\n",
    "    # Two lists of sentences\n",
    "    sentences1 = [correct, correct, correct, correct]\n",
    "    sentences2 = options\n",
    "\n",
    "    #Compute embedding for both lists\n",
    "    embeddings1 = model.encode(sentences1, convert_to_tensor=True)\n",
    "    embeddings2 = model.encode(sentences2, convert_to_tensor=True)\n",
    "\n",
    "    #Compute cosine-similarities\n",
    "    cosine_scores = util.cos_sim(embeddings1, embeddings2)\n",
    "\n",
    "    #Output the pairs with their score\n",
    "    for i in range(len(sentences2)):\n",
    "        if cosine_scores[i][i] < 0.15:\n",
    "            \n",
    "            #NER check here...\n",
    "            opt_entity = nlp(sentences2[i])\n",
    "            lemma_nouns_opt = get_lemma_nouns(sentences2[i])\n",
    "            \n",
    "            ans_entity = nlp(sentences1[i])\n",
    "            lemma_nouns_ans = get_lemma_nouns(sentences1[i])\n",
    "\n",
    "            #If the noun(s) in the answer choice can be tagged with an entity\n",
    "            if ans_entity.ents:\n",
    "                answer_entity = ans_entity.ents[0].label_\n",
    "            else:\n",
    "                answer_entity = None\n",
    "\n",
    "            if opt_entity.ents:\n",
    "                opt_entity = opt_entity.ents[0].label_\n",
    "            else:\n",
    "                opt_entity = None\n",
    "\n",
    "            #Couldn't find the noun nor the entity? Unable to parse effectively to make a judgement.\n",
    "            if len(lemma_nouns_ans) == 0 and len(lemma_nouns_opt) == 0:\n",
    "                return True\n",
    "            \n",
    "            #If the option in this case is none/all of the above, it won't be similar, so ignore this criteria\n",
    "            if not all_of_the_above(question) or not none_of_the_above(question):\n",
    "                return True\n",
    "\n",
    "            #Low distance like this means it likely shares some words and should not be flagged\n",
    "            if jaccard_similarity(sentences1[i], sentences2[i]) > .15 or Levenshtein.distance(sentences1[i], sentences2[i]) < (len(sentences1[i])*.7):\n",
    "                return True\n",
    "\n",
    "            #Before saying two distractors are plausible, let's have GPT-4 make a judgement call\n",
    "            #If GPT-4 is too generous/strict on this call, we can try using updated word embeddings from openai which might be better for the domain jargon\n",
    "            if are_they_similar(question.stem, sentences1[i], sentences2[i]):\n",
    "                print('LLM says they are similar: ', sentences1[i], ' -and- ', sentences2[i])\n",
    "                return True\n",
    "\n",
    "            print(\"Distractor not similar enough: {} \\t\\t {} \\t\\t Score: {:.4f}\".format(sentences1[i], sentences2[i], cosine_scores[i][i]))\n",
    "            return False\n",
    "            \n",
    "    return True\n",
    "\n",
    "#Statistic used for gauging the similarity and diversity of text\n",
    "def jaccard_similarity(str1, str2):\n",
    "    # Convert strings to sets of words\n",
    "    set1 = set(str1.split())\n",
    "    set2 = set(str2.split())\n",
    "\n",
    "    # Calculate intersection and union\n",
    "    intersection = set1.intersection(set2)\n",
    "    union = set1.union(set2)\n",
    "\n",
    "    # Calculate Jaccard Similarity\n",
    "    similarity = len(intersection) / len(union)\n",
    "    return similarity\n",
    "\n",
    "#The prompt is very basic, but has proved effective through a variety of domains\n",
    "def are_they_similar(stem, answer, option):\n",
    "    sysrole = \"\"\"You are an expert and an astute instructor. Given two options to a multiple-choice question, you will respond 'Yes' if they are related in some way or if option 2 is a somewhat plausible distractor to the question. You will respond with 'No' if the two options are not related or if option 2 is not a somewhat plausible distractor for the question.\n",
    "    Effectively you are seeing if both options are potential plausible distractors or answers for the question, but they do not need to be perfect.\n",
    "    \"\"\"\n",
    "    prompt = \"\"\"question: {}\n",
    "    option 1: {}\n",
    "    option 2: {}\n",
    "    \"\"\".format(stem, answer, option)\n",
    "    done = False\n",
    "    expert_reasoning = 'blank'\n",
    "    \n",
    "    # Generate a response\n",
    "    while(done == False):\n",
    "        try:\n",
    "            o = openai.chat.completions.create(\n",
    "              model=model_engine,\n",
    "              messages=[\n",
    "                 {\"role\": \"system\",\n",
    "                  \"content\": sysrole},\n",
    "                {\"role\": \"user\", \n",
    "                 \"content\": prompt},\n",
    "              ],\n",
    "              max_tokens = 4096,\n",
    "              temperature = 0.7\n",
    "             )\n",
    "            done = True \n",
    "        except Exception as error:\n",
    "            print('errored in LLM API call: ', error)\n",
    "            time.sleep(10)\n",
    "    completion = o\n",
    "    done = False\n",
    "\n",
    "    try:\n",
    "        expert_reasoning = completion.choices[0].message.content.lower()\n",
    "    except: \n",
    "        print('error with LLM: ', completion)\n",
    "    if 'yes' in expert_reasoning:\n",
    "        return True\n",
    "    elif 'no' in expert_reasoning:\n",
    "        return False\n",
    "        \n",
    "    return expert_reasoning"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2a56569c-84cc-41ba-8077-c54f508d4487",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true
   },
   "source": [
    "## None Of The Above\n",
    "Avoid none of the above as it only really measures students ability to detect incorrect answers "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "id": "cd69171a-423c-4dee-a69a-c977b9850e07",
   "metadata": {},
   "outputs": [],
   "source": [
    "def none_of_the_above(question):\n",
    "    for opt in question.options:\n",
    "        cleaned_opt = opt.strip().lower()\n",
    "        if 'none of the above' in cleaned_opt or ('none' in cleaned_opt and 'above' in cleaned_opt) or cleaned_opt.startswith('none of') or cleaned_opt == 'neither' or 'none' in question.options[len(question.options)-1]:\n",
    "            return False\n",
    "    return True"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1ff1b534-99e1-418d-920c-2f0ecc45a6bc",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true
   },
   "source": [
    "## All Of The Above\n",
    "Avoid all of the above options as students can guess correct responses based on partial information"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "id": "c3084749-6f1b-42fb-9cd4-2b761b7d7b68",
   "metadata": {},
   "outputs": [],
   "source": [
    "def all_of_the_above(question):\n",
    "    for opt in question.options:\n",
    "        cleaned_opt = opt.strip().lower()\n",
    "        if 'all of the above' in cleaned_opt or ('all' in cleaned_opt and 'above' in cleaned_opt) or ('all if the' in cleaned_opt)  or ('all of the' in cleaned_opt):\n",
    "            return False\n",
    "\n",
    "    return True"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "09faad15-79dc-4cfd-b276-28000edd7843",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true
   },
   "source": [
    "## Fill-In-The-Blank\n",
    "Avoid omitting words in the middle of the stem that students must insert from the options provided "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "id": "b00becc5-6f57-4466-b514-78a4527848ed",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Programming questions might contain a single underscore, so check for multiple\n",
    "def fill_in_the_blank(question):\n",
    "    if \"__\" in question.stem or ('fill in the blank' in question.stem.lower()):\n",
    "        return False\n",
    "    \n",
    "    return True"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bef9014d-8f97-45ef-bccc-ad6fd6f8c823",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true
   },
   "source": [
    "## True/False\n",
    "The options should not be a series of true/false statements"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "id": "bfe524b1-9144-4b3d-a3db-67b572a20958",
   "metadata": {},
   "outputs": [],
   "source": [
    "def true_or_false(question):\n",
    "    options = question.options.copy()\n",
    "    \n",
    "    #Check for true & false mentioned in the stem\n",
    "    for sent in question.stem.split('.'):\n",
    "        sent = sent.lower()\n",
    "        if 'false' in sent and 'true' in sent:\n",
    "            return False    \n",
    "    \n",
    "    for opt in options:\n",
    "        cleaned_opt = opt.strip().lower() \n",
    "        if cleaned_opt == 'true' or cleaned_opt == 'false' or cleaned_opt == 'yes'or cleaned_opt == 'no':\n",
    "            return False\n",
    "\n",
    "    return True"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "dcbc51e2-e837-41a3-b4c8-985ef27c671e",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true
   },
   "source": [
    "## Absolute Terms\n",
    "Avoid the use of absolute terms (e.g. never, always, all) in both the question stem as it can be confusing and the options as students are aware that they are almost always false"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "id": "11b7ca8e-36f7-4e6c-8ae1-7a244b70227d",
   "metadata": {},
   "outputs": [],
   "source": [
    "#The list of absolute terms can be different for the stem and options, but we need to be careful here, as sometimes these can be used in proper ways\n",
    "absolutes = [\"always\", \"never\", \"none\", \"all\", \"completely\", \"absolutely\", \"totally\", \"definitely\", \"incapable\", \"inevitable\"]\n",
    "def absolute_terms(question):\n",
    "\n",
    "    #Check for terms in the question stem, if we we find any, have GPT-4 help us verify the use of it.\n",
    "    stem = question.stem.lower()\n",
    "    if any(word in stem.split() for word in absolutes):\n",
    "        if not true_or_false(question):\n",
    "            return True\n",
    "        else:\n",
    "            return absolute_terms_verify(stem)#False\n",
    "\n",
    "    #Check for terms in the options, if we we find any, have GPT-4 help us verify the use of it.\n",
    "    absolutes_options = [\"always\", \"never\", \"none\", \"completely\", \"absolutely\", \"totally\", \"definitely\", \"incapable\", \"inevitable\", \"all\"]\n",
    "    for opt in question.options:\n",
    "        cleaned_opt = opt.strip().lower()\n",
    "        \n",
    "        #Count all, which is a special case, but not in the case of \"all of the above\"\n",
    "        if any(word in cleaned_opt for word in absolutes_options):        \n",
    "            if none_of_the_above(question) and all_of_the_above(question) and true_or_false(question):\n",
    "                if \"all\" in cleaned_opt: \n",
    "                    return not absolute_terms_verify(cleaned_opt)\n",
    "                return False\n",
    "    \n",
    "    return True\n",
    "\n",
    "#Prompt can be modified, specifically the five terms we provide as an example, but this one has proven quite successful. \n",
    "def absolute_terms_verify(prompt):\n",
    "    sysrole = \"\"\"You are an expert, nitpicky, and astute instructor. \n",
    "    Given a phrase that is used as part of a multiple-choice question, you will check if contains absolute terms, such as {}, that are used in a way which constitutes a blanket generalization or hyperbole. \n",
    "    It is fine if the text contains these terms, as long as they are not used in a way that might signal the phrase is clearly correct or incorrect.\n",
    "    You will respond 'Yes' if they are used in this way or 'No' if they are not.\n",
    "    \"\"\".format(\" \".join(['all', 'only', 'always', 'never', 'none']))\n",
    "    \n",
    "    done = False\n",
    "    expert_reasoning = 'blank'\n",
    "    # Generate a response\n",
    "    while(done == False):\n",
    "        try:\n",
    "            o = openai.chat.completions.create(\n",
    "              model=model_engine,\n",
    "              messages=[\n",
    "                 {\"role\": \"system\",\n",
    "                  \"content\": sysrole},\n",
    "                {\"role\": \"user\", \n",
    "                 \"content\": prompt},\n",
    "              ],\n",
    "              max_tokens = 4096,\n",
    "              temperature = 0.7\n",
    "             )\n",
    "            done = True \n",
    "        except Exception as error:\n",
    "            print('errored in GPT-4 API call: ', error)\n",
    "            time.sleep(10)\n",
    "    completion = o\n",
    "    done = False\n",
    "\n",
    "    try:\n",
    "        expert_reasoning = completion.choices[0].message.content.lower()\n",
    "    except: \n",
    "        print('error with LLM: ', completion)\n",
    "        \n",
    "    #Means an absolute term was used in a manner that makes the question flawed\n",
    "    if 'yes' in expert_reasoning: \n",
    "        return True\n",
    "    elif 'no' in expert_reasoning:\n",
    "        return False\n",
    "        \n",
    "    return expert_reasoning"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "083615c2-0b27-43c3-8bf6-2b5bfe42444b",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true
   },
   "source": [
    "## Longest Answer Correct\n",
    "Often the correct option is longer and includes more detailed information, which clues students to this option"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "id": "5d4b8509-78f6-470a-8263-70f527edf130",
   "metadata": {},
   "outputs": [],
   "source": [
    "#If the correct answer is noticably longer (more than 25%) than the second longest answer, flag it.\n",
    "def longest_answer_correct(question):\n",
    "\n",
    "    #Ignore this criteria for True/False questions\n",
    "    if not true_or_false(question) or '[SEP]' in question.correct_option:\n",
    "        return True\n",
    "        \n",
    "    correct = question.correct_option\n",
    "    options = question.options.copy()\n",
    "    for opt in question.correct_option.split('[SEP]'):\n",
    "        options.remove(opt.strip())\n",
    "\n",
    "    longest_option = 0\n",
    "    for opt in options:\n",
    "        if len(opt) >= longest_option:\n",
    "            longest_option = len(opt)\n",
    "        \n",
    "    #If the longest option is only by 25% or it's a three words or less, then this passes\n",
    "    if longest_option >= len(correct) *.75 or len(correct.split()) < 4:\n",
    "        return True\n",
    "    \n",
    "    return False"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3d3c2c16-62c1-41d9-891b-90d3f9e43b8d",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true
   },
   "source": [
    "## Negative worded\n",
    "Negatively worded stems are less likely to measure important learning outcomes and can confuse students"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "id": "95d72d49-6504-47d8-899f-cddcc4a1bab5",
   "metadata": {},
   "outputs": [],
   "source": [
    "#The list of negative words can potentially cause this to be too restrictive, particularly for words such as can't and won't\n",
    "def negative_worded_stem(question):\n",
    "    negatives = [\"none\", \"never\", \"without\", \"exclude\", \"deny\", \"refuse\", \"oppose\", \"dispute\", \"can't\", \"won't\", \"not\"] \n",
    "\n",
    "    stem = question.stem.lower()\n",
    "    if any(word in stem.split() for word in negatives):\n",
    "        return False\n",
    "\n",
    "    for sent in question.stem.split('.'):\n",
    "        sent = sent.lower()        \n",
    "        if 'which' in sent and ('false' in sent or 'not' in sent or 'incorrect' in sent or 'except' in sent) or \\\n",
    "        'what' in sent and ('false' in sent or 'not' in sent or 'incorrect' in sent or 'except' in sent):\n",
    "            return False    \n",
    "    \n",
    "    return True"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bfbf7aa1-71aa-4895-beca-ad0aec7b7a05",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true
   },
   "source": [
    "## Word Repeats\n",
    "Avoid similarly worded stems and correct responses or words repeated in the stem and correct response"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "id": "fb0bbeb0-01b2-4045-9b5d-762d6c39fb81",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Find the nouns in question.correct_option and question.stem --> stem them --> compare cosine similiary (usin sentence transformer)\n",
    "#Also check for the synonyms, compare them. However, if the word(s) are used in the other options, then it's fine.\n",
    "#Nouns: NN noun, singular ‘- desk’, NNS noun plural – ‘desks’, NNP proper noun, singular – ‘Harrison’, NNPS proper noun, plural – ‘Americans’ \n",
    "lemmatizer = WordNetLemmatizer()\n",
    "nouns = ['NN', 'NNS', 'NNP', 'NNPS']\n",
    "\n",
    "def word_repeats_in_stem_and_correct_answer(question):   \n",
    "    options = question.options.copy()\n",
    "    \n",
    "    all_options = ' '.join(options)\n",
    "    for opt in question.correct_option.split('[SEP]'):\n",
    "        options.remove(opt.strip())\n",
    "        \n",
    "    #This code checks for matching words, specifically nouns and verbs, between the correct answer and stem\n",
    "    word_types = ['NN', 'NNS', 'NNP', 'NNPS', 'VB', 'VBG', 'VBN', 'VBP', 'VBZ']\n",
    "    stem = strip_punctuation(question.stem)\n",
    "    matching_words = []\n",
    "    for wrd in stem.split():\n",
    "        if wrd not in stop_words and wrd in question.correct_option:\n",
    "            matching_words.append(wrd)\n",
    "            \n",
    "    matching_words = list(set(matching_words)).copy()\n",
    "    matching_words_copy = matching_words.copy()\n",
    "    for wrd in matching_words:\n",
    "        for opt in options:\n",
    "            if wrd in opt:\n",
    "                matching_words_copy.remove(wrd)\n",
    "                break\n",
    "\n",
    "    #If the word is longer than 4 characters, because non-matching verbs/nouns of smaller characters typically are not cues\n",
    "    if len([s for s in matching_words_copy if len(s) >= 4]) > 0:\n",
    "        again = []\n",
    "        tagged = nltk.pos_tag(matching_words_copy)\n",
    "        for t in tagged:\n",
    "            if t[1] in word_types:\n",
    "                again.append(t[0].lower())\n",
    "        if len(again) > 0:\n",
    "            if '[SEP]' in question.correct_option:\n",
    "                for mwc in matching_words_copy:\n",
    "                    if mwc in question.correct_option.split('[SEP]')[0] and mwc in question.correct_option.split('[SEP]')[1]:\n",
    "                        print('*** SEP')\n",
    "                        return False\n",
    "                    else:\n",
    "                        return True\n",
    "            else:\n",
    "                #There's the potential false positive where all answer choices are repeated in the question's stem\n",
    "                all_ops_in_stem = 0\n",
    "                for opt in question.options:\n",
    "                    opt = opt.lower()\n",
    "                    stem = question.stem.lower()\n",
    "                    if opt in stem:\n",
    "                        all_ops_in_stem = all_ops_in_stem + 1\n",
    "                if all_ops_in_stem == len(question.options):\n",
    "                    return True\n",
    "                \n",
    "                return False\n",
    "    return True\n",
    "\n",
    "def strip_punctuation(text):\n",
    "    return ''.join(char for char in text if char not in string.punctuation)\n",
    "\n",
    "#This is now used for Logical Cue\n",
    "def get_lemma_nouns(text):\n",
    "    all_nouns = []\n",
    "    tokenized = sent_tokenize(text)\n",
    "    \n",
    "    for i in tokenized:\n",
    "\n",
    "        # Word tokenizers is used to find the words and punctuation in a string\n",
    "        wordsList = nltk.word_tokenize(i)\n",
    "\n",
    "        # removing stop words from wordList\n",
    "        wordsList = [w for w in wordsList if not w in stop_words]\n",
    "\n",
    "        # Using a Tagger. Which is part-of-speech tagger or POS-tagger.\n",
    "        tagged = nltk.pos_tag(wordsList)\n",
    "        \n",
    "        # Add any nouns to this list\n",
    "        for t in tagged:\n",
    "            if t[1] in nouns:\n",
    "                all_nouns.append(t[0].lower())\n",
    "    \n",
    "    lemmatized_nouns = []\n",
    "    for n in all_nouns:\n",
    "        lemmatized_word = lemmatizer.lemmatize(n, pos=\"n\")\n",
    "        lemmatized_nouns.append(lemmatized_word.lower())\n",
    "    \n",
    "    return lemmatized_nouns"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1a49c0c9-68fd-4ed5-b307-cd55228c6a1e",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true
   },
   "source": [
    "## Logical Cue - This one is challenging, requires domain knowledge."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "id": "6a77330d-515c-4219-8691-edd5c6db81b0",
   "metadata": {},
   "outputs": [],
   "source": [
    "#An example of a logical cue is asking students to select the most appropriate pharmaceutical intervention for a problem and only having one or two options which\n",
    "#Using NER, if the question asks for a <certain type of noun, like a <person> then the options should all be <people> too.\n",
    "\n",
    "def avoid_logical_cues(question):\n",
    "    options = question.options.copy()\n",
    "\n",
    "    for opt in question.correct_option.split('[SEP]'):\n",
    "        options.remove(opt.strip())\n",
    "    \n",
    "    if len(options) < 2:\n",
    "        return True\n",
    "\n",
    "    #Works for MCQs with up to 5 options, any more and this will break, but eventually I'll change this to work for MCQs of any option length\n",
    "    lemma_nouns_answ = get_lemma_nouns(question.correct_option)\n",
    "    if len(options) == 2:\n",
    "        lemma_nouns_options = [get_lemma_nouns(options[0]), get_lemma_nouns(options[1])]\n",
    "    if len(options) == 3:\n",
    "        lemma_nouns_options = [get_lemma_nouns(options[0]), get_lemma_nouns(options[1]), get_lemma_nouns(options[2])]\n",
    "    if len(options) == 4:\n",
    "        lemma_nouns_options = [get_lemma_nouns(options[0]), get_lemma_nouns(options[1]), get_lemma_nouns(options[2]), get_lemma_nouns(options[3])]\n",
    "        \n",
    "    entities_in_options = []\n",
    "    for opt in lemma_nouns_options:\n",
    "        for val in opt:\n",
    "            doc = nlp(val)\n",
    "            if doc.ents:\n",
    "                entities_in_options.append(doc.ents[0].label_)\n",
    "    \n",
    "    entities_in_answer = []\n",
    "  \n",
    "    for val in lemma_nouns_answ:\n",
    "        doc = nlp(val)\n",
    "        \n",
    "        #If the noun(s) in the answer choice can be tagged with an entity\n",
    "        if doc.ents:\n",
    "            answer_entity = doc.ents[0].label_\n",
    "            if answer_entity not in entities_in_options:\n",
    "                return False \n",
    "\n",
    "    \n",
    "    #If the stem has a number and only one option has a number\n",
    "    numbers_in_stem = extract_all_numerical_values(question.stem)\n",
    "    numbers_in_options = 0\n",
    "    options_without_numbers = 0\n",
    "    if len(numbers_in_stem) > 0:\n",
    "        #If only one option has a numerical value and no numerical value is in the stem \n",
    "        for opt in options:\n",
    "           numbers_in_opt = extract_all_numerical_values(opt)\n",
    "           if len(numbers_in_opt) > 0:\n",
    "               numbers_in_options = numbers_in_options + 1\n",
    "        if numbers_in_options == 1:\n",
    "            return False\n",
    "    \n",
    "    return True"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7005fd19-41f4-44a6-9f70-6720442b7299",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true
   },
   "source": [
    "## Lost Sequence\n",
    "If options are numberical, they should go lowest to highest or vice-versa, not a random arrnagement"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "id": "80c25d69-3a7b-476b-a6a1-12f7c0d09bb3",
   "metadata": {},
   "outputs": [],
   "source": [
    "#If answer choices are numeric, sort them, compare to current order\n",
    "#If all but one are numerical, make sure they are in order and the \"word\" option is last.\n",
    "\n",
    "def lost_sequence(question):\n",
    "    options = question.options.copy() \n",
    "    opts = []\n",
    "    non_numerical_option = 0\n",
    "    for opt in options:\n",
    "        #First check for fractions\n",
    "        fraction = extract_fraction_to_float(opt)\n",
    "        if fraction:\n",
    "            opts.append(fraction)\n",
    "        else:\n",
    "            val = extract_all_numerical_values(opt)\n",
    "            if len(val) == 1:\n",
    "                opts.append(float(val[0].replace(',', '')))\n",
    "            else:\n",
    "                non_numerical_option = non_numerical_option + 1\n",
    "    \n",
    "    if non_numerical_option > 0 and not(non_numerical_option == 1 and len(opts) == len(options)-1):\n",
    "        return True\n",
    "\n",
    "    float_options = [float(x) for x in opts]    \n",
    "    sorted_options = sorted(float_options)\n",
    "    reverse_sorted_options = sorted(float_options, reverse=True)\n",
    "    \n",
    "    if sorted_options == float_options:\n",
    "        #Numeric options are sorted\n",
    "        return True\n",
    "    elif reverse_sorted_options == float_options:\n",
    "        #Numeric options are sorted in reverse order, which might make sense for the question\n",
    "        return True\n",
    "    else:\n",
    "        return False\n",
    "\n",
    "def extract_all_numerical_values(s):\n",
    "    pattern = r'-?\\d*(?:,\\d{3})*\\.\\d+|-?\\d+(?:,\\d{3})*'\n",
    "    return re.findall(pattern, s)\n",
    "\n",
    "# Regex pattern to match fractions with optional decimal numerator and/or denominator\n",
    "def extract_fraction_to_float(s):\n",
    "    pattern = r'-?\\b\\d+(\\.\\d+)?/\\d+(\\.\\d+)?\\b'\n",
    "    match = re.search(pattern, s)\n",
    "    if match:\n",
    "        a , b = match.group().split(\"/\")\n",
    "        fraction = float(a) / float(b)\n",
    "        return fraction\n",
    "    else:\n",
    "        return None"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7cfd3dae-ff12-44cb-b30c-1df276d79aa0",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true
   },
   "source": [
    "## More Than One Correct\n",
    "There are two ways to do this, one being to present the options of the MCQ along with the question to the LLM and ask if more than one correct. This has a ton of false positives however, so we're going wiht a more basic approach of \"can the LLM correctly answer the problem\", however this does not necessarily tell us if more than one is correct, but the assumption is if the LLM gets it, then that is likely the singular correct answer. However, the LLM incorrectly answering may just mean the question is difficult and requires higher Blooms. At some point this criteria should be refined."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "id": "c77fe16e-5cc5-4522-9b08-86995409fdee",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Using GPT-4 for QA, the model can confirm if the correct answer is correct, however it is not that accurate.\n",
    "#In the future, utilize a better QA model as improvements are made.\n",
    "\n",
    "def more_than_one_correct(question):   \n",
    "    #Can adjust this prompt to be much more domain specific, but it works fine for the 15 test questions I'm using.\n",
    "\n",
    "    sysrole = \"You are an expert and an astute instructor. Given a multiple-choice quesiton and answer, you will confirm if it is a possible correct answer to the question or not. If it is a possible correct answer, respond with 'Yes' and if it is not then respond with 'No'\"\n",
    "    prompt = \"\"\"\n",
    "        question: {}\n",
    "        \n",
    "        answer: {}\n",
    "    \"\"\".format(question.stem, question.correct_option)\n",
    "\n",
    "    done = False\n",
    "    expert_reasoning = 'blank'\n",
    "    # Generate a response\n",
    "    while(done == False):\n",
    "        try:\n",
    "            o = openai.chat.completions.create(\n",
    "              model=model_engine,\n",
    "              messages=[\n",
    "                 {\"role\": \"system\",\n",
    "                  \"content\": sysrole},\n",
    "                {\"role\": \"user\", \n",
    "                 \"content\": prompt},\n",
    "              ],\n",
    "              max_tokens = 4096,\n",
    "              temperature = 0.7\n",
    "             )\n",
    "            done = True \n",
    "        except Exception as error:\n",
    "            print('errored in LLM API call: ', error)\n",
    "            time.sleep(10)\n",
    "    completion = o\n",
    "    done = False\n",
    "\n",
    "    try:\n",
    "        expert_reasoning = completion.choices[0].message.content.lower()\n",
    "    except: \n",
    "        print('error with LLM: ', completion)\n",
    "    if 'yes' in expert_reasoning.lower():\n",
    "        return True\n",
    "    else:\n",
    "        return False "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2c89088a-9ee3-4c3a-bc05-bb0fb346d212",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true
   },
   "source": [
    "## Complex or K-type\n",
    "Avoid questions that have a range of correct responses, that ask students to select from a number of possible combinations of the responses"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "id": "f53332fb-6ad0-431d-99fb-1f39e25914da",
   "metadata": {},
   "outputs": [],
   "source": [
    "#If the answer options share the same words between one another and there are commas present then it's k type\n",
    "def complex_k_type(question):    \n",
    "    options = question.options.copy()\n",
    "    for opt in question.correct_option.split('[SEP]'):\n",
    "        options.remove(opt.strip())\n",
    "\n",
    "    if not all_of_the_above(question) or not none_of_the_above(question) or not true_or_false(question):\n",
    "        return True \n",
    "        \n",
    "    if len(options) < 3:\n",
    "        return True \n",
    "    \n",
    "    #check if the options contain a comma\n",
    "    contain_a_comma = 0\n",
    "    for opt in options:\n",
    "        if ',' in opt:\n",
    "            contain_a_comma += 1\n",
    "    contain_a_comma = contain_a_comma == len(options)\n",
    "    \n",
    "    lemma_nouns_answ = get_lemma_nouns(question.correct_option)\n",
    "    lemma_nouns_options = [get_lemma_nouns(options[0]), get_lemma_nouns(options[1]), get_lemma_nouns(options[2])]\n",
    "    \n",
    "    options_that_share_noun = 0\n",
    "    for lno in lemma_nouns_options:   \n",
    "        repeating_nouns = list(set(lno).intersection(lemma_nouns_answ))\n",
    "        if (len(repeating_nouns) > 0) and (len(lno) > 0):\n",
    "            options_that_share_noun += 1\n",
    "\n",
    "    #Yes or No options are fine and might contain repeat noun, so ignore those if all options are effectively yes/no + reason \n",
    "    yes_or_no = 0\n",
    "    for opt in options:\n",
    "        opt = opt.lower()\n",
    "        if \"yes\" in opt or \"no\" in opt:\n",
    "            yes_or_no = yes_or_no + 1\n",
    "    if yes_or_no == len(options):\n",
    "        return True\n",
    "    \n",
    "    #Options share a key word, there are multiple nouns in the options, and they have a comma suggesting it might be a k-type question\n",
    "    if options_that_share_noun > 0 and contain_a_comma:\n",
    "       return False\n",
    "    \n",
    "    #After removing any list notation in the answer choices, see if they contain the same words\n",
    "    cleaned_options = []\n",
    "    for opt in options:\n",
    "        cleaned_options.append(clean_string(opt))\n",
    "\n",
    "    options_set_list = [set(i.split()) for i in cleaned_options]\n",
    "    if options_set_list[0] == options_set_list[1] and options_set_list[0] == options_set_list[2]:\n",
    "        return False\n",
    "\n",
    "    return complex_k_type_verify(question)\n",
    "\n",
    "def clean_string(string):\n",
    "    # remove whitespace\n",
    "    cleaned_string = string.strip()\n",
    "    \n",
    "    # remove punctuation\n",
    "    cleaned_string = re.sub(r'[^\\w\\s]', '', cleaned_string)\n",
    "    \n",
    "    # remove list notation\n",
    "    cleaned_string = re.sub(r'\\b(i|ii|iii|iv|v|vi|vii|viii|ix|x|xi|xii|xiii|xiv|xv|xvi|xvii|xviii|xix|xx)\\b', '', cleaned_string)\n",
    "    cleaned_string = re.sub(r'\\b(A|B|C|D|E|F)\\b', '', cleaned_string)\n",
    "    return cleaned_string\n",
    "\n",
    "\n",
    "def complex_k_type_verify(question):\n",
    "    options = question.options.copy()\n",
    "    for opt in question.correct_option.split('[SEP]'):\n",
    "        options.remove(opt.strip())\n",
    "\n",
    "    sysrole = \"\"\"Answer only with \"Yes\" or \"No\".\"\"\"\n",
    "    \n",
    "    #They also might have options that refer to other options in the list. --> This was on the very end of the statement below.\n",
    "    prompt = \"\"\"\n",
    "    Are some of these options to a multiple-choice questions partially different combinations of one another, similar to being a K-type question? They may have values separated by \"and\", \"or\", \"only\" or divided by commas or semicolons.\n",
    "    \n",
    "    {}\n",
    "    \"\"\".format(options)\n",
    "    \n",
    "    done = False\n",
    "    expert_reasoning = 'blank'\n",
    "    while(done == False):\n",
    "        try:\n",
    "            o = openai.chat.completions.create(\n",
    "              model=model_engine,\n",
    "              messages=[\n",
    "                 {\"role\": \"system\",\n",
    "                  \"content\": sysrole},\n",
    "                {\"role\": \"user\", \n",
    "                 \"content\": prompt},\n",
    "              ],\n",
    "              max_tokens = 4096,\n",
    "              temperature = 0.7\n",
    "             )\n",
    "            done = True \n",
    "        except Exception as error:\n",
    "            print('errored in GPT-4 API call: ', error)\n",
    "            time.sleep(10)\n",
    "    completion = o\n",
    "    done = False\n",
    "\n",
    "\n",
    "    # Print the response\n",
    "    try:\n",
    "        expert_reasoning = completion.choices[0].message.content.lower()\n",
    "    except: \n",
    "        print('error with LLM: ', completion)\n",
    "    if 'yes' in expert_reasoning: #Means it is used in the bad way\n",
    "        return False\n",
    "    elif 'no' in expert_reasoning:\n",
    "        return True\n",
    "        \n",
    "    return expert_reasoning"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "11a46d89-56a8-490a-a09c-ebd1cb710e2a",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true
   },
   "source": [
    "## Ambiguous or Unclear Information\n",
    "Questions and all options should be written in clear, unambiguous language"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "id": "951f058f-3edb-4619-9758-f6a26ae87a61",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the model checkpoint at Ashishkr/query_wellformedness_score were not used when initializing RobertaForSequenceClassification: ['roberta.pooler.dense.weight', 'roberta.pooler.dense.bias']\n",
      "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n"
     ]
    }
   ],
   "source": [
    "#Roberta model from: https://huggingface.co/cointegrated/roberta-large-cola-krishna2020\n",
    "#In addition to cola, added a query wellformedness score metric from https://huggingface.co/Ashishkr/query_wellformedness_score \n",
    "cola = pipeline('text-classification', model='cointegrated/roberta-large-cola-krishna2020',truncation=True)\n",
    "tokenizer = AutoTokenizer.from_pretrained(\"Ashishkr/query_wellformedness_score\", ignore_mismatched_parameters=True)\n",
    "model = AutoModelForSequenceClassification.from_pretrained(\"Ashishkr/query_wellformedness_score\")\n",
    "def check_string(s):\n",
    "    # Check for presence of '<' or '>'\n",
    "    if '<' in s or '>' in s:\n",
    "        return True\n",
    "\n",
    "    # Check for two or more ':'\n",
    "    if len(re.findall(r':', s)) >= 2:\n",
    "        return True\n",
    "\n",
    "    # Check for two or more ':'\n",
    "    if len(re.findall(r'=', s)) >= 2:\n",
    "        return True\n",
    "        \n",
    "    return False\n",
    "\n",
    "def ambiguous_unclear_information(question):\n",
    "    stem= question.stem\n",
    "    sentences = [stem]\n",
    "    \n",
    "    features = tokenizer(sentences, padding=True, truncation=True, return_tensors=\"pt\")\n",
    "    model.eval()\n",
    "    with torch.no_grad():\n",
    "        scores = model(**features).logits\n",
    "    if scores.item() <= .3:\n",
    "        if check_string(question.stem):\n",
    "            return True\n",
    "        return False\n",
    "    elif scores.item() >= 1.0:\n",
    "        return True\n",
    "    \n",
    "    output = cola(question.stem)\n",
    "    score = output[0]['score']\n",
    "    if score < .7:\n",
    "        return False\n",
    "    for opt in question.options:\n",
    "        opt_score = cola(opt)[0]['score']\n",
    "        if len(opt) > 10 and opt_score <= .7: #This parameter can be modified, depending on how strict we want to be.\n",
    "            return False\n",
    "    \n",
    "    return ambiguous_unclear_information_verify(question)\n",
    "    \n",
    "#This method has a tendency to be overly critical, so we have reduced the prompt down to this.\n",
    "def ambiguous_unclear_information_verify(question):\n",
    "    sysrole = \"\"\"Respond only with \"Yes\" or \"No\".\"\"\"\n",
    "\n",
    "    prompt = \"\"\"Is the following multiple-choice question unclear or ambiguous in a way that would make it difficult for a student that is knowledgable in the content to answer it given a set of choices?\n",
    "    The question text should be akin to a typical multiple-choice question in terms of clarity and ambiguity. \n",
    "\n",
    "    question: {}\n",
    "    answer: {}\n",
    "    options: {}\n",
    "    \"\"\".format(question.stem, question.correct_option, question.options)\n",
    "    \n",
    "    done = False\n",
    "    expert_reasoning = 'blank'\n",
    "    # Generate a response\n",
    "    while(done == False):\n",
    "        try:\n",
    "            o = openai.chat.completions.create(\n",
    "              model=model_engine,\n",
    "              messages=[\n",
    "                 {\"role\": \"system\",\n",
    "                  \"content\": sysrole},\n",
    "                {\"role\": \"user\", \n",
    "                 \"content\": prompt},\n",
    "              ],\n",
    "              max_tokens = 4096,\n",
    "              temperature = 0.7\n",
    "             )\n",
    "            done = True \n",
    "        except Exception as error:\n",
    "            print('errored in LLM API call: ', error)\n",
    "            time.sleep(10)\n",
    "    completion = o\n",
    "    done = False\n",
    "\n",
    "    # Print the response\n",
    "    try:\n",
    "        expert_reasoning = completion.choices[0].message.content.lower()\n",
    "    except: \n",
    "        print('error with LLM: ', completion)\n",
    "    if 'yes' in expert_reasoning: #Means it is confusing, flag it.\n",
    "        return False\n",
    "    elif 'no' in expert_reasoning:\n",
    "        return True\n",
    "        \n",
    "    return expert_reasoning"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f2738e90-af92-4bb8-ae2c-54cf85d51394",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true
   },
   "source": [
    "## Gratuitous Information\n",
    "Avoid unnecessary information in the stem that is not required to answer the question"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "id": "7f11f16f-8c9d-487c-9580-f8fa4460ca07",
   "metadata": {},
   "outputs": [],
   "source": [
    "def gratuitous_information_in_stem(question):  \n",
    "    #How effective are lexical richness measures for differentiations of vocabulary proficiency? A comprehensive examination with clustering analysis\n",
    "    #From: https://github.com/LSYS/LexicalRichness\n",
    "    stem = LexicalRichness(question.stem)\n",
    "    \n",
    "    if stem.cttr > 4.5:\n",
    "        return False\n",
    "    \n",
    "    return gratuitous_information_in_stem_verify(question)\n",
    "\n",
    "\n",
    "def gratuitous_information_in_stem_verify(question):\n",
    "    options = question.options.copy()\n",
    "    for opt in question.correct_option.split('[SEP]'):\n",
    "        options.remove(opt.strip())\n",
    "\n",
    "    sysrole = \"\"\"Answer only with \"Yes\" or \"No\".\"\"\"\n",
    "    prompt = \"\"\"\n",
    "    Does this question contain gratuitous information, such that a student may get confused due to the extra unnecessary details?\n",
    "    The question does not need to contain perfect wording and it does not have to be overly concise, it is fine to have a little extraneous information.\n",
    "    It should contain a typical amount of information that is commonly used in multiple-choice questions.\n",
    "\n",
    "    question: {}\n",
    "    answer: {}\n",
    "    \"\"\".format(question.stem, question.correct_option, options)\n",
    "    \n",
    "    done = False\n",
    "    expert_reasoning = 'blank'\n",
    "    while(done == False):\n",
    "        try:\n",
    "            o = openai.chat.completions.create(\n",
    "              model=model_engine,\n",
    "              messages=[\n",
    "                 {\"role\": \"system\",\n",
    "                  \"content\": sysrole},\n",
    "                {\"role\": \"user\", \n",
    "                 \"content\": prompt},\n",
    "              ],\n",
    "              max_tokens = 4096,\n",
    "              temperature = 0.7\n",
    "             )\n",
    "            done = True \n",
    "        except Exception as error:\n",
    "            print('errored in LLM API call: ', error)\n",
    "            time.sleep(10)\n",
    "    completion = o\n",
    "    done = False\n",
    "\n",
    "\n",
    "    # Print the response\n",
    "    try:\n",
    "        expert_reasoning = completion.choices[0].message.content.lower()\n",
    "    except: \n",
    "        print('error with LLM: ', completion)\n",
    "    if 'yes' in expert_reasoning: #Means it is used in the bad way\n",
    "        return False\n",
    "    elif 'no' in expert_reasoning:\n",
    "        return True\n",
    "        \n",
    "    return expert_reasoning"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d4b4ff8d-6d6a-45bd-a52c-62ede6ee349b",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true
   },
   "source": [
    "## Convergence Cues\n",
    "Avoid convergence cues in options where there are different combinations of multiple components to the answer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "id": "21a2d7e4-f1fe-4328-bc2d-e3e40626fa0a",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Check for synonyms, because they'll know it's the word they've most recently come across in the text\n",
    "#The correct option is likely to be used more (when in pairs, etc.) --> k-type (super similar by description)\n",
    "\n",
    "def avoid_convergence_cues(question):\n",
    "    #So here we check for synonyms used in the words, in case they get lazy with distractors\n",
    "    options = question.options.copy()\n",
    "    for opt in question.correct_option.split('[SEP]'):\n",
    "       options.remove(opt.strip())\n",
    "\n",
    "    #No convergence cues when there are just two options\n",
    "    if len(options) < 3:\n",
    "        return True\n",
    "\n",
    "    lemma_nouns_answ = get_lemma_nouns(question.correct_option)\n",
    "    lemma_nouns_options = [get_lemma_nouns(options[0]), get_lemma_nouns(options[1]), get_lemma_nouns(options[2])]\n",
    "    \n",
    "    #Checking for synonyms \n",
    "    synonyms = []\n",
    "    for noun in lemma_nouns_answ:\n",
    "        for syn in wn.synsets(noun):\n",
    "            for l in syn.lemmas():\n",
    "                synonyms.append(l.name().lower().replace('_', ' '))\n",
    "    \n",
    "    for opt in lemma_nouns_options:\n",
    "        repeating_nouns_synonyms = list(set(synonyms).intersection(opt))\n",
    "        if len(repeating_nouns_synonyms) > 0:       \n",
    "            \n",
    "            #if the repeat is not in every answer choice, flag it.\n",
    "            for rns in repeating_nouns_synonyms:           \n",
    "                flag = True\n",
    "                for value in lemma_nouns_options:\n",
    "                    if rns not in value:\n",
    "                        return avoid_convergence_cues_verify(question)\n",
    "    \n",
    "    return True\n",
    "\n",
    "def avoid_convergence_cues_verify(question):\n",
    "    options = question.options.copy()\n",
    "\n",
    "    sysrole = \"\"\"Answer only with \"Yes\" or \"No\".\"\"\"\n",
    "    prompt = \"\"\"\n",
    "    Are some of these options to a multiple-choice question partially different combinations of one another, potentially signaling convergence cues? They might have options that refer to other options in the list.\n",
    "    Ignore options that are similar to \"Yes\" or \"No\" followed by an explanation.\n",
    "    \n",
    "    {}\n",
    "    \"\"\".format(options)\n",
    "    \n",
    "    done = False\n",
    "    expert_reasoning = 'blank'\n",
    "    while(done == False):\n",
    "        try:\n",
    "            o = openai.chat.completions.create(\n",
    "              model=model_engine,\n",
    "              messages=[\n",
    "                 {\"role\": \"system\",\n",
    "                  \"content\": sysrole},\n",
    "                {\"role\": \"user\", \n",
    "                 \"content\": prompt},\n",
    "              ],\n",
    "              max_tokens = 4096,\n",
    "              temperature = 0.7\n",
    "             )\n",
    "            done = True \n",
    "        except Exception as error:\n",
    "            print('errored in LLM API call: ', error)\n",
    "            time.sleep(10)\n",
    "    completion = o\n",
    "    done = False\n",
    "\n",
    "    try:\n",
    "        expert_reasoning = completion.choices[0].message.content.lower()\n",
    "    except: \n",
    "        print('error with LLM: ', completion)\n",
    "    if 'yes' in expert_reasoning: # Means it is used in the bad way\n",
    "        return False\n",
    "    elif 'no' in expert_reasoning:\n",
    "        return True\n",
    "        \n",
    "    return expert_reasoning"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "71bc7cec-899d-4b12-8696-6db9e5d9a94f",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true
   },
   "source": [
    "## Grammatical Cues\n",
    "All options should be grammatically consistent with the stem and should be parallel in style and form"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "id": "8fcce92b-cf06-4601-a464-74f840592c29",
   "metadata": {},
   "outputs": [],
   "source": [
    "#If verb exists in answer choice, ensure it's the same tense as verb in other options\n",
    "#We want the stem to be the same, but as long as all the answers are the same, then it's fine, to avoid false positive.\n",
    "#https://huggingface.co/Unbabel/gec-t5_small\n",
    "def grammatical_cues_in_stem(question):\n",
    "    answer_tense = get_verb_tense(question.correct_option)\n",
    "\n",
    "    #The simplest option is to ensure the answer and other options are in the same tense, everything else was too high on false positives\n",
    "    options = question.options.copy()\n",
    "    for opt in question.correct_option.split('[SEP]'):\n",
    "        options.remove(opt.strip())\n",
    "    \n",
    "    for opt in options:\n",
    "        opt_tense = get_verb_tense(opt)    \n",
    "        if opt_tense != 'none' and answer_tense != 'none' and answer_tense is not opt_tense:\n",
    "            return False\n",
    "            \n",
    "    return True\n",
    "\n",
    "#Longer options might contain verbs of different tenses.\n",
    "#We want options that specifically have a single tense (past or present) and for it to be consistent with all other options.\n",
    "def get_verb_tense(text):\n",
    "    verbs = []\n",
    "    doc = nlp(text)\n",
    "    for token in doc:\n",
    "        if token.pos_ == 'VERB':\n",
    "            if token.tag_ in ['VBP', 'VBZ']:\n",
    "                verbs.append('present')\n",
    "            elif token.tag_ in ['VBD', 'VBN']:\n",
    "                verbs.append('past')\n",
    "            else:\n",
    "                verbs.append('none')\n",
    "\n",
    "    verb_tenses = list(set(verbs))\n",
    "    if len(verb_tenses) == 1 and verb_tenses[0] == 'past':\n",
    "        return 'past'\n",
    "    elif len(verb_tenses) == 1 and verb_tenses[0] == 'present':\n",
    "        return 'present'\n",
    "    return 'none'"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "116522a0-b405-47e1-99ce-300161bd18ad",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true
   },
   "source": [
    "## Vague Terms\n",
    "Avoid the use of vague terms (e.g. frequently, occasionally) in the options as there is seldom agreement on their actual meaning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "id": "8fd2f608-b5db-4b93-9617-2f15fbb1d531",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Like the ohter criteria that use a list of terms, these can be modified\n",
    "def vague_terms(question):\n",
    "    vagues = [\"frequently\", \"occasionally\", \"rarely\", \"seldom\", \"sometimes\", \"usually\", \"regularly\", \"periodically\", \"infrequently\", \"generally\", \"nearly\", \"more or less\", \"somewhat\", \"partly\"]\n",
    "    \n",
    "    #check the options then check the stem\n",
    "    for opt in question.options:\n",
    "        opt = opt.lower()\n",
    "        if any(word in opt for word in vagues):\n",
    "            return False\n",
    "\n",
    "    #In particular, these words can sometimes be used in the stem in a way that is not a flaw, but more likely than not, it is\n",
    "    if any(word in question.stem.lower() for word in vagues):\n",
    "        return False\n",
    "    \n",
    "    return True"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "80a44d86-528b-44c3-8d36-9220092ea637",
   "metadata": {},
   "source": [
    "## Unfocused Stem\n",
    "The stem should present a clear and focused question that can be understood and answered without looking at the options"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "id": "14287260-5c60-4038-90ca-5b10caa1afc9",
   "metadata": {},
   "outputs": [],
   "source": [
    "def unfocused_stem(question):   \n",
    "    if not true_or_false(question) or not all_of_the_above(question) or not none_of_the_above(question) or not fill_in_the_blank(question):\n",
    "        return True\n",
    "        \n",
    "    #Traits of an unfocused question (not being a question, etc.)\n",
    "    if '?' not in question.stem and \":\" not in question.stem:\n",
    "        if not question.stem.endswith(('.', ':', '?', ';')):\n",
    "            return False\n",
    "\n",
    "        if not check_if_first_word_is_a_verb(question.stem):\n",
    "            return False\n",
    "        \n",
    "        contains_question = False\n",
    "        doc = nlp(question.stem)\n",
    "        for sent in doc.sents:\n",
    "            if is_question(sent.text.strip()):\n",
    "                contains_question = True\n",
    "                break\n",
    "                \n",
    "        return contains_question\n",
    "    else:\n",
    "        return True\n",
    "\n",
    "    return unfocused_stem_verify(question)\n",
    "\n",
    "def check_if_first_word_is_a_verb(sent):\n",
    "    d = nlp(sent)\n",
    "    token = d[0] # gets the first token in a sentence\n",
    "    if token.pos_ == \"VERB\" and token.dep_ == \"ROOT\": # checks if the first token is a verb and root or not\n",
    "        return True\n",
    "    return False\n",
    "\n",
    "\n",
    "#From https://stackoverflow.com/questions/4083060/determine-if-a-sentence-is-an-inquiry\n",
    "def is_question(sent):\n",
    "    d = nlp(sent)\n",
    "    token = d[0] # gets the first token in a sentence\n",
    "    if token.pos_ == \"VERB\" and token.dep_ == \"ROOT\": # checks if the first token is a verb and root or not\n",
    "        return True\n",
    "    for token in d: # loops through the sentence and checks for WH tokens\n",
    "        if token.tag_ == \"WDT\" or token.tag_ == \"WP\" or token.tag_ == \"WP$\" or token.tag_ == \"WRB\" or token.text == '?':\n",
    "            return True\n",
    "    return  False\n",
    "\n",
    "\n",
    "#Teachers should avoid using MCQs with unfocused stems which do not ask a clear question or state a clear problem in the sentence completion format\n",
    "#The stem should present a clear and focused question that can be understood and answered without looking at the options\n",
    "def unfocused_stem_verify(question):\n",
    "    options = question.options.copy()\n",
    "    for opt in question.correct_option.split('[SEP]'):\n",
    "        options.remove(opt.strip())\n",
    "\n",
    "    sysrole = \"\"\"Answer only with \"Yes\" or \"No\".\"\"\"\n",
    "    prompt = \"\"\"\n",
    "    Does the following multiple-choice question have an unfocused stem which does not ask a clear question or state a clear poblem in the sentence completion format?\n",
    "    Ignore questions with options that are similar to \"yes\"/\"no\" or \"true\"/\"false\" followed by an explanation.\n",
    "    \n",
    "    question: {}\n",
    "    answer: {}\n",
    "    options: {}\n",
    "    \"\"\".format(question.stem, question.correct_option, options)\n",
    "    \n",
    "    done = False\n",
    "    expert_reasoning = 'blank'\n",
    "    while(done == False):\n",
    "        try:\n",
    "            o = openai.chat.completions.create(\n",
    "              model=model_engine,\n",
    "              messages=[\n",
    "                 {\"role\": \"system\",\n",
    "                  \"content\": sysrole},\n",
    "                {\"role\": \"user\", \n",
    "                 \"content\": prompt},\n",
    "              ],\n",
    "              max_tokens = 4096,\n",
    "              temperature = 0.7\n",
    "             )\n",
    "            done = True \n",
    "        except Exception as error:\n",
    "            print('errored in LLM API call: ', error)\n",
    "            time.sleep(10)\n",
    "    completion = o\n",
    "    done = False\n",
    "\n",
    "    try:\n",
    "        expert_reasoning = completion.choices[0].message.content\n",
    "    except: \n",
    "        print('error with LLM: ', completion)\n",
    "    if expert_reasoning == 'Yes': #Means it is used in the bad way\n",
    "        return False\n",
    "    elif expert_reasoning == 'No':\n",
    "        return True\n",
    "        \n",
    "    return expert_reasoning"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "64b015b6-d6bc-4e43-8f04-d82e0c62fc22",
   "metadata": {},
   "source": [
    "# Other Metrics (Perplexity, Diversity, Grammatical Error, Cognitive Complexity)\n",
    "\n",
    "In addition to IWF, calculate these other commonly used metrics to see how they evaluate, Answerability is a fifth metric that I am currently leaving out. Just like BLEU, METEOR, ROGUE, etc. these metrics often do not correlate with human judgements and are not indicators of flawed/bad educational multiple-choice questions like the IWF criteria are. You can still compute them because it's easy enough, but you should put little faith in them.\r\n",
    "\r\n",
    "Note, these metrics aren't great indicators, you can read about it here: https://arxiv.org/pdf/2405.20529"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0c3f0dc3-0682-4b48-94ed-55653f9fa417",
   "metadata": {},
   "source": [
    "## Perplexity \n",
    "This assesses a language model's ability to predict question and answer text based on its training data. Lower scores suggest more coherent questions and answers with predictable language patterns, whereas higher scores indicate complexity or atypical text, suggesting the questions could be unclear or poorly structured. <br/>\n",
    "<b>NOTE</b>: This will be very slow for a large amount of questions, anything greater than 30 questions will take quite some time."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "id": "2a728cf1-0785-4fce-adc9-b22ce5dc7be1",
   "metadata": {},
   "outputs": [],
   "source": [
    "def perplexity(questions):\n",
    "    nl = ', '\n",
    "    predictions = []\n",
    "    for index, row in questions.iterrows():\n",
    "        stem=row['text'].strip()\n",
    "        non_empty_values = [row[col].strip() for col in ['a','b','c','d'] if row[col].strip()]\n",
    "        row_string = nl.join(non_empty_values)\n",
    "        predictions.append(stem + ' ' + row_string)\n",
    "    \n",
    "    perplexity = load(\"perplexity\", module_type=\"metric\")\n",
    "    results = perplexity.compute(predictions=predictions, add_start_token=False, model_id='gpt2-large')\n",
    "    return results['perplexities']"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "905d96ac-e318-4fdf-b716-a00f7ed5ae98",
   "metadata": {},
   "source": [
    "## Diversity\n",
    "Using Distinct-3, this evaluates the range in vocabulary, structure, and content across generated texts, ensuring a variety of questions and answers and reducing repetition. A higher diversity score indicates greater uniqueness among MCQs, avoiding repetitive phrases and templated patterns. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "id": "31dc2604-dc17-4200-8507-f11cdd68af34",
   "metadata": {},
   "outputs": [],
   "source": [
    "def diversity(questions):\n",
    "    predictions = []\n",
    "    per_question = []\n",
    "    for index, row in questions.iterrows():\n",
    "        stem=row['text'].strip()\n",
    "        non_empty_values = [row[col].strip() for col in ['a','b','c','d'] if row[col].strip()]\n",
    "        row_string = ', '.join(non_empty_values)\n",
    "\n",
    "        predictions.append(stem + ' ' + row_string)\n",
    "    \n",
    "    distinct_3_total = 0\n",
    "    for o in predictions:\n",
    "        dist3 = calculate_distinct_3(o)\n",
    "        distinct_3_total = distinct_3_total + dist3\n",
    "        per_question.append(dist3)\n",
    "\n",
    "    print('distinct_3_total: ', (distinct_3_total)/len(predictions))   \n",
    "    print('ngram_diversity_total: ', ngram_diversity(predictions))\n",
    "    print('length: ', len(predictions))\n",
    "\n",
    "    return per_question\n",
    "\n",
    "def calculate_distinct_3(text):\n",
    "    # Tokenize the text\n",
    "    tokens = word_tokenize(text)\n",
    "\n",
    "    # Generate trigrams\n",
    "    trigrams_list = list(trigrams(tokens))\n",
    "\n",
    "    # Count unique trigrams\n",
    "    unique_trigrams = set(trigrams_list)\n",
    "    num_unique_trigrams = len(unique_trigrams)\n",
    "\n",
    "    # Count total trigrams\n",
    "    total_trigrams = len(trigrams_list)\n",
    "\n",
    "    # Calculate Distinct-3\n",
    "    if total_trigrams > 0:\n",
    "        distinct_3 = num_unique_trigrams / total_trigrams\n",
    "    else:\n",
    "        distinct_3 = 0\n",
    "\n",
    "    return distinct_3\n",
    "\n",
    "def ngram_diversity(options, n=3):\n",
    "    all_ngrams = [ngram for option in options for ngram in ngrams(word_tokenize(option), n)]\n",
    "    unique_ngrams = set(all_ngrams)\n",
    "    return len(unique_ngrams) / len(all_ngrams)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "015d5690-30cd-40b0-b22b-0822aecae055",
   "metadata": {},
   "source": [
    "## Grammatical Error\n",
    "This uses a Python wrapper for https://languagetool.org/ currently we are using the free API endpoint, so if it's used excessively we might get IP blocked. Grammatical errors pinpoint grammar violations, such as incorrect verb tense or spelling, quantified for each MCQ."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "id": "6fcf1983-b487-49c0-9fc7-4ca4436f4341",
   "metadata": {},
   "outputs": [],
   "source": [
    "def check_grammar(text):\n",
    "    tool = language_tool_python.LanguageToolPublicAPI('en-US')\n",
    "    matches = tool.check(text)\n",
    "    return len(matches), matches\n",
    "\n",
    "predictions = []\n",
    "ques = {}\n",
    "def grammatical_error(questions):\n",
    "    total_errors = 0\n",
    "    errorsList = []\n",
    "    for index, row in questions.iterrows():\n",
    "        stem=row['text'].strip()\n",
    "        num_errors, errors = check_grammar(stem)\n",
    "        total_errors = total_errors + num_errors\n",
    "        errorsList.append(num_errors)\n",
    "\n",
    "    print('total_errors: ', (total_errors/len(questions)))   \n",
    "    print('length: ', len(questions))\n",
    "    print('length of errors: ', len(errorsList))\n",
    "\n",
    "    return errorsList"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "057a3db5-0fbb-4a24-9089-d2a66f0cab27",
   "metadata": {},
   "source": [
    "## Cognitive Complexity\n",
    "This is measured by Bloom's Taxonomy, although some research has done it by the \"difficulty\" of the question, which a LLM can assess, but Bloom's is a better fit. Additionally, this might be redundant since the Bloom's label is included in the question's construction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "id": "f9d0582d-0993-450f-bad4-25a629b74921",
   "metadata": {},
   "outputs": [],
   "source": [
    "def cognitive_complexity(questions):\n",
    "    bloom_labels = []\n",
    "    predictions = []\n",
    "    for index, row in questions.iterrows():\n",
    "        stem=row['text'].strip()\n",
    "        non_empty_values = [row[col].strip() for col in ['a','b','c','d'] if row[col].strip()]\n",
    "        row_string = '\\n'.join(non_empty_values)\n",
    "        predictions.append(stem + '\\n' + row_string)\n",
    "\n",
    "    sysrole = \"You are an expert in pedagogy and an astute instructor here to classify a multiple-choice questions provided to you with one of the six level's of Bloom's Taxonomy\"\n",
    "    prompt = \"\"\"Given the multiple-choice question below, please respond with that level of Bloom's Revised Taxonomy it falls into and nothing else.\n",
    "        {}\n",
    "        \"\"\"\n",
    "    for q in predictions:\n",
    "        p = prompt.format(q)\n",
    "        done = False\n",
    "    \n",
    "        while(done == False):\n",
    "            try:\n",
    "                o = openai.chat.completions.create(\n",
    "                  model=model_engine,\n",
    "                  messages=[\n",
    "                     {\"role\": \"system\",\n",
    "                      \"content\": sysrole},\n",
    "                    {\"role\": \"user\", \n",
    "                     \"content\": p},\n",
    "                  ],\n",
    "                  max_tokens = 4096,\n",
    "                  temperature = 0.7\n",
    "                 )\n",
    "                done = True \n",
    "            except Exception as error:\n",
    "                print('errored in LLM API call: ', error)\n",
    "                time.sleep(10)\n",
    "        completion = o\n",
    "        done = False\n",
    "    \n",
    "        try:\n",
    "            expert_reasoning = completion.choices[0].message.content\n",
    "            bloom_labels.append(expert_reasoning)\n",
    "        except: \n",
    "            print('error with LLM: ', completion)\n",
    "    return bloom_labels"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "55054a9c-ebe5-40e3-be58-d643f7af1cf6",
   "metadata": {},
   "source": [
    "# Formatting Your CSV of MCQs"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f526d878-c06d-4016-aadc-98705e86c768",
   "metadata": {},
   "source": [
    "| id | text | answer | a | b | c | d |\r\n",
    "|----------|----------|----------|----------|----------|----------|----------|\r\n",
    "| Data 1   | Data 2   | Data 3   | Data 4   | Data 5   | Data 6   | Data 7   |\r\n",
    "| Data 8   | Data 9   | Data 10  | Data 11  | Data 12  | Data 13  | Data 14 |\r\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ece7db81-8d41-4145-aad5-b73257b8c6f9",
   "metadata": {},
   "source": [
    "### id: A unique number\n",
    "### text: The question's stem\n",
    "### answer: The text of the correct response, this should match the text in one of the a/b/c/d columns\n",
    "### a-e: The text for the corresponding option"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "28b715b5-4e7c-4e4d-be17-f2873c902202",
   "metadata": {},
   "source": [
    "# 19 Item-Writing Flaws Criteria - Running the code"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "id": "0a2a936f-8d71-4803-b1ab-120d22009b7d",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-----  ambiguous_unclear_information  -----  tufts\\mcqs.csv\n",
      "ambiguous_unclear_information  matches:  1\n",
      "-----  implausible_distractors  -----  tufts\\mcqs.csv\n",
      "-- GPT-4 says they are similar:  Determining the return to play timeline  -and-  Initiating the rehabilitation process\n",
      "-- GPT-4 says they are similar:  The StARRT framework  -and-  Evidence-based Practice Model\n",
      "-- GPT-4 says they are similar:  Younger age, male gender, and positive psychological outlook  -and-  Significant financial resources\n",
      "-- GPT-4 says they are similar:  The injury’s nature, demands of the sport, and affected body area  -and-  A consensus on predetermined criteria\n",
      "Distractor not similar enough: A blend of physical health and psychological readiness \t\t Endorsement from the entire team \t\t Score: 0.0320\n",
      "implausible_distractors  matches:  1\n",
      "-----  none_of_the_above  -----  tufts\\mcqs.csv\n",
      "none_of_the_above  matches:  0\n",
      "-----  longest_answer_correct  -----  tufts\\mcqs.csv\n",
      "longest_answer_correct  matches:  7\n",
      "-----  gratuitous_information_in_stem  -----  tufts\\mcqs.csv\n",
      "gratuitous_information_in_stem  matches:  0\n",
      "-----  true_or_false  -----  tufts\\mcqs.csv\n",
      "true_or_false  matches:  0\n",
      "-----  avoid_convergence_cues  -----  tufts\\mcqs.csv\n",
      "avoid_convergence_cues  matches:  1\n",
      "-----  avoid_logical_cues  -----  tufts\\mcqs.csv\n",
      "avoid_logical_cues  matches:  1\n",
      "-----  all_of_the_above  -----  tufts\\mcqs.csv\n",
      "all_of_the_above  matches:  1\n",
      "-----  fill_in_the_blank  -----  tufts\\mcqs.csv\n",
      "fill_in_the_blank  matches:  0\n",
      "-----  absolute_terms  -----  tufts\\mcqs.csv\n",
      "absolute_terms  matches:  1\n",
      "-----  word_repeats_in_stem_and_correct_answer  -----  tufts\\mcqs.csv\n",
      "word_repeats_in_stem_and_correct_answer  matches:  2\n",
      "-----  unfocused_stem  -----  tufts\\mcqs.csv\n",
      "unfocused_stem  matches:  2\n",
      "-----  complex_k_type  -----  tufts\\mcqs.csv\n",
      "complex_k_type  matches:  1\n",
      "-----  grammatical_cues_in_stem  -----  tufts\\mcqs.csv\n",
      "grammatical_cues_in_stem  matches:  0\n",
      "-----  lost_sequence  -----  tufts\\mcqs.csv\n",
      "lost_sequence  matches:  0\n",
      "-----  vague_terms  -----  tufts\\mcqs.csv\n",
      "vague_terms  matches:  0\n",
      "-----  more_than_one_correct  -----  tufts\\mcqs.csv\n",
      "more_than_one_correct  matches:  2\n",
      "-----  negative_worded_stem  -----  tufts\\mcqs.csv\n",
      "negative_worded_stem  matches:  0\n"
     ]
    }
   ],
   "source": [
    "all_criteria = ['ambiguous_unclear_information', \n",
    "    'implausible_distractors', \n",
    "    'none_of_the_above',\n",
    "    'longest_answer_correct',\n",
    "    'gratuitous_information_in_stem',\n",
    "    'true_or_false', \n",
    "    'avoid_convergence_cues',\n",
    "    'avoid_logical_cues', \n",
    "    'all_of_the_above', \n",
    "    'fill_in_the_blank', \n",
    "    'absolute_terms', \n",
    "    'word_repeats_in_stem_and_correct_answer', \n",
    "    'unfocused_stem', \n",
    "    'complex_k_type', \n",
    "    'grammatical_cues_in_stem',\n",
    "    'lost_sequence', \n",
    "    'vague_terms', \n",
    "    'more_than_one_correct', \n",
    "    'negative_worded_stem'] \n",
    "\n",
    "files = ['TODO.csv'] #Update to the file path containing your CSV\n",
    "all_data = {}\n",
    "for criteria in all_criteria:\n",
    "    match_total = 0\n",
    "    for file in files:\n",
    "        print('----- ', criteria, ' ----- ', file)\n",
    "        data = pd.read_csv(file)\n",
    "        data = data.fillna('')\n",
    "        combined_data = pd.concat([data])\n",
    "        questions = []\n",
    "        for index, row in combined_data.iterrows():\n",
    "            question = MultipleChoiceQuestion(\n",
    "                stem=row['text'],\n",
    "                options = [row[col].strip() for col in ['a', 'b', 'c', 'd', 'e'] if col in row and pd.notna(row[col]) and row[col].strip()],\n",
    "                correct_option= row['answer'].strip(),\n",
    "                qid = row['id'],\n",
    "                quality = 0\n",
    "            )\n",
    "            questions.append(question)\n",
    "        \n",
    "        auto_iwf_results = []\n",
    "        matches = []\n",
    "        for q in questions:\n",
    "            ids = globals()[criteria](q)\n",
    "            if ids:\n",
    "                ids = 0\n",
    "            else:\n",
    "                matches.append(q.stem)\n",
    "                ids = 1\n",
    "            auto_iwf_results.append(ids)\n",
    "            \n",
    "\n",
    "        all_data[criteria] = auto_iwf_results\n",
    "        match_total = match_total + len(matches)\n",
    "    print(criteria, ' matches: ', match_total) \n",
    "    \n",
    "df = pd.DataFrame(all_data)\n",
    "df.to_csv('RESULTS.csv', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "088786ef-9462-4f99-a90d-b80867fb7b62",
   "metadata": {},
   "source": [
    "# Other Metrics (Perplexity, Diversity, Grammatical Error, Cognitive Complexity) - Running the code"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "id": "ab3fc9b7-6b81-4557-9f0e-e3d66860d476",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-----  perplexity  -----  tufts\\mcqs.csv\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using pad_token, but it is not set yet.\n",
      "100%|████████████████████████████████████████████████████████████████████████████████████| 1/1 [00:10<00:00, 10.46s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-----  diversity  -----  tufts\\mcqs.csv\n",
      "distinct_3_total:  0.9972020446596718\n",
      "ngram_diversity_total:  0.9743589743589743\n",
      "length:  14\n",
      "-----  grammatical_error  -----  tufts\\mcqs.csv\n",
      "total_errors:  0.0\n",
      "length:  14\n",
      "length of errors:  14\n",
      "-----  cognitive_complexity  -----  tufts\\mcqs.csv\n"
     ]
    }
   ],
   "source": [
    "## Calc the other metrics\n",
    "other_metrics = ['perplexity',\n",
    "                 'diversity',\n",
    "                 'grammatical_error',\n",
    "                 'cognitive_complexity']\n",
    "\n",
    "files = ['TODO_OTHER.csv']\n",
    "metric_data = {}\n",
    "for metric in other_metrics:\n",
    "    for file in files:\n",
    "        print('----- ', metric, ' ----- ', file)\n",
    "        data = pd.read_csv(file)\n",
    "        data = data.fillna('')\n",
    "        combined_data = pd.concat([data]) #This is used for multiple files\n",
    "        \n",
    "        result = globals()[metric](combined_data)\n",
    "        metric_data[metric] = result\n",
    "\n",
    "df = pd.DataFrame(metric_data)\n",
    "df.to_csv('RESULTS_OTHER.csv', index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}