main.part1.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "# Solving Cryptic Crosswords with LLMs: Part 1\n",
        "date created: 04.08.2023"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "## Installing modules"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ycDZN7LR0G5y",
        "outputId": "fdef1139-082c-44a3-c2a1-caf55d5484f2"
      },
      "outputs": [],
      "source": [
        "! pip install git+https://github.com/huggingface/transformers"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "qx2-PuIszmZF",
        "outputId": "69a92c45-0d4a-4be2-97b1-c13b8bc1accc"
      },
      "outputs": [],
      "source": [
        "! pip install torch datasets evaluate accelerate sentencepiece"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "68lMM09QwnMG"
      },
      "outputs": [],
      "source": [
        "# import modules\n",
        "import pandas as pd\n",
        "import numpy as np\n",
        "import ast\n",
        "import json\n",
        "import plotly.graph_objects as go\n",
        "from plotly.subplots import make_subplots\n",
        "\n",
        "# import parameters\n",
        "from utils.parameters import *"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "## Data Extraction and Transformation"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 337
        },
        "id": "4Y5uKdhYyqEM",
        "outputId": "e0166f5c-2eaf-4387-8df2-9d505dc73fb3"
      },
      "outputs": [],
      "source": [
        "# Import data\n",
        "clues_raw = pd.read_csv(clues_path_raw).dropna().sample(frac=1)\n",
        "clues = clues_raw.copy()[['rowid', 'clue', 'answer', 'definition']]\n",
        "\n",
        "# Transform columns into the format required by the trainer module\n",
        "clues['rowid'] = clues['rowid'].astype(str)\n",
        "clues['question'] = clues['clue']\n",
        "clues['context'] = clues['definition'] \n",
        "clues['answers'] = clues['answer'].map(lambda x : {\"text\" : [x], \"answer_start\" : [0]})\n",
        "clues['answers'].apply(lambda x : ast.literal_eval(str(x)))\n",
        "clues = clues.rename(columns={'rowid' : 'id'})\n",
        "clues = clues[['id', 'question', 'context', 'answers']].dropna()\n",
        "\n",
        "# Print examples\n",
        "clues.head()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "FYXC8Ccvysda"
      },
      "outputs": [],
      "source": [
        "# Split data into train, validation, test\n",
        "\n",
        "train_val = clues.sample(frac=0.9,random_state=200)\n",
        "test = clues.drop(train_val.index)\n",
        "train = train_val.sample(frac=0.9,random_state=200)\n",
        "validation = train_val.drop(train.index)\n",
        "\n",
        "# Save data\n",
        "clues.to_csv(clues_path_processed, index=False)\n",
        "train.to_csv(clues_path_train, index=False)\n",
        "validation.to_csv(clues_path_validation, index=False)\n",
        "test.to_csv(clues_path_test, index=False)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "## Fine Tuning"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "N.B. the modelling parameters are in the parameter.py file."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "TtuUqMLby-7t",
        "outputId": "105b6243-8889-4328-f774-b7a0dfad3dac"
      },
      "outputs": [],
      "source": [
        "! python ./utils/run_seq2seq_qa.py \\\n",
        "  --model_name_or_path {model_name_t5} \\\n",
        "  --train_file {clues_path_train} \\\n",
        "  --validation_file {clues_path_validation} \\\n",
        "  --test_file {clues_path_test} \\\n",
        "  --question_column question \\\n",
        "  --context_column context \\\n",
        "  --answer_column answers \\\n",
        "  --do_train \\\n",
        "  --do_eval \\\n",
        "  --do_pred \\\n",
        "  --predict_with_generate \\\n",
        "  --version_2_with_negative \\\n",
        "  --per_device_train_batch_size {batch_size} \\\n",
        "  --learning_rate {lr} \\\n",
        "  --num_train_epochs {num_epochs} \\\n",
        "  --max_seq_length {max_seq_length} \\\n",
        "  --overwrite_output_dir {overwrite_dir} \\\n",
        "  --output_dir {output_dir}"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "nlf5qvnalyKI"
      },
      "source": [
        "## Predictions"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "Q5MPcACMzWMc",
        "outputId": "1fbf4461-1f8d-4451-8489-a474ca75294b"
      },
      "outputs": [],
      "source": [
        "# Read prediction output file\n",
        "predictions = pd.read_json(predictions_path)\n",
        "predictions = predictions.rename(columns={'id' : 'rowid'})\n",
        "\n",
        "# Show some examples\n",
        "predictions.head()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 374
        },
        "id": "hl5mQOMv9JMM",
        "outputId": "fe953526-8c45-44c5-aec8-4a7c75d1efc6"
      },
      "outputs": [],
      "source": [
        "# Join and compare with clues dataset to see correct / incorrect answers\n",
        "\n",
        "compare = clues_raw.merge(predictions, on='rowid')[['clue', 'definition', 'answer', 'prediction_text']]\n",
        "compare['correct_len'] = np.where(compare['prediction_text'].str.len() == compare['answer'].str.len(), 1, 0)\n",
        "compare['correct'] = np.where(compare['prediction_text'] == compare['answer'], 1, 0)\n",
        "compare['correct_len_1'] = np.where(abs(compare['prediction_text'].str.len() - compare['answer'].str.len()) <=1, 1, 0)\n",
        "\n",
        "# Get stats of correct vs incorrect cols\n",
        "compare.groupby(['correct', 'correct_len', 'correct_len_1']).count()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab": {
          "background_save": true
        },
        "id": "fU3f48SNqMHg",
        "outputId": "fa03fb03-a42d-4b02-d121-08dee8855ecf"
      },
      "source": [
        "## Plot loss vs epochs"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "with open(f'{output_dir}/trainer_state.json', 'rb') as f:\n",
        "    tr = json.load(f)\n",
        "\n",
        "epoch_list = [0]\n",
        "loss_list = [None]\n",
        "learning_rate_list = [lr]\n",
        "\n",
        "# Collect the list of each metric\n",
        "for x in tr['log_history'][:-1]:\n",
        "    epoch_list.append(x['epoch'])\n",
        "    loss_list.append(x['loss'])\n",
        "    learning_rate_list.append(x['learning_rate'])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "df = pd.DataFrame(dict(\n",
        "    epoch = epoch_list,\n",
        "    loss = loss_list,\n",
        "    learning_rate = learning_rate_list\n",
        "))\n",
        "\n",
        "\n",
        "# Create figure with secondary y-axis\n",
        "fig = make_subplots(specs=[[{\"secondary_y\": True}]])\n",
        "\n",
        "# Create traces\n",
        "fig.add_trace(go.Scatter(x=epoch_list, y=loss_list,\n",
        "                    mode='lines',\n",
        "                    name='loss'))\n",
        "fig.add_trace(go.Scatter(x=epoch_list, y=learning_rate_list, \n",
        "                    mode='lines+markers',\n",
        "                    name='learning_rate'), secondary_y=True)\n",
        "\n",
        "# Set x-axis title\n",
        "fig.update_xaxes(title_text=\"Epoch\")\n",
        "\n",
        "# Set y-axes titles\n",
        "fig.update_yaxes(title_text=\"Loss\", secondary_y=False)\n",
        "fig.update_yaxes(title_text=\"Learning Rate\", secondary_y=True)\n"
      ]
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.11.4"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}