LMSYS_v2.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "machine_shape": "hm",
      "gpuType": "L4",
      "authorship_tag": "ABX9TyN/hQ7j5yd5DI7xcn6wpGLt",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "b056953c59394960899edbb054436ce2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_67c1f521134e40a09199b218b74e1551",
              "IPY_MODEL_f4ec9d8ca380473083cf406935e8ee76",
              "IPY_MODEL_ccf1ddef571941be9208b4fdb911a4de"
            ],
            "layout": "IPY_MODEL_c43947730f644432ba178d5c71f8ade9"
          }
        },
        "67c1f521134e40a09199b218b74e1551": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_77b0e24c731d40c9866795a0ff4f1559",
            "placeholder": "​",
            "style": "IPY_MODEL_c205f33398e74e258e8a5ca57458f5c8",
            "value": "Loading checkpoint shards: 100%"
          }
        },
        "f4ec9d8ca380473083cf406935e8ee76": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_63c8fab7a07343b2973b3c06f61ac66d",
            "max": 4,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_95800940fd2e43d39c1c5182c2267f16",
            "value": 4
          }
        },
        "ccf1ddef571941be9208b4fdb911a4de": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_12010fe0cd8749fe8a9d055f0a65a571",
            "placeholder": "​",
            "style": "IPY_MODEL_58dec8a4d4144595b5d467cf72263a1b",
            "value": " 4/4 [05:50&lt;00:00, 72.04s/it]"
          }
        },
        "c43947730f644432ba178d5c71f8ade9": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "77b0e24c731d40c9866795a0ff4f1559": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c205f33398e74e258e8a5ca57458f5c8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "63c8fab7a07343b2973b3c06f61ac66d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "95800940fd2e43d39c1c5182c2267f16": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "12010fe0cd8749fe8a9d055f0a65a571": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "58dec8a4d4144595b5d467cf72263a1b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/daichisaito-cs/LMSYS/blob/main/LMSYS_v2.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "#  Setup"
      ],
      "metadata": {
        "id": "ylbbUkOmfnXW"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# @title Config\n",
        "class Config:\n",
        "    name = \"baseline-001\"\n",
        "    only_inference = False\n",
        "\n",
        "    model_name = \"roberta-base\"\n",
        "    learning_rate = 1e-5\n",
        "    max_length = 256\n",
        "    epochs = 8\n",
        "    batch_size = 16\n",
        "\n",
        "    n_fold = 5\n",
        "    trn_fold = [0, 1, 2, 3, 4]\n",
        "    seed = 2022\n",
        "    target_col = \"target\"\n",
        "    debug = False\n",
        "\n",
        "    # Colab Env\n",
        "    upload_from_colab = True\n",
        "    api_path = \"/content/drive/Shareddrives/inSane/Workspace/kaggle/kaggle.json\"\n",
        "    drive_path = \"/content/drive/Shareddrives/inSane/Workspace/kaggle/LMSYS_Challenge\"\n",
        "\n",
        "    # Kaggle Env\n",
        "    kaggle_dataset_path = None"
      ],
      "metadata": {
        "id": "887l1Bf7yiv9"
      },
      "execution_count": 1,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# @title Libraries\n",
        "import os\n",
        "import json\n",
        "import warnings\n",
        "import shutil\n",
        "import logging\n",
        "import joblib\n",
        "import random\n",
        "import datetime\n",
        "import sys\n",
        "\n",
        "import pandas as pd\n",
        "import numpy as np\n",
        "import matplotlib.pyplot as plt\n",
        "import seaborn as sns\n",
        "\n",
        "from tqdm.auto import tqdm\n",
        "from sklearn.model_selection import KFold\n",
        "from sklearn.metrics import mean_squared_error\n",
        "\n",
        "import tensorflow as tf\n",
        "from tensorflow.keras import backend as K"
      ],
      "metadata": {
        "cellView": "form",
        "id": "auNwMxoYzKVv"
      },
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# @title Utils\n",
        "\n",
        "class Logger:\n",
        "    \"\"\"参考) https://github.com/ghmagazine/kagglebook/blob/master/ch04-model-interface/code/util.py\"\"\"\n",
        "    def __init__(self, path):\n",
        "        self.general_logger = logging.getLogger(path)\n",
        "        stream_handler = logging.StreamHandler()\n",
        "        file_general_handler = logging.FileHandler(os.path.join(path, 'Experiment.log'))\n",
        "        if len(self.general_logger.handlers) == 0:\n",
        "            self.general_logger.addHandler(stream_handler)\n",
        "            self.general_logger.addHandler(file_general_handler)\n",
        "            self.general_logger.setLevel(logging.INFO)\n",
        "\n",
        "    def info(self, message):\n",
        "        # display time\n",
        "        self.general_logger.info('[{}] - {}'.format(self.now_string(), message))\n",
        "\n",
        "    @staticmethod\n",
        "    def now_string():\n",
        "        return str(datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'))\n",
        "\n",
        "def seed_everything(seed=42):\n",
        "# 参考https://qiita.com/kaggle_grandmaster-arai-san/items/d59b2fb7142ec7e270a5\n",
        "    random.seed(seed)\n",
        "    os.environ['PYTHONHASHSEED'] = str(seed)\n",
        "    np.random.seed(seed)\n",
        "    tf.random.set_seed(seed)"
      ],
      "metadata": {
        "cellView": "form",
        "id": "DqRLVOeryg2g"
      },
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# @title Colab settings\n",
        "COLAB = \"google.colab\" in sys.modules\n",
        "\n",
        "if COLAB:\n",
        "    print(\"This environment is Google Colab\")\n",
        "\n",
        "    # mount\n",
        "    from google.colab import drive\n",
        "    if not os.path.isdir(\"/content/drive\"):\n",
        "        drive.mount('/content/drive')\n",
        "\n",
        "    # import library\n",
        "    !pip install --quiet transformers\n",
        "    !pip install --quiet iterative-stratification\n",
        "    !pip install --quiet tensorflow-addons\n",
        "\n",
        "    !pip install -q -U bitsandbytes\n",
        "    !pip install -q -U transformers\n",
        "    # !pip install transformers==4.40.0\n",
        "    !pip install -q -U tokenizers\n",
        "    !pip install -q -U peft\n",
        "\n",
        "    # use kaggle api (need kaggle token)\n",
        "    f = open(Config.api_path, 'r')\n",
        "    json_data = json.load(f)\n",
        "    os.environ[\"KAGGLE_USERNAME\"] = json_data[\"username\"]\n",
        "    os.environ[\"KAGGLE_KEY\"] = json_data[\"key\"]\n",
        "\n",
        "    # set dirs\n",
        "    DRIVE = Config.drive_path\n",
        "    EXP = (Config.name if Config.name is not None\n",
        "           else get(\"http://172.28.0.2:9000/api/sessions\").json()[0][\"name\"][:-6])\n",
        "    INPUT = os.path.join(DRIVE, \"Input\")\n",
        "    OUTPUT = os.path.join(DRIVE, \"Output\")\n",
        "    SUBMISSION = os.path.join(DRIVE, \"Submission\")\n",
        "    OUTPUT_EXP = os.path.join(OUTPUT, EXP)\n",
        "    EXP_MODEL = os.path.join(OUTPUT_EXP, \"model\")\n",
        "    EXP_FIG = os.path.join(OUTPUT_EXP, \"fig\")\n",
        "    EXP_PREDS = os.path.join(OUTPUT_EXP, \"preds\")\n",
        "\n",
        "    # make dirs\n",
        "    for d in [INPUT, SUBMISSION, EXP_MODEL, EXP_FIG, EXP_PREDS]:\n",
        "        os.makedirs(d, exist_ok=True)\n",
        "\n",
        "    if not os.path.isfile(os.path.join(INPUT, \"train.csv.zip\")):\n",
        "        # load dataset\n",
        "        ! kaggle competitions download -c lmsys-chatbot-arena -p $INPUT\n",
        "\n",
        "    # utils\n",
        "    logger = Logger(OUTPUT_EXP)\n",
        "\n",
        "else:\n",
        "    print(\"This environment is Kaggle Kernel\")\n",
        "\n",
        "    # set dirs\n",
        "    # INPUT = \"../input/lmsys-chatbot-arena\"\n",
        "    INPUT = \"../input\"\n",
        "    EXP, OUTPUT, SUBMISSION = \"./\", \"./\", \"./\"\n",
        "    EXP_MODEL = os.path.join(EXP, \"model\")\n",
        "    EXP_FIG = os.path.join(EXP, \"fig\")\n",
        "    EXP_PREDS = os.path.join(EXP, \"preds\")\n",
        "\n",
        "    # libraries\n",
        "    !pip install -q -U bitsandbytes --no-index --find-links ../input/llm-detect-pip/\n",
        "    !pip install -q -U transformers --no-index --find-links ../input/llm-detect-pip/\n",
        "    !pip install -q -U tokenizers --no-index --find-links ../input/llm-detect-pip/\n",
        "    !pip install -q -U peft --no-index --find-links ../input/llm-detect-pip/\n",
        "\n",
        "    # copy dirs\n",
        "    if Config.kaggle_dataset_path is not None:\n",
        "        KD_MODEL = os.path.join(Config.kaggle_dataset_path, \"model\")\n",
        "        KD_EXP_PREDS = os.path.join(Config.kaggle_dataset_path, \"preds\")\n",
        "        shutil.copytree(KD_MODEL, EXP_MODEL)\n",
        "        shutil.copytree(KD_EXP_PREDS, EXP_PREDS)\n",
        "\n",
        "    # make dirs\n",
        "    for d in [EXP_MODEL, EXP_FIG, EXP_PREDS]:\n",
        "        os.makedirs(d, exist_ok=True)\n",
        "\n",
        "    # utils\n",
        "    logger = Logger(EXP)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "rjVcUkl3zRYm",
        "outputId": "e80a9f9e-f464-42f9-dd40-8464b90bed63"
      },
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "This environment is Google Colab\n",
            "Mounted at /content/drive\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m611.8/611.8 kB\u001b[0m \u001b[31m14.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m119.8/119.8 MB\u001b[0m \u001b[31m14.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m21.3/21.3 MB\u001b[0m \u001b[31m72.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m9.3/9.3 MB\u001b[0m \u001b[31m67.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m251.6/251.6 kB\u001b[0m \u001b[31m7.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m309.4/309.4 kB\u001b[0m \u001b[31m34.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hlmsys-chatbot-arena.zip: Skipping, found more recently modified local copy (use --force to force download)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# @title Other settings\n",
        "# utils\n",
        "warnings.filterwarnings(\"ignore\")\n",
        "sns.set(style='whitegrid')\n",
        "seed_everything(seed=Config.seed)\n",
        "\n",
        "# 2nd import\n",
        "from transformers import AutoTokenizer, TFAutoModel, WarmUp\n",
        "import tensorflow_addons as tfa"
      ],
      "metadata": {
        "cellView": "form",
        "id": "ISkXHuKRNp0L"
      },
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# libraries"
      ],
      "metadata": {
        "id": "JwpGbZq7nCXZ"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from threading import Thread\n",
        "import gc\n",
        "import os\n",
        "import io\n",
        "import json\n",
        "import random\n",
        "import pickle\n",
        "import zipfile\n",
        "import datetime\n",
        "import time\n",
        "\n",
        "import torch\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "from transformers import AutoTokenizer, LlamaModel, LlamaForSequenceClassification, BitsAndBytesConfig\n",
        "from peft import get_peft_config, PeftModel, PeftConfig, get_peft_model, LoraConfig, TaskType\n",
        "from torch.cuda.amp import autocast\n",
        "from IPython.display import display\n",
        "import torch.nn.functional as F\n",
        "import tokenizers\n",
        "!pip list | grep torch"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "FhJFNLsNpXiB",
        "outputId": "7add776a-0f7d-4803-9311-170b7b643ca8"
      },
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "torch                            2.3.0+cu121\n",
            "torchaudio                       2.3.0+cu121\n",
            "torchsummary                     1.5.1\n",
            "torchtext                        0.18.0\n",
            "torchvision                      0.18.0+cu121\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "torch.backends.cuda.enable_mem_efficient_sdp(False)\n",
        "torch.backends.cuda.enable_flash_sdp(False)\n",
        "\n",
        "MODEL_NAME = os.path.join(INPUT, 'llama-3-8b-chat-hf') if COLAB else os.path.join(INPUT, 'llama-3/transformers/8b-chat-hf/1')\n",
        "WEIGHTS_PATH = os.path.join(INPUT, 'lmsys-model/model')\n",
        "\n",
        "# if COLAB:\n",
        "#     MODEL_NAME = '/content/drive/Shareddrives/inSane/Workspace/kaggle/LMSYS_Challenge/Input/llama-3-8b-chat-hf'\n",
        "#     WEIGHTS_PATH = '/content/drive/Shareddrives/inSane/Workspace/kaggle/LMSYS_Challenge/lmsys-model/model'\n",
        "# else:\n",
        "#     MODEL_NAME = '/kaggle/input/llama-3/transformers/8b-chat-hf/1'\n",
        "#     WEIGHTS_PATH = '/kaggle/input/lmsys-model/model'\n",
        "MAX_LENGTH = 1284\n",
        "BATCH_SIZE = 8\n",
        "DEVICE = torch.device(\"cuda\")"
      ],
      "metadata": {
        "id": "gXzG230jpIYr"
      },
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Prepare data"
      ],
      "metadata": {
        "id": "PFQCCbWcplkl"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "train = pd.read_csv(os.path.join(INPUT, \"lmsys-chatbot-arena\", \"train.csv\"))\n",
        "test = pd.read_csv(os.path.join(INPUT, \"lmsys-chatbot-arena\", \"test.csv\"))\n",
        "sample_sub = pd.read_csv(os.path.join(INPUT, \"lmsys-chatbot-arena\", \"sample_submission.csv\"))"
      ],
      "metadata": {
        "id": "ZvKv_SVtnFHm"
      },
      "execution_count": 12,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# concatenate strings in list\n",
        "def process(input_str):\n",
        "    stripped_str = input_str.strip('[]')\n",
        "    sentences = [s.strip('\"') for s in stripped_str.split('\",\"')]\n",
        "    return  ' '.join(sentences)\n",
        "\n",
        "test.loc[:, 'prompt'] = test['prompt'].apply(process)\n",
        "test.loc[:, 'response_a'] = test['response_a'].apply(process)\n",
        "test.loc[:, 'response_b'] = test['response_b'].apply(process)\n",
        "\n",
        "display(sample_sub)\n",
        "display(test.head(5))\n",
        "\n",
        "# Prepare text for model\n",
        "test['text'] = 'User prompt: ' + test['prompt'] +  '\\n\\nModel A :\\n' + test['response_a'] +'\\n\\n--------\\n\\nModel B:\\n'  + test['response_b']\n",
        "print(test['text'][0])"
      ],
      "metadata": {
        "id": "pMyMgG04qclo",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 537
        },
        "outputId": "00a5c7af-0b0e-45ce-b48e-2c2704be25f0"
      },
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "        id  winner_model_a  winner_model_b  winner_tie\n",
              "0   136060        0.333333        0.333333    0.333333\n",
              "1   211333        0.333333        0.333333    0.333333\n",
              "2  1233961        0.333333        0.333333    0.333333"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-6393f33e-a00b-4e92-a64d-4ff5044e8014\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>id</th>\n",
              "      <th>winner_model_a</th>\n",
              "      <th>winner_model_b</th>\n",
              "      <th>winner_tie</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>136060</td>\n",
              "      <td>0.333333</td>\n",
              "      <td>0.333333</td>\n",
              "      <td>0.333333</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>211333</td>\n",
              "      <td>0.333333</td>\n",
              "      <td>0.333333</td>\n",
              "      <td>0.333333</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>1233961</td>\n",
              "      <td>0.333333</td>\n",
              "      <td>0.333333</td>\n",
              "      <td>0.333333</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-6393f33e-a00b-4e92-a64d-4ff5044e8014')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-6393f33e-a00b-4e92-a64d-4ff5044e8014 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-6393f33e-a00b-4e92-a64d-4ff5044e8014');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-9d80cee1-203b-4971-8d2d-07fb4e117f77\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-9d80cee1-203b-4971-8d2d-07fb4e117f77')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-9d80cee1-203b-4971-8d2d-07fb4e117f77 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "\n",
              "  <div id=\"id_3bdfe2ea-b57b-4096-b9c2-984ff3d83b5f\">\n",
              "    <style>\n",
              "      .colab-df-generate {\n",
              "        background-color: #E8F0FE;\n",
              "        border: none;\n",
              "        border-radius: 50%;\n",
              "        cursor: pointer;\n",
              "        display: none;\n",
              "        fill: #1967D2;\n",
              "        height: 32px;\n",
              "        padding: 0 0 0 0;\n",
              "        width: 32px;\n",
              "      }\n",
              "\n",
              "      .colab-df-generate:hover {\n",
              "        background-color: #E2EBFA;\n",
              "        box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "        fill: #174EA6;\n",
              "      }\n",
              "\n",
              "      [theme=dark] .colab-df-generate {\n",
              "        background-color: #3B4455;\n",
              "        fill: #D2E3FC;\n",
              "      }\n",
              "\n",
              "      [theme=dark] .colab-df-generate:hover {\n",
              "        background-color: #434B5C;\n",
              "        box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "        filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "        fill: #FFFFFF;\n",
              "      }\n",
              "    </style>\n",
              "    <button class=\"colab-df-generate\" onclick=\"generateWithVariable('sample_sub')\"\n",
              "            title=\"Generate code using this dataframe.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M7,19H8.4L18.45,9,17,7.55,7,17.6ZM5,21V16.75L18.45,3.32a2,2,0,0,1,2.83,0l1.4,1.43a1.91,1.91,0,0,1,.58,1.4,1.91,1.91,0,0,1-.58,1.4L9.25,21ZM18.45,9,17,7.55Zm-12,3A5.31,5.31,0,0,0,4.9,8.1,5.31,5.31,0,0,0,1,6.5,5.31,5.31,0,0,0,4.9,4.9,5.31,5.31,0,0,0,6.5,1,5.31,5.31,0,0,0,8.1,4.9,5.31,5.31,0,0,0,12,6.5,5.46,5.46,0,0,0,6.5,12Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "    <script>\n",
              "      (() => {\n",
              "      const buttonEl =\n",
              "        document.querySelector('#id_3bdfe2ea-b57b-4096-b9c2-984ff3d83b5f button.colab-df-generate');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      buttonEl.onclick = () => {\n",
              "        google.colab.notebook.generateWithVariable('sample_sub');\n",
              "      }\n",
              "      })();\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "dataframe",
              "variable_name": "sample_sub",
              "summary": "{\n  \"name\": \"sample_sub\",\n  \"rows\": 3,\n  \"fields\": [\n    {\n      \"column\": \"id\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 613299,\n        \"min\": 136060,\n        \"max\": 1233961,\n        \"num_unique_values\": 3,\n        \"samples\": [\n          136060,\n          211333,\n          1233961\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"winner_model_a\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.0,\n        \"min\": 0.3333333333333333,\n        \"max\": 0.3333333333333333,\n        \"num_unique_values\": 1,\n        \"samples\": [\n          0.3333333333333333\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"winner_model_b\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.0,\n        \"min\": 0.3333333333333333,\n        \"max\": 0.3333333333333333,\n        \"num_unique_values\": 1,\n        \"samples\": [\n          0.3333333333333333\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"winner_tie\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.0,\n        \"min\": 0.3333333333333333,\n        \"max\": 0.3333333333333333,\n        \"num_unique_values\": 1,\n        \"samples\": [\n          0.3333333333333333\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "        id                                             prompt  \\\n",
              "0   136060  I have three oranges today, I ate an orange ye...   \n",
              "1   211333  You are a mediator in a heated political debat...   \n",
              "2  1233961  How to initialize the classification head when...   \n",
              "\n",
              "                                          response_a  \\\n",
              "0                        You have two oranges today.   \n",
              "1  Thank you for sharing the details of the situa...   \n",
              "2  When you want to initialize the classification...   \n",
              "\n",
              "                                          response_b  \n",
              "0  You still have three oranges. Eating an orange...  \n",
              "1  Mr Reddy and Ms Blue both have valid points in...  \n",
              "2  To initialize the classification head when per...  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-b6ff6025-df02-418d-b8fb-3209ce9d55cb\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>id</th>\n",
              "      <th>prompt</th>\n",
              "      <th>response_a</th>\n",
              "      <th>response_b</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>136060</td>\n",
              "      <td>I have three oranges today, I ate an orange ye...</td>\n",
              "      <td>You have two oranges today.</td>\n",
              "      <td>You still have three oranges. Eating an orange...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>211333</td>\n",
              "      <td>You are a mediator in a heated political debat...</td>\n",
              "      <td>Thank you for sharing the details of the situa...</td>\n",
              "      <td>Mr Reddy and Ms Blue both have valid points in...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>1233961</td>\n",
              "      <td>How to initialize the classification head when...</td>\n",
              "      <td>When you want to initialize the classification...</td>\n",
              "      <td>To initialize the classification head when per...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-b6ff6025-df02-418d-b8fb-3209ce9d55cb')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-b6ff6025-df02-418d-b8fb-3209ce9d55cb button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-b6ff6025-df02-418d-b8fb-3209ce9d55cb');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-ba1fd9d6-c1eb-4c1a-85b1-51cfdc16e698\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-ba1fd9d6-c1eb-4c1a-85b1-51cfdc16e698')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-ba1fd9d6-c1eb-4c1a-85b1-51cfdc16e698 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "dataframe",
              "summary": "{\n  \"name\": \"print(test['text'][0])\",\n  \"rows\": 3,\n  \"fields\": [\n    {\n      \"column\": \"id\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 613299,\n        \"min\": 136060,\n        \"max\": 1233961,\n        \"num_unique_values\": 3,\n        \"samples\": [\n          136060,\n          211333,\n          1233961\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"prompt\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 3,\n        \"samples\": [\n          \"I have three oranges today, I ate an orange yesterday. How many oranges do I have?\",\n          \"You are a mediator in a heated political debate between two opposing parties. Mr Reddy is very hung up on semantic definitions of sex and gender, and believes that women are adult human females. Meanwhile Ms Blue is extremely fluid with definitions and does not care about truth. He (Ms blue uses he\\\\/him pronouns) insists that anybody can be any gender, gametes don't mean anything, and that men can get pregnant. You, Mr Goddy are tasked with helping them both find a middle ground.\",\n          \"How to initialize the classification head when I do transfer learning. For example, I have a pre-trained vision transformer on ImageNet, and now I want to finetune it on StanfordCars I want to do full finetuning\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"response_a\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 3,\n        \"samples\": [\n          \"You have two oranges today.\",\n          \"Thank you for sharing the details of the situation. As a mediator, I understand the importance of finding a middle ground that both parties can agree upon.\\\\n\\\\nIt is important to recognize that language and definitions can be complex and can vary depending on cultural and societal norms. At the same time, it is also important to acknowledge the biological differences between males and females.\\\\n\\\\nOne possible approach to finding a middle ground could be to focus on respecting individuals' self-identification and autonomy while also acknowledging the biological differences between males and females. This could involve using inclusive language that reflects individuals' gender identities while also recognizing the importance of biological sex in certain contexts, such as medical treatment and reproductive health.\\\\n\\\\nIt may also be helpful to explore ways to create a more inclusive society where individuals of all gender identities are valued and respected. This could involve promoting education and awareness about gender diversity and creating policies and practices that support individuals' gender identities and expression.\\\\n\\\\nUltimately, the goal of mediation is to find a solution that meets the needs and concerns of both parties. By focusing on mutual respect, open communication, and a willingness to find common ground, it is possible to reach a resolution that allows both parties to move forward in a positive and productive way.\",\n          \"When you want to initialize the classification head for transfer learning, you can follow these steps:\\\\n\\\\n  1. Load the pre-trained model: First, load the pre-trained vision transformer model on ImageNet. You can use popular deep learning libraries like PyTorch or TensorFlow to load the model.\\\\n  2. Freeze the pre-trained layers: Next, you need to freeze the pre-trained layers of the model so that they don't get updated during the training process. This is because the pre-trained layers have learned useful features that can be reused for the new task.\\\\n  3. Add a new classification head: Now, you need to add a new classification head to the model. The classification head is responsible for outputting the predicted class for a given input image. The size of the classification head depends on the number of classes in the new task. For example, if you are working on the StanfordCars dataset, which has 196 classes, you would need a classification head with 196 output neurons.\\\\n  4. Initialize the classification head: Initialize the weights of the classification head randomly or using pre-trained weights. This is important because the initial weights can have a significant impact on the performance of the model.\\\\n  5. Train the model: Finally, you can train the model on the new task using your labeled data. During training, only the weights of the classification head will be updated, while the weights of the frozen pre-trained layers will remain unchanged. This allows the model to learn task-specific features while leveraging the pre-trained features.\\\\n\\\\nNote that there are different ways to initialize the classification head, such as using Xavier initialization, Kaiming initialization, or He initialization. You can choose the initialization method that works best for your specific task and model architecture. If you want to do full finetuning of the pre-trained vision transformer on the StanfordCars dataset, you can follow these steps:\\\\n\\\\n  1. Load the pre-trained model: Load the pre-trained vision transformer model on ImageNet. You can use popular deep learning libraries like PyTorch or TensorFlow to load the model.\\\\n  2. Remove the last classification layer: The pre-trained model typically has a last classification layer that is specific to the ImageNet dataset. To adapt the model to the StanfordCars dataset, you need to remove this layer.\\\\n  3. Add a new classification head: Add a new classification head to the model. The classification head is responsible for outputting the predicted class for a given input image. The size of the classification head depends on the number of classes in the StanfordCars dataset, which has 196 classes. You can add a fully connected layer with 196 output neurons as the new classification head.\\\\n  4. Initialize the classification head: Initialize the weights of the classification head randomly or using pre-trained weights. This is important because the initial weights can have a significant impact on the performance of the model.\\\\n  5. Freeze the pre-trained layers: Freeze the pre-trained layers of the model so that they don't get updated during the training process. This is because the pre-trained layers have learned useful features that can be reused for the new task.\\\\n  6. Train the model: Finally, you can train the model on the StanfordCars dataset using your labeled data. During training, all the weights of the model will be updated, including the weights of the frozen pre-trained layers. This allows the model to learn task-specific features while leveraging the pre-trained features.\\\\n\\\\nNote that when you do full finetuning, you need to use a lower learning rate and a smaller number of training epochs compared to transfer learning, because the model has already learned a lot of useful features from the pre-trained weights. You can experiment with different learning rates and training schedules to find the best hyperparameters for your specific task and model architecture.\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"response_b\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 3,\n        \"samples\": [\n          \"You still have three oranges. Eating an orange yesterday does not affect the number of oranges you have today.\",\n          \"Mr Reddy and Ms Blue both have valid points in their arguments. On one hand, Mr Reddy is correct that women are adult human females, as defined by their biological and physical characteristics. However, Ms Blue is right that gender is a more fluid concept and that people can identify with any gender they choose. A compromise could be that gender is defined by an individual's self-identification, while sex is defined by their biological characteristics.\",\n          \"To initialize the classification head when performing transfer learning, follow these steps:\\\\n\\\\n1. Load the pre-trained vision transformer model that was trained on ImageNet.\\\\n2. Replace the existing classification head with a new one suitable for the StanfordCars dataset.\\\\n3. Initialize the weights of the new classification head with random values or pre-trained weights if available.\\\\n4. Freeze the weights of the vision transformer backbone to prevent them from being updated during training.\\\\n5. Train the model on the StanfordCars dataset, fine-tuning the classification head while keeping the backbone frozen.\\\\n\\\\nHere's an example implementation in Python using PyTorch:\\\\n\\\\n```python\\\\nimport torch\\\\nimport torch.nn as nn\\\\nimport torchvision.models as models\\\\n\\\\n# Step 1: Load pre-trained vision transformer model\\\\nvision_transformer = models.vit_base_patch16_224(pretrained=True)\\\\n\\\\n# Step 2: Replace classification head\\\\nnum_classes = 196  # Number of classes in StanfordCars dataset\\\\nvision_transformer.head = nn.Linear(vision_transformer.head.in_features, num_classes)\\\\n\\\\n# Step 3: Initialize weights of new classification head\\\\nnn.init.xavier_uniform_(vision_transformer.head.weight)\\\\nnn.init.constant_(vision_transformer.head.bias, 0)\\\\n\\\\n# Step 4: Freeze the weights of the backbone\\\\nfor param in vision_transformer.parameters():\\\\n    param.requires_grad = False\\\\n\\\\n# Step 5: Train the model on StanfordCars dataset\\\\n# ...\\\\n```\\\\n\\\\nIn this example, we load the pre-trained vision transformer model using `torchvision.models.vit_base_patch16_224`. We then replace the `head` (classification head) of the model with a new linear layer (`nn.Linear`) that has the appropriate number of output classes for the StanfordCars dataset.\\\\n\\\\nAfter replacing the classification head, we initialize its weights using Xavier initialization (`nn.init.xavier_uniform_`) and set its biases to zero (`nn.init.constant_`).\\\\n\\\\nFinally, we freeze the weights of the vision transformer backbone by setting `requires_grad=False` for all parameters. This ensures that only the weights of the classification head will be updated during training on the StanfordCars dataset. If you want to perform full finetuning, where both the classification head and the backbone of the vision transformer are updated during training on the StanfordCars dataset, you can omit freezing the backbone weights. Here's an updated version of the code:\\\\n\\\\n```python\\\\nimport torch\\\\nimport torch.nn as nn\\\\nimport torchvision.models as models\\\\n\\\\n# Step 1: Load pre-trained vision transformer model\\\\nvision_transformer = models.vit_base_patch16_224(pretrained=True)\\\\n\\\\n# Step 2: Replace classification head\\\\nnum_classes = 196  # Number of classes in StanfordCars dataset\\\\nvision_transformer.head = nn.Linear(vision_transformer.head.in_features, num_classes)\\\\n\\\\n# Step 3: Initialize weights of new classification head\\\\nnn.init.xavier_uniform_(vision_transformer.head.weight)\\\\nnn.init.constant_(vision_transformer.head.bias, 0)\\\\n\\\\n# Step 4: Train the model on StanfordCars dataset\\\\n# ...\\\\n```\\\\n\\\\nIn this case, you can directly train the vision transformer model on the StanfordCars dataset without freezing any weights. During training, both the weights of the classification head and the backbone will be updated.\\\\n\\\\nKeep in mind that full finetuning may require more computational resources and a larger labeled dataset compared to freezing the backbone. It can help the model learn more specific features related to the StanfordCars dataset but may also increase the risk of overfitting if the dataset is small. Monitoring the training progress and adjusting hyperparameters, such as learning rate and regularization, can help mitigate overfitting.\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "User prompt: I have three oranges today, I ate an orange yesterday. How many oranges do I have?\n",
            "\n",
            "Model A :\n",
            "You have two oranges today.\n",
            "\n",
            "--------\n",
            "\n",
            "Model B:\n",
            "You still have three oranges. Eating an orange yesterday does not affect the number of oranges you have today.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Tokenize"
      ],
      "metadata": {
        "id": "kHLAFkkWqgun"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "tokenizer = AutoTokenizer.from_pretrained(os.path.join(INPUT, 'lmsys-model/tokenizer'))\n",
        "\n",
        "tokens = tokenizer(test['text'].tolist(), padding='max_length',\n",
        "                   max_length=MAX_LENGTH, truncation=True, return_tensors='pt')\n",
        "\n",
        "INPUT_IDS = tokens['input_ids'].to(DEVICE, dtype=torch.int32)\n",
        "ATTENTION_MASKS = tokens['attention_mask'].to(DEVICE, dtype=torch.int32)\n",
        "\n",
        "# Move tensors to CPU and convert them to lists\n",
        "input_ids_cpu = [tensor.cpu().tolist() for tensor in INPUT_IDS]\n",
        "attention_masks_cpu = [tensor.cpu().tolist() for tensor in ATTENTION_MASKS]\n",
        "\n",
        "data = pd.DataFrame()\n",
        "data['INPUT_IDS'] = input_ids_cpu\n",
        "data['ATTENTION_MASKS'] = attention_masks_cpu\n",
        "data[:2]"
      ],
      "metadata": {
        "id": "gWJq9rshqjEF",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 150
        },
        "outputId": "9a88f063-89da-49b8-a3c0-4f905aced5b8"
      },
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                           INPUT_IDS  \\\n",
              "0  [1502, 10137, 25, 358, 617, 2380, 85138, 3432,...   \n",
              "1  [1502, 10137, 25, 1472, 527, 264, 69030, 304, ...   \n",
              "\n",
              "                                     ATTENTION_MASKS  \n",
              "0  [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...  \n",
              "1  [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-d2ebc562-e132-40e3-af88-277747e1ce07\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>INPUT_IDS</th>\n",
              "      <th>ATTENTION_MASKS</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>[1502, 10137, 25, 358, 617, 2380, 85138, 3432,...</td>\n",
              "      <td>[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>[1502, 10137, 25, 1472, 527, 264, 69030, 304, ...</td>\n",
              "      <td>[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-d2ebc562-e132-40e3-af88-277747e1ce07')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-d2ebc562-e132-40e3-af88-277747e1ce07 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-d2ebc562-e132-40e3-af88-277747e1ce07');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-fbccf94c-ba56-41b2-a4db-72e426a4c828\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-fbccf94c-ba56-41b2-a4db-72e426a4c828')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-fbccf94c-ba56-41b2-a4db-72e426a4c828 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "dataframe",
              "summary": "{\n  \"name\": \"data[:2]\",\n  \"rows\": 2,\n  \"fields\": [\n    {\n      \"column\": \"INPUT_IDS\",\n      \"properties\": {\n        \"dtype\": \"object\",\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"ATTENTION_MASKS\",\n      \"properties\": {\n        \"dtype\": \"object\",\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"
            }
          },
          "metadata": {},
          "execution_count": 15
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Load llama model"
      ],
      "metadata": {
        "id": "lWVx4TX7qpY0"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# BitsAndBytes configuration\n",
        "bnb_config =  BitsAndBytesConfig(\n",
        "    load_in_8bit=True,\n",
        "    bnb_8bit_compute_dtype=torch.float16,\n",
        "    bnb_8bit_use_double_quant=False)\n",
        "\n",
        "# Load base model on GPU 0\n",
        "device0 = torch.device('cuda:0')\n",
        "\n",
        "base_model_0 = LlamaForSequenceClassification.from_pretrained(\n",
        "    MODEL_NAME,\n",
        "    num_labels=3,\n",
        "    torch_dtype=torch.float16,\n",
        "    quantization_config=bnb_config,\n",
        "    device_map='cuda:0')\n",
        "base_model_0.config.pad_token_id = tokenizer.pad_token_id"
      ],
      "metadata": {
        "id": "k5Kg0jhyqs1b",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 123,
          "referenced_widgets": [
            "b056953c59394960899edbb054436ce2",
            "67c1f521134e40a09199b218b74e1551",
            "f4ec9d8ca380473083cf406935e8ee76",
            "ccf1ddef571941be9208b4fdb911a4de",
            "c43947730f644432ba178d5c71f8ade9",
            "77b0e24c731d40c9866795a0ff4f1559",
            "c205f33398e74e258e8a5ca57458f5c8",
            "63c8fab7a07343b2973b3c06f61ac66d",
            "95800940fd2e43d39c1c5182c2267f16",
            "12010fe0cd8749fe8a9d055f0a65a571",
            "58dec8a4d4144595b5d467cf72263a1b"
          ]
        },
        "outputId": "48e1a93b-3e48-4bad-e10f-150e396068fc"
      },
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Unused kwargs: ['bnb_8bit_compute_dtype', 'bnb_8bit_use_double_quant']. These kwargs are not used in <class 'transformers.utils.quantization_config.BitsAndBytesConfig'>.\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Loading checkpoint shards:   0%|          | 0/4 [00:00<?, ?it/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "b056953c59394960899edbb054436ce2"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Some weights of LlamaForSequenceClassification were not initialized from the model checkpoint at /content/drive/Shareddrives/inSane/Workspace/kaggle/LMSYS_Challenge/Input/llama-3-8b-chat-hf and are newly initialized: ['score.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Load weights"
      ],
      "metadata": {
        "id": "YkvBqAi4qv8U"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# LoRa configuration\n",
        "peft_config = LoraConfig(\n",
        "    r=16,\n",
        "    lora_alpha=32,\n",
        "    lora_dropout=0.10,\n",
        "    bias='none',\n",
        "    inference_mode=True,\n",
        "    task_type=TaskType.SEQ_CLS,\n",
        "    target_modules=['o_proj', 'v_proj'])"
      ],
      "metadata": {
        "id": "L7hzTxF2qxSU"
      },
      "execution_count": 17,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Get peft\n",
        "model_0 = get_peft_model(base_model_0, peft_config).to(device0)\n",
        "#Load weights\n",
        "model_0.load_state_dict(torch.load(WEIGHTS_PATH), strict=False)\n",
        "model_0.eval()\n",
        "\n",
        "#Trainable Parameters\n",
        "model_0.print_trainable_parameters()"
      ],
      "metadata": {
        "id": "ZvJTUZaaqxur",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "5228f56c-1fe7-4d60-c3e4-8949e381148a"
      },
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "trainable params: 12,288 || all params: 7,511,764,992 || trainable%: 0.0002\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "gc.collect()"
      ],
      "metadata": {
        "id": "3QsGebQ6qzxD",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "fa20d6b1-7570-4177-e22b-3a185d4f6017"
      },
      "execution_count": 19,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "60"
            ]
          },
          "metadata": {},
          "execution_count": 19
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Inference"
      ],
      "metadata": {
        "id": "QHRJ43Yoq1Dh"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def inference(df, model, device, batch_size=BATCH_SIZE):\n",
        "    input_ids = torch.tensor(df['INPUT_IDS'].values.tolist(), dtype=torch.long)\n",
        "    attention_mask = torch.tensor(df['ATTENTION_MASKS'].values.tolist(), dtype=torch.long)\n",
        "\n",
        "    generated_class_a = []\n",
        "    generated_class_b = []\n",
        "    generated_class_c = []\n",
        "\n",
        "    model.eval()\n",
        "\n",
        "    for start_idx in range(0, len(df), batch_size):\n",
        "        end_idx = min(start_idx + batch_size, len(df))\n",
        "        batch_input_ids = input_ids[start_idx:end_idx].to(device)\n",
        "        batch_attention_mask = attention_mask[start_idx:end_idx].to(device)\n",
        "\n",
        "        with torch.no_grad():\n",
        "            with autocast():\n",
        "                outputs = model(\n",
        "                    input_ids=batch_input_ids,\n",
        "                    attention_mask=batch_attention_mask\n",
        "                )\n",
        "\n",
        "        probabilities = torch.softmax(outputs.logits, dim=-1).cpu().numpy()\n",
        "\n",
        "        generated_class_a.extend(probabilities[:, 0])\n",
        "        generated_class_b.extend(probabilities[:, 1])\n",
        "        generated_class_c.extend(probabilities[:, 2])\n",
        "\n",
        "    df['winner_model_a'] = generated_class_a\n",
        "    df['winner_model_b'] = generated_class_b\n",
        "    df['winner_tie'] = generated_class_c\n",
        "\n",
        "    torch.cuda.empty_cache()\n",
        "\n",
        "    return df"
      ],
      "metadata": {
        "id": "I_-spthYq1_f"
      },
      "execution_count": 20,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "st = time.time()\n",
        "\n",
        "# 単一のGPUデバイスを使用\n",
        "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
        "\n",
        "# モデルをGPUに移動\n",
        "model_0 = model_0.to(device)\n",
        "\n",
        "# 全データを一度に処理\n",
        "results = inference(data, model_0, device)\n",
        "\n",
        "print(f\"Processing complete. Total time: {time.time() - st}\")\n",
        "\n",
        "TARGETS = ['winner_model_a', 'winner_model_b', 'winner_tie']\n",
        "\n",
        "sample_sub[TARGETS] = results[TARGETS]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "BBipv0B2uwGM",
        "outputId": "e7f72097-0d7a-4256-91c0-eb036f52664d"
      },
      "execution_count": 25,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "We detected that you are passing `past_key_values` as a tuple and this is deprecated and will be removed in v4.43. Please use an appropriate `Cache` class (https://huggingface.co/docs/transformers/v4.41.3/en/internal/generation_utils#transformers.Cache)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Processing complete. Total time: 99.08912372589111\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "llama_preds = data[TARGETS].values"
      ],
      "metadata": {
        "id": "n-kTpf0tq7J2"
      },
      "execution_count": 26,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# LGBM + tfidf"
      ],
      "metadata": {
        "id": "XVnCJO50q7n5"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# TAG = 'lmsys-chatbot-arena'\n",
        "# RUNPOD = os.path.exists('/workspace/')\n",
        "# KAGGLE = not RUNPOD\n",
        "# if KAGGLE:\n",
        "#     print('kaggle')"
      ],
      "metadata": {
        "id": "9b_URbi1q9OL"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "try:\n",
        "    import pandas as pd\n",
        "except:\n",
        "    !pip install -q kaggle\n",
        "    !pip install -q pandas matplotlib scipy joblib scikit-learn lightgbm\n",
        "    !pip install -q protobuf\n",
        "    !pip install -q numba"
      ],
      "metadata": {
        "id": "lExCQAhxq-PN"
      },
      "execution_count": 27,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# DATA = '/data/' if RUNPOD else 'data/' \\\n",
        "#         if not os.path.exists('/kaggle/') \\\n",
        "#             else '/kaggle/input/{}/'.format(TAG)\n",
        "\n",
        "# if RUNPOD:\n",
        "#     if not os.path.exists('~/.kaggle/kaggle.json'):\n",
        "#         !mkdir -p ~/.kaggle\n",
        "#         !cp /workspace/kaggle.json ~/.kaggle/kaggle.json\n",
        "#         !chmod 600 /root/.kaggle/kaggle.json\n",
        "\n",
        "#     if not os.path.exists('/workspace/' + TAG + '.zip'):\n",
        "#         !kaggle competitions download $TAG -p /workspace/\n",
        "\n",
        "#     if not os.path.exists('/data/'):\n",
        "#         import zipfile\n",
        "#         zipfile.ZipFile('/workspace/' + TAG + '.zip').extractall('/data/')"
      ],
      "metadata": {
        "id": "C7fOBS2Pq_Lk"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "params = {}\n",
        "if False:\n",
        "    pass;\n",
        "    params['subsample'] = 30\n",
        "else:\n",
        "    params['fold'] = -1\n",
        "\n",
        "\n",
        "params['n_epochs'] = 1\n",
        "params['n_lgb'] = 1\n",
        "params['model'] = 'microsoft/deberta-v3-small'"
      ],
      "metadata": {
        "id": "2AS3Id8VrVTT"
      },
      "execution_count": 28,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# params = {}\n",
        "FULL = params.get('fold', 0) < 0\n",
        "N_FOLDS = int(params.get('n_folds', 3));\n",
        "FOLD = int(params.get('fold', 0))\n",
        "SEED = int(params.get('seed', 3))\n",
        "SS = int(params.get('subsample', 1))\n",
        "\n",
        "print(N_FOLDS, FOLD, SEED, SS)"
      ],
      "metadata": {
        "id": "4DXgYgf-rYGZ",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "0b0c5d9b-bd0e-412d-d3f4-2f99473626cb"
      },
      "execution_count": 29,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "3 -1 3 1\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from sklearn.model_selection import StratifiedKFold\n",
        "\n",
        "def get_folds(train):\n",
        "    return list(StratifiedKFold(N_FOLDS, random_state = SEED, shuffle = True)\\\n",
        "                    .split(X = np.zeros(len(train)), y = train.iloc[:, -3:].idxmax(1)))\n",
        "\n",
        "train_ids, test_ids = get_folds(train)[FOLD] if not FULL else [list(range(len(train))), []]\n",
        "if SS > 1:\n",
        "    train_ids, test_ids = train_ids[::SS], test_ids[::SS]\n",
        "\n",
        "print(len(train_ids), len(test_ids));  assert set(train_ids) & set(test_ids) == set()"
      ],
      "metadata": {
        "id": "EAUbzs-brYha",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "703b72f4-1fc4-44ce-bee9-f0bafd4704fd"
      },
      "execution_count": 30,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "57477 0\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "TRAIN = False\n",
        "INFER = True\n",
        "SAVE = False"
      ],
      "metadata": {
        "id": "-5GvVybIrYmM"
      },
      "execution_count": 31,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import lightgbm as lgb\n",
        "from sklearn.feature_extraction.text import CountVectorizer"
      ],
      "metadata": {
        "id": "uqN09WRxrdZn"
      },
      "execution_count": 32,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "LGB = True\n",
        "TRAIN_LGB = TRAIN and LGB and params.get('n_lgb', 1) > 0\n",
        "INFER_LGB = not TRAIN and LGB"
      ],
      "metadata": {
        "id": "yIbo5-OYreid"
      },
      "execution_count": 33,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "cvec  = pickle.load(open(os.path.join(INPUT, 'lsys-models-4/') + 'cvec.pkl', 'rb'))\n",
        "ccvec = pickle.load(open(os.path.join(INPUT, 'lsys-models-4/') + 'ccvec.pkl', 'rb'))"
      ],
      "metadata": {
        "id": "qg-KuQ7prfq7"
      },
      "execution_count": 42,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def symlog(x):\n",
        "    return (np.sign(x) * np.log1p(np.abs(x))).astype(np.float32)\n",
        "\n",
        "def dense(x):\n",
        "    x = np.asarray(x.astype(np.float32).todense())\n",
        "    x = symlog(x)\n",
        "    return x\n",
        "\n",
        "def get_features(df):\n",
        "    pfeat = np.hstack([dense(v.transform(df[c]))\n",
        "                for v in [cvec, ccvec]\n",
        "                    for c in ['prompt', ]])\n",
        "    afeat = np.hstack([dense(v.transform(df[c]))\n",
        "                for c in ['response_a', ]\n",
        "                    for v in [cvec, ccvec]\n",
        "                ])\n",
        "    bfeat = np.hstack([dense(v.transform(df[c]))\n",
        "                for c in ['response_b', ]\n",
        "                    for v in [cvec, ccvec]\n",
        "                ])\n",
        "\n",
        "    v = np.hstack([\n",
        "          afeat - bfeat, np.abs(afeat - bfeat),\n",
        "        ])\n",
        "    try:\n",
        "        v = v / (len(all_vote_models) if len(df) < len(train) else 1)\n",
        "    except:\n",
        "        pass\n",
        "\n",
        "    extras = []\n",
        "    EXTRAS = ['\\n', '\\n\\n', '.', ' ', '\",\"']\n",
        "    for e in EXTRAS:\n",
        "        for c in ['prompt', 'response_a', 'response_b']:\n",
        "            extras.append(df[c].str.count(e).values)\n",
        "\n",
        "    extras.append(df[c].str.len())\n",
        "    extras.append(df[c].str.split().apply(lambda x: len(x)))\n",
        "\n",
        "    extras = np.stack(extras, axis = 1)\n",
        "    extras = np.hstack([extras ** 0.5, np.log1p(extras)])\n",
        "    return np.hstack([v, extras])"
      ],
      "metadata": {
        "id": "-9ObD0Ynrgug"
      },
      "execution_count": 43,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "lgb_models = pickle.load(open(os.path.join(INPUT, 'lsys-models-4/') + 'lgb_models.pkl', 'rb'))"
      ],
      "metadata": {
        "id": "8M4N_DIjrh-c"
      },
      "execution_count": 44,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "if INFER and params.get('n_lgb', 1) > 0:\n",
        "    df = test\n",
        "    yps = []; b = 1000\n",
        "    for i in range(0, len(df), b):\n",
        "        arr = get_features(df.iloc[i: i + b])\n",
        "        ypms = []\n",
        "        for model in lgb_models:\n",
        "            ypms.append(model.predict_proba(arr))\n",
        "        yps.append(np.stack(ypms).mean(0))\n",
        "        print('.', end = '')\n",
        "\n",
        "        if len(yps) % 2 == 0:\n",
        "            gc.collect()\n",
        "    print()\n",
        "\n",
        "    yp = np.concatenate(yps)"
      ],
      "metadata": {
        "id": "i9t0EsarrjXc",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "8cdb47d1-0496-48c6-f69e-4dbd68bf6c47"
      },
      "execution_count": 45,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            ".\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "lgb_preds = yp"
      ],
      "metadata": {
        "id": "b4cdjVIhrkXk"
      },
      "execution_count": 46,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Blend predictions"
      ],
      "metadata": {
        "id": "-Dwjck7VrnOB"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "lgb_wt = 0.3\n",
        "preds = lgb_wt * lgb_preds + (1 - lgb_wt) * llama_preds"
      ],
      "metadata": {
        "id": "jhERRyznrpHV"
      },
      "execution_count": 47,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import numpy as np\n",
        "\n",
        "# 確率値からone-hotベクトルへの変換\n",
        "one_hot_preds = np.zeros_like(preds)\n",
        "one_hot_preds[np.arange(len(preds)), preds.argmax(1)] = 1\n",
        "\n",
        "# %%\n",
        "# one_hot_predsをDataFrameに変換して出力する例\n",
        "out_one_hot = pd.DataFrame(one_hot_preds, index=df.id, columns=train.columns[-3:])\n",
        "display(out_one_hot.head())\n",
        "\n",
        "# %%\n",
        "# このDataFrameをCSVファイルに保存\n",
        "# out_one_hot.to_csv('submission.csv')"
      ],
      "metadata": {
        "id": "ujQWg_SzrqgZ",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 175
        },
        "outputId": "91683bea-f2f9-449e-ccc0-e4c36ab44299"
      },
      "execution_count": 54,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "         winner_model_a  winner_model_b  winner_tie\n",
              "id                                                 \n",
              "136060              0.0             1.0         0.0\n",
              "211333              1.0             0.0         0.0\n",
              "1233961             0.0             1.0         0.0"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-bbe13f62-1635-4c98-a91b-1edd275740f4\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>winner_model_a</th>\n",
              "      <th>winner_model_b</th>\n",
              "      <th>winner_tie</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>id</th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>136060</th>\n",
              "      <td>0.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>211333</th>\n",
              "      <td>1.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1233961</th>\n",
              "      <td>0.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-bbe13f62-1635-4c98-a91b-1edd275740f4')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-bbe13f62-1635-4c98-a91b-1edd275740f4 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-bbe13f62-1635-4c98-a91b-1edd275740f4');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-5389558f-ef92-4601-be99-b0554a7110bd\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-5389558f-ef92-4601-be99-b0554a7110bd')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-5389558f-ef92-4601-be99-b0554a7110bd button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "dataframe",
              "summary": "{\n  \"name\": \"out_one_hot\",\n  \"rows\": 3,\n  \"fields\": [\n    {\n      \"column\": \"id\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 613299,\n        \"min\": 136060,\n        \"max\": 1233961,\n        \"num_unique_values\": 3,\n        \"samples\": [\n          136060,\n          211333,\n          1233961\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"winner_model_a\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.5773502691896258,\n        \"min\": 0.0,\n        \"max\": 1.0,\n        \"num_unique_values\": 2,\n        \"samples\": [\n          1.0,\n          0.0\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"winner_model_b\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.5773502691896258,\n        \"min\": 0.0,\n        \"max\": 1.0,\n        \"num_unique_values\": 2,\n        \"samples\": [\n          0.0,\n          1.0\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"winner_tie\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.0,\n        \"min\": 0.0,\n        \"max\": 0.0,\n        \"num_unique_values\": 1,\n        \"samples\": [\n          0.0\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "out = pd.DataFrame(preds, index=df.id, columns=train.columns[-3:])\n",
        "display(out.head())"
      ],
      "metadata": {
        "id": "IvE4Dvm-rrp-",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 175
        },
        "outputId": "178f80de-4758-4f6d-a579-b6765faf3099"
      },
      "execution_count": 49,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "         winner_model_a  winner_model_b  winner_tie\n",
              "id                                                 \n",
              "136060         0.164693        0.551753    0.283749\n",
              "211333         0.427557        0.296565    0.276195\n",
              "1233961        0.273949        0.439712    0.286534"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-68f8c801-c58e-4563-b185-a08ff2005e03\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>winner_model_a</th>\n",
              "      <th>winner_model_b</th>\n",
              "      <th>winner_tie</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>id</th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>136060</th>\n",
              "      <td>0.164693</td>\n",
              "      <td>0.551753</td>\n",
              "      <td>0.283749</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>211333</th>\n",
              "      <td>0.427557</td>\n",
              "      <td>0.296565</td>\n",
              "      <td>0.276195</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1233961</th>\n",
              "      <td>0.273949</td>\n",
              "      <td>0.439712</td>\n",
              "      <td>0.286534</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-68f8c801-c58e-4563-b185-a08ff2005e03')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-68f8c801-c58e-4563-b185-a08ff2005e03 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-68f8c801-c58e-4563-b185-a08ff2005e03');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-2f245b9f-9713-4a02-b974-82542bdb8bee\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-2f245b9f-9713-4a02-b974-82542bdb8bee')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-2f245b9f-9713-4a02-b974-82542bdb8bee button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "dataframe",
              "summary": "{\n  \"name\": \"display(out\",\n  \"rows\": 3,\n  \"fields\": [\n    {\n      \"column\": \"id\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 613299,\n        \"min\": 136060,\n        \"max\": 1233961,\n        \"num_unique_values\": 3,\n        \"samples\": [\n          136060,\n          211333,\n          1233961\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"winner_model_a\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.1320538667139755,\n        \"min\": 0.16469306676012307,\n        \"max\": 0.4275565871755128,\n        \"num_unique_values\": 3,\n        \"samples\": [\n          0.16469306676012307,\n          0.4275565871755128,\n          0.27394942689345136\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"winner_model_b\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.12790938372322325,\n        \"min\": 0.29656539311828284,\n        \"max\": 0.5517530517360411,\n        \"num_unique_values\": 3,\n        \"samples\": [\n          0.5517530517360411,\n          0.29656539311828284,\n          0.4397115695250817\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"winner_tie\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.005349607531499087,\n        \"min\": 0.27619540251870434,\n        \"max\": 0.28653431608146696,\n        \"num_unique_values\": 3,\n        \"samples\": [\n          0.28374919400383586,\n          0.27619540251870434,\n          0.28653431608146696\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# make submission\n",
        "print(\"# ---------- # Make Submission # ---------- #\")\n",
        "filename = Config.name + \".csv\" if COLAB else \"submission.csv\"\n",
        "\n",
        "out.to_csv(os.path.join(SUBMISSION, filename), index=False)"
      ],
      "metadata": {
        "id": "E8DGOXvBmA5j",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "bbf2da24-102f-4b2a-a107-b7de31fcb7d6"
      },
      "execution_count": 52,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "# ---------- # Make Submission # ---------- #\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "Config.name"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "id": "1vS2P8H3zP3C",
        "outputId": "ae8f403c-8da7-4168-a19b-f448cd89ac97"
      },
      "execution_count": 53,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'baseline-001'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 53
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "lg30VGkkzXbc"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}