bert-fine-tune-on-discord-data.ipynb

{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":7837629,"sourceType":"datasetVersion","datasetId":4594308},{"sourceId":7839317,"sourceType":"datasetVersion","datasetId":4595521}],"dockerImageVersionId":30665,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-03-15T04:51:42.280415Z","iopub.execute_input":"2024-03-15T04:51:42.280661Z","iopub.status.idle":"2024-03-15T04:51:43.366265Z","shell.execute_reply.started":"2024-03-15T04:51:42.280637Z","shell.execute_reply":"2024-03-15T04:51:43.365333Z"},"trusted":true},"execution_count":1,"outputs":[{"name":"stdout","text":"/kaggle/input/disc-data/disc_6.csv\n/kaggle/input/all-discord-data/combined_df.csv\n","output_type":"stream"}]},{"cell_type":"code","source":"! pip install -U accelerate\n! pip install datasets transformers\n!apt install git-lfs","metadata":{"execution":{"iopub.status.busy":"2024-03-15T04:51:43.367747Z","iopub.execute_input":"2024-03-15T04:51:43.368504Z","iopub.status.idle":"2024-03-15T04:52:13.795326Z","shell.execute_reply.started":"2024-03-15T04:51:43.368459Z","shell.execute_reply":"2024-03-15T04:52:13.794142Z"},"trusted":true},"execution_count":2,"outputs":[{"name":"stdout","text":"Requirement already satisfied: accelerate in /opt/conda/lib/python3.10/site-packages (0.27.2)\nCollecting accelerate\n  Downloading accelerate-0.28.0-py3-none-any.whl.metadata (18 kB)\nRequirement already satisfied: numpy>=1.17 in /opt/conda/lib/python3.10/site-packages (from accelerate) (1.26.4)\nRequirement already satisfied: packaging>=20.0 in /opt/conda/lib/python3.10/site-packages (from accelerate) (21.3)\nRequirement already satisfied: psutil in /opt/conda/lib/python3.10/site-packages (from accelerate) (5.9.3)\nRequirement already satisfied: pyyaml in /opt/conda/lib/python3.10/site-packages (from accelerate) (6.0.1)\nRequirement already satisfied: torch>=1.10.0 in /opt/conda/lib/python3.10/site-packages (from accelerate) (2.1.2)\nRequirement already satisfied: huggingface-hub in /opt/conda/lib/python3.10/site-packages (from accelerate) (0.20.3)\nRequirement already satisfied: safetensors>=0.3.1 in /opt/conda/lib/python3.10/site-packages (from accelerate) (0.4.2)\nRequirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /opt/conda/lib/python3.10/site-packages (from packaging>=20.0->accelerate) (3.1.1)\nRequirement already satisfied: filelock in /opt/conda/lib/python3.10/site-packages (from torch>=1.10.0->accelerate) (3.13.1)\nRequirement already satisfied: typing-extensions in /opt/conda/lib/python3.10/site-packages (from torch>=1.10.0->accelerate) (4.9.0)\nRequirement already satisfied: sympy in /opt/conda/lib/python3.10/site-packages (from torch>=1.10.0->accelerate) (1.12)\nRequirement already satisfied: networkx in /opt/conda/lib/python3.10/site-packages (from torch>=1.10.0->accelerate) (3.2.1)\nRequirement already satisfied: jinja2 in /opt/conda/lib/python3.10/site-packages (from torch>=1.10.0->accelerate) (3.1.2)\nRequirement already satisfied: fsspec in /opt/conda/lib/python3.10/site-packages (from torch>=1.10.0->accelerate) (2024.2.0)\nRequirement already satisfied: requests in /opt/conda/lib/python3.10/site-packages (from huggingface-hub->accelerate) (2.31.0)\nRequirement already satisfied: tqdm>=4.42.1 in /opt/conda/lib/python3.10/site-packages (from huggingface-hub->accelerate) (4.66.1)\nRequirement already satisfied: MarkupSafe>=2.0 in /opt/conda/lib/python3.10/site-packages (from jinja2->torch>=1.10.0->accelerate) (2.1.3)\nRequirement already satisfied: charset-normalizer<4,>=2 in /opt/conda/lib/python3.10/site-packages (from requests->huggingface-hub->accelerate) (3.3.2)\nRequirement already satisfied: idna<4,>=2.5 in /opt/conda/lib/python3.10/site-packages (from requests->huggingface-hub->accelerate) (3.6)\nRequirement already satisfied: urllib3<3,>=1.21.1 in /opt/conda/lib/python3.10/site-packages (from requests->huggingface-hub->accelerate) (1.26.18)\nRequirement already satisfied: certifi>=2017.4.17 in /opt/conda/lib/python3.10/site-packages (from requests->huggingface-hub->accelerate) (2024.2.2)\nRequirement already satisfied: mpmath>=0.19 in /opt/conda/lib/python3.10/site-packages (from sympy->torch>=1.10.0->accelerate) (1.3.0)\nDownloading accelerate-0.28.0-py3-none-any.whl (290 kB)\n\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m290.1/290.1 kB\u001b[0m \u001b[31m2.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0ma \u001b[36m0:00:01\u001b[0m\n\u001b[?25hInstalling collected packages: accelerate\n  Attempting uninstall: accelerate\n    Found existing installation: accelerate 0.27.2\n    Uninstalling accelerate-0.27.2:\n      Successfully uninstalled accelerate-0.27.2\nSuccessfully installed accelerate-0.28.0\nRequirement already satisfied: datasets in /opt/conda/lib/python3.10/site-packages (2.1.0)\nRequirement already satisfied: transformers in /opt/conda/lib/python3.10/site-packages (4.38.1)\nRequirement already satisfied: numpy>=1.17 in /opt/conda/lib/python3.10/site-packages (from datasets) (1.26.4)\nRequirement already satisfied: pyarrow>=5.0.0 in /opt/conda/lib/python3.10/site-packages (from datasets) (11.0.0)\nRequirement already satisfied: dill in /opt/conda/lib/python3.10/site-packages (from datasets) (0.3.8)\nRequirement already satisfied: pandas in /opt/conda/lib/python3.10/site-packages (from datasets) (2.1.4)\nRequirement already satisfied: requests>=2.19.0 in /opt/conda/lib/python3.10/site-packages (from datasets) (2.31.0)\nRequirement already satisfied: tqdm>=4.62.1 in /opt/conda/lib/python3.10/site-packages (from datasets) (4.66.1)\nRequirement already satisfied: xxhash in /opt/conda/lib/python3.10/site-packages (from datasets) (3.4.1)\nRequirement already satisfied: multiprocess in /opt/conda/lib/python3.10/site-packages (from datasets) (0.70.16)\nRequirement already satisfied: fsspec>=2021.05.0 in /opt/conda/lib/python3.10/site-packages (from fsspec[http]>=2021.05.0->datasets) (2024.2.0)\nRequirement already satisfied: aiohttp in /opt/conda/lib/python3.10/site-packages (from datasets) (3.9.1)\nRequirement already satisfied: huggingface-hub<1.0.0,>=0.1.0 in /opt/conda/lib/python3.10/site-packages (from datasets) (0.20.3)\nRequirement already satisfied: packaging in /opt/conda/lib/python3.10/site-packages (from datasets) (21.3)\nRequirement already satisfied: responses<0.19 in /opt/conda/lib/python3.10/site-packages (from datasets) (0.18.0)\nRequirement already satisfied: filelock in /opt/conda/lib/python3.10/site-packages (from transformers) (3.13.1)\nRequirement already satisfied: pyyaml>=5.1 in /opt/conda/lib/python3.10/site-packages (from transformers) (6.0.1)\nRequirement already satisfied: regex!=2019.12.17 in /opt/conda/lib/python3.10/site-packages (from transformers) (2023.12.25)\nRequirement already satisfied: tokenizers<0.19,>=0.14 in /opt/conda/lib/python3.10/site-packages (from transformers) (0.15.2)\nRequirement already satisfied: safetensors>=0.4.1 in /opt/conda/lib/python3.10/site-packages (from transformers) (0.4.2)\nRequirement already satisfied: attrs>=17.3.0 in /opt/conda/lib/python3.10/site-packages (from aiohttp->datasets) (23.2.0)\nRequirement already satisfied: multidict<7.0,>=4.5 in /opt/conda/lib/python3.10/site-packages (from aiohttp->datasets) (6.0.4)\nRequirement already satisfied: yarl<2.0,>=1.0 in /opt/conda/lib/python3.10/site-packages (from aiohttp->datasets) (1.9.3)\nRequirement already satisfied: frozenlist>=1.1.1 in /opt/conda/lib/python3.10/site-packages (from aiohttp->datasets) (1.4.1)\nRequirement already satisfied: aiosignal>=1.1.2 in /opt/conda/lib/python3.10/site-packages (from aiohttp->datasets) (1.3.1)\nRequirement already satisfied: async-timeout<5.0,>=4.0 in /opt/conda/lib/python3.10/site-packages (from aiohttp->datasets) (4.0.3)\nRequirement already satisfied: typing-extensions>=3.7.4.3 in /opt/conda/lib/python3.10/site-packages (from huggingface-hub<1.0.0,>=0.1.0->datasets) (4.9.0)\nRequirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /opt/conda/lib/python3.10/site-packages (from packaging->datasets) (3.1.1)\nRequirement already satisfied: charset-normalizer<4,>=2 in /opt/conda/lib/python3.10/site-packages (from requests>=2.19.0->datasets) (3.3.2)\nRequirement already satisfied: idna<4,>=2.5 in /opt/conda/lib/python3.10/site-packages (from requests>=2.19.0->datasets) (3.6)\nRequirement already satisfied: urllib3<3,>=1.21.1 in /opt/conda/lib/python3.10/site-packages (from requests>=2.19.0->datasets) (1.26.18)\nRequirement already satisfied: certifi>=2017.4.17 in /opt/conda/lib/python3.10/site-packages (from requests>=2.19.0->datasets) (2024.2.2)\nRequirement already satisfied: python-dateutil>=2.8.2 in /opt/conda/lib/python3.10/site-packages (from pandas->datasets) (2.8.2)\nRequirement already satisfied: pytz>=2020.1 in /opt/conda/lib/python3.10/site-packages (from pandas->datasets) (2023.3.post1)\nRequirement already satisfied: tzdata>=2022.1 in /opt/conda/lib/python3.10/site-packages (from pandas->datasets) (2023.4)\nRequirement already satisfied: six>=1.5 in /opt/conda/lib/python3.10/site-packages (from python-dateutil>=2.8.2->pandas->datasets) (1.16.0)\nReading package lists... Done\nBuilding dependency tree       \nReading state information... Done\ngit-lfs is already the newest version (2.9.2-1).\n0 upgraded, 0 newly installed, 0 to remove and 38 not upgraded.\n","output_type":"stream"}]},{"cell_type":"code","source":"from huggingface_hub import notebook_login, login\n\nnotebook_login()","metadata":{"execution":{"iopub.status.busy":"2024-03-14T03:36:38.314646Z","iopub.execute_input":"2024-03-14T03:36:38.315024Z","iopub.status.idle":"2024-03-14T03:36:38.664641Z","shell.execute_reply.started":"2024-03-14T03:36:38.314995Z","shell.execute_reply":"2024-03-14T03:36:38.663778Z"},"trusted":true},"execution_count":3,"outputs":[{"output_type":"display_data","data":{"text/plain":"VBox(children=(HTML(value='<center> <img\\nsrc=https://huggingface.co/front/assets/huggingface_logo-noborder.sv…","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"ab20a09a09504298b5a6a6048d05aeac"}},"metadata":{}}]},{"cell_type":"code","source":"import transformers\n\nprint(transformers.__version__)","metadata":{"execution":{"iopub.status.busy":"2024-03-14T03:36:38.666562Z","iopub.execute_input":"2024-03-14T03:36:38.666835Z","iopub.status.idle":"2024-03-14T03:36:47.768009Z","shell.execute_reply.started":"2024-03-14T03:36:38.666811Z","shell.execute_reply":"2024-03-14T03:36:47.767004Z"},"trusted":true},"execution_count":4,"outputs":[{"name":"stdout","text":"4.38.1\n","output_type":"stream"}]},{"cell_type":"code","source":"from datasets import load_dataset, Dataset, DatasetDict\nimport pandas as pd\nfrom sklearn.model_selection import train_test_split","metadata":{"execution":{"iopub.status.busy":"2024-03-14T03:36:47.769381Z","iopub.execute_input":"2024-03-14T03:36:47.769816Z","iopub.status.idle":"2024-03-14T03:36:49.549940Z","shell.execute_reply.started":"2024-03-14T03:36:47.769788Z","shell.execute_reply":"2024-03-14T03:36:49.549073Z"},"trusted":true},"execution_count":5,"outputs":[]},{"cell_type":"code","source":"!pip install evaluate transformers[sentencepiece]","metadata":{"execution":{"iopub.status.busy":"2024-03-14T03:36:49.551130Z","iopub.execute_input":"2024-03-14T03:36:49.551723Z","iopub.status.idle":"2024-03-14T03:37:03.627085Z","shell.execute_reply.started":"2024-03-14T03:36:49.551688Z","shell.execute_reply":"2024-03-14T03:37:03.625866Z"},"trusted":true},"execution_count":6,"outputs":[{"name":"stdout","text":"Collecting evaluate\n  Downloading evaluate-0.4.1-py3-none-any.whl.metadata (9.4 kB)\nRequirement already satisfied: transformers[sentencepiece] in /opt/conda/lib/python3.10/site-packages (4.38.1)\nRequirement already satisfied: datasets>=2.0.0 in /opt/conda/lib/python3.10/site-packages (from evaluate) (2.1.0)\nRequirement already satisfied: numpy>=1.17 in /opt/conda/lib/python3.10/site-packages (from evaluate) (1.26.4)\nRequirement already satisfied: dill in /opt/conda/lib/python3.10/site-packages (from evaluate) (0.3.8)\nRequirement already satisfied: pandas in /opt/conda/lib/python3.10/site-packages (from evaluate) (2.1.4)\nRequirement already satisfied: requests>=2.19.0 in /opt/conda/lib/python3.10/site-packages (from evaluate) (2.31.0)\nRequirement already satisfied: tqdm>=4.62.1 in /opt/conda/lib/python3.10/site-packages (from evaluate) (4.66.1)\nRequirement already satisfied: xxhash in /opt/conda/lib/python3.10/site-packages (from evaluate) (3.4.1)\nRequirement already satisfied: multiprocess in /opt/conda/lib/python3.10/site-packages (from evaluate) (0.70.16)\nRequirement already satisfied: fsspec>=2021.05.0 in /opt/conda/lib/python3.10/site-packages (from fsspec[http]>=2021.05.0->evaluate) (2024.2.0)\nRequirement already satisfied: huggingface-hub>=0.7.0 in /opt/conda/lib/python3.10/site-packages (from evaluate) (0.20.3)\nRequirement already satisfied: packaging in /opt/conda/lib/python3.10/site-packages (from evaluate) (21.3)\nRequirement already satisfied: responses<0.19 in /opt/conda/lib/python3.10/site-packages (from evaluate) (0.18.0)\nRequirement already satisfied: filelock in /opt/conda/lib/python3.10/site-packages (from transformers[sentencepiece]) (3.13.1)\nRequirement already satisfied: pyyaml>=5.1 in /opt/conda/lib/python3.10/site-packages (from transformers[sentencepiece]) (6.0.1)\nRequirement already satisfied: regex!=2019.12.17 in /opt/conda/lib/python3.10/site-packages (from transformers[sentencepiece]) (2023.12.25)\nRequirement already satisfied: tokenizers<0.19,>=0.14 in /opt/conda/lib/python3.10/site-packages (from transformers[sentencepiece]) (0.15.2)\nRequirement already satisfied: safetensors>=0.4.1 in /opt/conda/lib/python3.10/site-packages (from transformers[sentencepiece]) (0.4.2)\nRequirement already satisfied: sentencepiece!=0.1.92,>=0.1.91 in /opt/conda/lib/python3.10/site-packages (from transformers[sentencepiece]) (0.2.0)\nRequirement already satisfied: protobuf in /opt/conda/lib/python3.10/site-packages (from transformers[sentencepiece]) (3.20.3)\nRequirement already satisfied: pyarrow>=5.0.0 in /opt/conda/lib/python3.10/site-packages (from datasets>=2.0.0->evaluate) (11.0.0)\nRequirement already satisfied: aiohttp in /opt/conda/lib/python3.10/site-packages (from datasets>=2.0.0->evaluate) (3.9.1)\nRequirement already satisfied: typing-extensions>=3.7.4.3 in /opt/conda/lib/python3.10/site-packages (from huggingface-hub>=0.7.0->evaluate) (4.9.0)\nRequirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /opt/conda/lib/python3.10/site-packages (from packaging->evaluate) (3.1.1)\nRequirement already satisfied: charset-normalizer<4,>=2 in /opt/conda/lib/python3.10/site-packages (from requests>=2.19.0->evaluate) (3.3.2)\nRequirement already satisfied: idna<4,>=2.5 in /opt/conda/lib/python3.10/site-packages (from requests>=2.19.0->evaluate) (3.6)\nRequirement already satisfied: urllib3<3,>=1.21.1 in /opt/conda/lib/python3.10/site-packages (from requests>=2.19.0->evaluate) (1.26.18)\nRequirement already satisfied: certifi>=2017.4.17 in /opt/conda/lib/python3.10/site-packages (from requests>=2.19.0->evaluate) (2024.2.2)\nRequirement already satisfied: python-dateutil>=2.8.2 in /opt/conda/lib/python3.10/site-packages (from pandas->evaluate) (2.8.2)\nRequirement already satisfied: pytz>=2020.1 in /opt/conda/lib/python3.10/site-packages (from pandas->evaluate) (2023.3.post1)\nRequirement already satisfied: tzdata>=2022.1 in /opt/conda/lib/python3.10/site-packages (from pandas->evaluate) (2023.4)\nRequirement already satisfied: attrs>=17.3.0 in /opt/conda/lib/python3.10/site-packages (from aiohttp->datasets>=2.0.0->evaluate) (23.2.0)\nRequirement already satisfied: multidict<7.0,>=4.5 in /opt/conda/lib/python3.10/site-packages (from aiohttp->datasets>=2.0.0->evaluate) (6.0.4)\nRequirement already satisfied: yarl<2.0,>=1.0 in /opt/conda/lib/python3.10/site-packages (from aiohttp->datasets>=2.0.0->evaluate) (1.9.3)\nRequirement already satisfied: frozenlist>=1.1.1 in /opt/conda/lib/python3.10/site-packages (from aiohttp->datasets>=2.0.0->evaluate) (1.4.1)\nRequirement already satisfied: aiosignal>=1.1.2 in /opt/conda/lib/python3.10/site-packages (from aiohttp->datasets>=2.0.0->evaluate) (1.3.1)\nRequirement already satisfied: async-timeout<5.0,>=4.0 in /opt/conda/lib/python3.10/site-packages (from aiohttp->datasets>=2.0.0->evaluate) (4.0.3)\nRequirement already satisfied: six>=1.5 in /opt/conda/lib/python3.10/site-packages (from python-dateutil>=2.8.2->pandas->evaluate) (1.16.0)\nDownloading evaluate-0.4.1-py3-none-any.whl (84 kB)\n\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m84.1/84.1 kB\u001b[0m \u001b[31m806.9 kB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\u001b[36m0:00:01\u001b[0m\n\u001b[?25hInstalling collected packages: evaluate\nSuccessfully installed evaluate-0.4.1\n","output_type":"stream"}]},{"cell_type":"code","source":"!ls -la","metadata":{"execution":{"iopub.status.busy":"2024-03-14T03:37:03.628505Z","iopub.execute_input":"2024-03-14T03:37:03.628813Z","iopub.status.idle":"2024-03-14T03:37:04.636201Z","shell.execute_reply.started":"2024-03-14T03:37:03.628775Z","shell.execute_reply":"2024-03-14T03:37:04.634921Z"},"trusted":true},"execution_count":7,"outputs":[{"name":"stdout","text":"total 12\ndrwxr-xr-x 3 root root 4096 Mar 14 03:35 .\ndrwxr-xr-x 5 root root 4096 Mar 14 03:35 ..\ndrwxr-xr-x 2 root root 4096 Mar 14 03:35 .virtual_documents\n","output_type":"stream"}]},{"cell_type":"code","source":"df = pd.read_csv(\"/kaggle/input/disc_6.csv\").dropna().drop(\"Author_ID\", axis=1)\n# df_2 = pd.read_csv(\"/content/drive/MyDrive/Data/disc_data/disc_7.csv\")\n\ntrain, test = train_test_split(df, test_size=0.2)\n#test, val = train_test_split(test, test_size=0.5)","metadata":{"execution":{"iopub.status.busy":"2024-03-14T03:44:46.574082Z","iopub.execute_input":"2024-03-14T03:44:46.574793Z","iopub.status.idle":"2024-03-14T03:44:47.396768Z","shell.execute_reply.started":"2024-03-14T03:44:46.574758Z","shell.execute_reply":"2024-03-14T03:44:47.395574Z"},"trusted":true},"execution_count":20,"outputs":[]},{"cell_type":"code","source":"train.head()","metadata":{"execution":{"iopub.status.busy":"2024-03-14T03:44:52.124569Z","iopub.execute_input":"2024-03-14T03:44:52.125221Z","iopub.status.idle":"2024-03-14T03:44:52.134942Z","shell.execute_reply.started":"2024-03-14T03:44:52.125188Z","shell.execute_reply":"2024-03-14T03:44:52.133924Z"},"trusted":true},"execution_count":21,"outputs":[{"execution_count":21,"output_type":"execute_result","data":{"text/plain":"                         Message\n704932                 :o\\nshow!\n442973                       Fuc\n639576        MRS OBAMA GET DOWN\n387338    I DON'T KNOW\\nIM SORRY\n690011  you cant hide the truth!","text/html":"<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>Message</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>704932</th>\n      <td>:o\\nshow!</td>\n    </tr>\n    <tr>\n      <th>442973</th>\n      <td>Fuc</td>\n    </tr>\n    <tr>\n      <th>639576</th>\n      <td>MRS OBAMA GET DOWN</td>\n    </tr>\n    <tr>\n      <th>387338</th>\n      <td>I DON'T KNOW\\nIM SORRY</td>\n    </tr>\n    <tr>\n      <th>690011</th>\n      <td>you cant hide the truth!</td>\n    </tr>\n  </tbody>\n</table>\n</div>"},"metadata":{}}]},{"cell_type":"code","source":"test.head()","metadata":{"execution":{"iopub.status.busy":"2024-03-14T03:44:53.724235Z","iopub.execute_input":"2024-03-14T03:44:53.725088Z","iopub.status.idle":"2024-03-14T03:44:53.733485Z","shell.execute_reply.started":"2024-03-14T03:44:53.725055Z","shell.execute_reply":"2024-03-14T03:44:53.732592Z"},"trusted":true},"execution_count":22,"outputs":[{"execution_count":22,"output_type":"execute_result","data":{"text/plain":"                                                  Message\n595776                                     fren answer me\n209890                                      how do u know\n676675                                         IDK A TOBY\n125952                                  This is all of us\n305821  Since its igns job to rate\\nI think they overrate","text/html":"<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>Message</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>595776</th>\n      <td>fren answer me</td>\n    </tr>\n    <tr>\n      <th>209890</th>\n      <td>how do u know</td>\n    </tr>\n    <tr>\n      <th>676675</th>\n      <td>IDK A TOBY</td>\n    </tr>\n    <tr>\n      <th>125952</th>\n      <td>This is all of us</td>\n    </tr>\n    <tr>\n      <th>305821</th>\n      <td>Since its igns job to rate\\nI think they overrate</td>\n    </tr>\n  </tbody>\n</table>\n</div>"},"metadata":{}}]},{"cell_type":"code","source":"train_ds = Dataset.from_pandas(train)\ntest_ds = Dataset.from_pandas(test)\n#validation_ds = Dataset.from_pandas(val)\n\ndatasets = DatasetDict({\n    'train': train_ds,\n    'test': test_ds,\n    #'valid': validation_ds\n})","metadata":{"execution":{"iopub.status.busy":"2024-03-14T03:45:12.687186Z","iopub.execute_input":"2024-03-14T03:45:12.688004Z","iopub.status.idle":"2024-03-14T03:45:12.817547Z","shell.execute_reply.started":"2024-03-14T03:45:12.687968Z","shell.execute_reply":"2024-03-14T03:45:12.816677Z"},"trusted":true},"execution_count":23,"outputs":[]},{"cell_type":"code","source":"datasets[\"train\"][11]","metadata":{"execution":{"iopub.status.busy":"2024-03-14T03:45:15.326174Z","iopub.execute_input":"2024-03-14T03:45:15.326883Z","iopub.status.idle":"2024-03-14T03:45:15.333260Z","shell.execute_reply.started":"2024-03-14T03:45:15.326845Z","shell.execute_reply":"2024-03-14T03:45:15.332289Z"},"trusted":true},"execution_count":24,"outputs":[{"execution_count":24,"output_type":"execute_result","data":{"text/plain":"{'Message': '5 admins geez', '__index_level_0__': 132385}"},"metadata":{}}]},{"cell_type":"code","source":"!git config --global user.email \"emailsiddha@gmail.com\"\n!git config --global user.name \"Siddharth Gupta\"","metadata":{"execution":{"iopub.status.busy":"2024-03-14T03:39:50.754464Z","iopub.execute_input":"2024-03-14T03:39:50.755110Z","iopub.status.idle":"2024-03-14T03:39:52.748722Z","shell.execute_reply.started":"2024-03-14T03:39:50.755077Z","shell.execute_reply":"2024-03-14T03:39:52.747522Z"},"trusted":true},"execution_count":15,"outputs":[]},{"cell_type":"code","source":"from transformers import AutoTokenizer\n\n\nmodel_checkpoint = \"distilbert-base-uncased\"\ntokenizer = AutoTokenizer.from_pretrained(model_checkpoint)","metadata":{"execution":{"iopub.status.busy":"2024-03-14T03:43:47.510107Z","iopub.execute_input":"2024-03-14T03:43:47.510521Z","iopub.status.idle":"2024-03-14T03:43:48.684371Z","shell.execute_reply.started":"2024-03-14T03:43:47.510490Z","shell.execute_reply":"2024-03-14T03:43:48.683293Z"},"trusted":true},"execution_count":18,"outputs":[{"output_type":"display_data","data":{"text/plain":"tokenizer_config.json:   0%|          | 0.00/28.0 [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"fc9455cba59b4a12950cc828f1afa7da"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"config.json:   0%|          | 0.00/483 [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"e35d5b1316724d749afc4cd37a125652"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"vocab.txt:   0%|          | 0.00/232k [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"771d6867417c40978157b026a62b339b"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"tokenizer.json:   0%|          | 0.00/466k [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"36be8a06cf7c4e69999de829eabf9399"}},"metadata":{}}]},{"cell_type":"code","source":"datasets","metadata":{"execution":{"iopub.status.busy":"2024-03-14T03:45:40.716441Z","iopub.execute_input":"2024-03-14T03:45:40.717227Z","iopub.status.idle":"2024-03-14T03:45:40.722993Z","shell.execute_reply.started":"2024-03-14T03:45:40.717183Z","shell.execute_reply":"2024-03-14T03:45:40.722007Z"},"trusted":true},"execution_count":25,"outputs":[{"execution_count":25,"output_type":"execute_result","data":{"text/plain":"DatasetDict({\n    train: Dataset({\n        features: ['Message', '__index_level_0__'],\n        num_rows: 586284\n    })\n    test: Dataset({\n        features: ['Message', '__index_level_0__'],\n        num_rows: 146572\n    })\n})"},"metadata":{}}]},{"cell_type":"code","source":"def tokenize_function(examples):\n    result = tokenizer(examples[\"Message\"])\n    if tokenizer.is_fast:\n        result[\"word_ids\"] = [result.word_ids(i) for i in range(len(result[\"input_ids\"]))]\n    return result\n\n\n# Use batched=True to activate fast multithreading!\ntokenized_datasets = datasets.map(\n    tokenize_function, batched=True, remove_columns=[\"Message\",\"__index_level_0__\"]\n)\ntokenized_datasets","metadata":{"execution":{"iopub.status.busy":"2024-03-14T03:51:32.566377Z","iopub.execute_input":"2024-03-14T03:51:32.567257Z","iopub.status.idle":"2024-03-14T03:52:00.909573Z","shell.execute_reply.started":"2024-03-14T03:51:32.567223Z","shell.execute_reply":"2024-03-14T03:52:00.908642Z"},"trusted":true},"execution_count":27,"outputs":[{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/587 [00:00<?, ?ba/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"23b890448a6c4f148f1685c2490b3a1d"}},"metadata":{}},{"name":"stderr","text":"Token indices sequence length is longer than the specified maximum sequence length for this model (1141 > 512). Running this sequence through the model will result in indexing errors\n","output_type":"stream"},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/147 [00:00<?, ?ba/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"dff072db22a54b40a95959faa860696e"}},"metadata":{}},{"execution_count":27,"output_type":"execute_result","data":{"text/plain":"DatasetDict({\n    train: Dataset({\n        features: ['input_ids', 'attention_mask', 'word_ids'],\n        num_rows: 586284\n    })\n    test: Dataset({\n        features: ['input_ids', 'attention_mask', 'word_ids'],\n        num_rows: 146572\n    })\n})"},"metadata":{}}]},{"cell_type":"code","source":"chunk_size = 32","metadata":{"execution":{"iopub.status.busy":"2024-03-14T03:55:51.679079Z","iopub.execute_input":"2024-03-14T03:55:51.679470Z","iopub.status.idle":"2024-03-14T03:55:51.684136Z","shell.execute_reply.started":"2024-03-14T03:55:51.679443Z","shell.execute_reply":"2024-03-14T03:55:51.682918Z"},"trusted":true},"execution_count":40,"outputs":[]},{"cell_type":"code","source":"tokenized_samples = tokenized_datasets[\"train\"][:3]","metadata":{"execution":{"iopub.status.busy":"2024-03-14T03:55:51.847769Z","iopub.execute_input":"2024-03-14T03:55:51.848196Z","iopub.status.idle":"2024-03-14T03:55:51.853891Z","shell.execute_reply.started":"2024-03-14T03:55:51.848164Z","shell.execute_reply":"2024-03-14T03:55:51.852892Z"},"trusted":true},"execution_count":41,"outputs":[]},{"cell_type":"code","source":"concatenated_examples = {\n    k: sum(tokenized_samples[k], []) for k in tokenized_samples.keys()}","metadata":{"execution":{"iopub.status.busy":"2024-03-14T03:55:52.006735Z","iopub.execute_input":"2024-03-14T03:55:52.007127Z","iopub.status.idle":"2024-03-14T03:55:52.011968Z","shell.execute_reply.started":"2024-03-14T03:55:52.007097Z","shell.execute_reply":"2024-03-14T03:55:52.010910Z"},"trusted":true},"execution_count":42,"outputs":[]},{"cell_type":"code","source":"total_length = len(concatenated_examples[\"input_ids\"])\nchunks = {\n    k: [t[i : i + chunk_size] for i in range(0, total_length, chunk_size)]\n    for k, t in concatenated_examples.items()\n}\n\nfor chunk in chunks[\"input_ids\"]:\n    print(f\"'>>> Chunk length: {len(chunk)}'\")","metadata":{"execution":{"iopub.status.busy":"2024-03-14T03:55:52.164834Z","iopub.execute_input":"2024-03-14T03:55:52.165749Z","iopub.status.idle":"2024-03-14T03:55:52.171928Z","shell.execute_reply.started":"2024-03-14T03:55:52.165714Z","shell.execute_reply":"2024-03-14T03:55:52.170920Z"},"trusted":true},"execution_count":43,"outputs":[{"name":"stdout","text":"'>>> Chunk length: 18'\n","output_type":"stream"}]},{"cell_type":"code","source":"train.describe()","metadata":{"execution":{"iopub.status.busy":"2024-03-14T03:56:11.391680Z","iopub.execute_input":"2024-03-14T03:56:11.392727Z","iopub.status.idle":"2024-03-14T03:56:12.148019Z","shell.execute_reply.started":"2024-03-14T03:56:11.392680Z","shell.execute_reply":"2024-03-14T03:56:12.146906Z"},"trusted":true},"execution_count":44,"outputs":[{"execution_count":44,"output_type":"execute_result","data":{"text/plain":"        Message\ncount    586284\nunique   431904\ntop     Welcome\nfreq       4751","text/html":"<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>Message</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>count</th>\n      <td>586284</td>\n    </tr>\n    <tr>\n      <th>unique</th>\n      <td>431904</td>\n    </tr>\n    <tr>\n      <th>top</th>\n      <td>Welcome</td>\n    </tr>\n    <tr>\n      <th>freq</th>\n      <td>4751</td>\n    </tr>\n  </tbody>\n</table>\n</div>"},"metadata":{}}]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}