tokenize.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "ef3e5e41",
   "metadata": {},
   "outputs": [],
   "source": [
    "from transformers import AutoTokenizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "e3ad606b",
   "metadata": {},
   "outputs": [],
   "source": [
    "text = \"Also, here is a photo that represents an ideal life I'd love to lead. You can probably tell what my favourite colour is from my profile picture and the photo above.\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "2813af3f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Also, here is a photo that represents an ideal life I'd love to lead. You can probably tell what my favourite colour is from my profile picture and the photo above.\n"
     ]
    }
   ],
   "source": [
    "print(text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "af0b19ba",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a65cb378ef6045eca723d0b11a6b3e95",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading:   0%|          | 0.00/28.0 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c8b40b32cf834c5b887a0b270321abaf",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading:   0%|          | 0.00/570 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "20078e39d3054b2a857a63a0aae80b55",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading:   0%|          | 0.00/226k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d6bba3a7c22247908c8145b42171a38c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading:   0%|          | 0.00/455k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[101, 2036, 1010, 2182, 2003, 1037, 6302, 2008, 5836, 2019, 7812, 2166, 1045, 1005, 1040, 2293, 2000, 2599, 1012, 2017, 2064, 2763, 2425, 2054, 2026, 8837, 6120, 2003, 2013, 2026, 6337, 3861, 1998, 1996, 6302, 2682, 1012, 102]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "\"[CLS] also, here is a photo that represents an ideal life i'd love to lead. you can probably tell what my favourite colour is from my profile picture and the photo above. [SEP]\""
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Initialize the tokenizer\n",
    "tokenizer=AutoTokenizer.from_pretrained('bert-base-uncased')\n",
    "\n",
    "# Encoding with the tokenizer\n",
    "inputs=tokenizer.encode(text)\n",
    "print(inputs)\n",
    "tokenizer.decode(inputs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "c5854473",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[101, 2036, 1010, 2182, 2003, 1037, 6302, 2008, 5836, 2019, 7812, 2166, 1045, 1005, 1040, 2293, 2000, 2599, 1012, 2017, 2064, 2763, 2425, 2054, 2026, 8837, 6120, 2003, 2013, 2026, 6337, 3861, 1998, 1996, 6302, 2682, 1012, 102]\n"
     ]
    }
   ],
   "source": [
    "print(inputs)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}