MakeMoreMLP.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "1506c74f-c3ce-490d-919b-7e2c3f90af0e",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Following along with\n",
    "# https://www.youtube.com/watch?v=TCH_1BHY58I&list=PLAqhIrjkxbuWI23v9cThsA9GvCAUhRvKZ&index=3\n",
    "# https://github.com/karpathy/nn-zero-to-hero/blob/master/lectures/makemore/makemore_part2_mlp.ipynb\n",
    "import torch\n",
    "import random\n",
    "\n",
    "# for making figures\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "4751fe67-8bdd-490c-a2f8-79e16cb765c2",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# read in all the words - helps us build words\n",
    "all_words = open(\"words.txt\", \"r\").read().splitlines()\n",
    "MAX_WORDS = min(10_000, len(all_words))\n",
    "random.seed(6_6_1978)  # pin the RNG\n",
    "words = random.sample(all_words, MAX_WORDS)\n",
    "random.shuffle(words)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "f5121f4f-a0b8-4ff1-b2b2-399e2ef9872e",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['beatris',\n",
       " 'itzayana',\n",
       " 'raelyn',\n",
       " 'janalise',\n",
       " 'haseeb',\n",
       " 'kyian',\n",
       " 'lloyd',\n",
       " 'aidric',\n",
       " 'aizik',\n",
       " 'ivon',\n",
       " 'aryannah',\n",
       " 'wayne',\n",
       " 'tauren',\n",
       " 'mailey',\n",
       " 'manases',\n",
       " 'lamine',\n",
       " 'statham',\n",
       " 'kyro',\n",
       " 'jalee',\n",
       " 'keaston']"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "random.sample(all_words, 20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "b21c8106-b83b-48c0-823a-0c8948934281",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# build the vocabulary of characters and mappings to/from integers\n",
    "chars = sorted(list(set(\"\".join(words))))\n",
    "stoi = {s: i + 1 for i, s in enumerate(chars)}\n",
    "stoi[\".\"] = 0\n",
    "itos = {i: s for s, i in stoi.items()}\n",
    "\n",
    "\n",
    "def string_to_index(s):\n",
    "    return stoi[s]\n",
    "\n",
    "\n",
    "def index_to_string(s):\n",
    "    return itos[s]\n",
    "\n",
    "\n",
    "def to_word(t):\n",
    "    return \"\".join([itos[i.item()] for i in t])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "f01aaa4c-588b-468d-8814-3ef0644117a9",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# build the dataset\n",
    "def build_dataset(words: list[str]):\n",
    "    TERMINAL = \".\"\n",
    "\n",
    "    block_size = (\n",
    "        4  # context length: how many characters do we take to predict the next one?\n",
    "    )\n",
    "    X, Y = [], []\n",
    "    for word in words[:]:\n",
    "        # print(word)\n",
    "        context = [\n",
    "            string_to_index(TERMINAL)\n",
    "        ] * block_size  # we start with a full terminal string\n",
    "        for char in word + \".\":\n",
    "            ix = string_to_index(char)\n",
    "            X.append(context)\n",
    "            Y.append(ix)\n",
    "            # print(''.join(itos[i] for i in context), '--->', itos[ix])\n",
    "            context = context[1:] + [ix]  # crop and append\n",
    "\n",
    "    X = torch.tensor(X)\n",
    "    Y = torch.tensor(Y)\n",
    "    return X, Y\n",
    "\n",
    "\n",
    "X, Y = build_dataset(words)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "21f66f00-55a9-4e3e-b485-3a050c66547f",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "hima=>m == tensor([ 8,  9, 13,  1]), 13 \n",
      "imam=>a == tensor([ 9, 13,  1, 13]), 1 \n",
      "mama=>n == tensor([13,  1, 13,  1]), 14 \n",
      "aman=>d == tensor([ 1, 13,  1, 14]), 4 \n",
      "mand=>a == tensor([13,  1, 14,  4]), 1 \n",
      "anda=>. == tensor([ 1, 14,  4,  1]), 0 \n",
      "....=>c == tensor([0, 0, 0, 0]), 3 \n",
      "...c=>r == tensor([0, 0, 0, 3]), 18 \n",
      "..cr=>i == tensor([ 0,  0,  3, 18]), 9 \n",
      ".cri=>s == tensor([ 0,  3, 18,  9]), 19 \n",
      "cris=>p == tensor([ 3, 18,  9, 19]), 16 \n",
      "risp=>i == tensor([18,  9, 19, 16]), 9 \n",
      "ispi=>n == tensor([ 9, 19, 16,  9]), 14 \n",
      "spin=>. == tensor([19, 16,  9, 14]), 0 \n",
      "....=>c == tensor([0, 0, 0, 0]), 3 \n",
      "...c=>e == tensor([0, 0, 0, 3]), 5 \n",
      "..ce=>d == tensor([0, 0, 3, 5]), 4 \n",
      ".ced=>r == tensor([0, 3, 5, 4]), 18 \n",
      "cedr=>i == tensor([ 3,  5,  4, 18]), 9 \n",
      "edri=>c == tensor([ 5,  4, 18,  9]), 3 \n",
      "dric=>. == tensor([ 4, 18,  9,  3]), 0 \n",
      "....=>l == tensor([0, 0, 0, 0]), 12 \n",
      "...l=>e == tensor([ 0,  0,  0, 12]), 5 \n",
      "..le=>n == tensor([ 0,  0, 12,  5]), 14 \n",
      ".len=>i == tensor([ 0, 12,  5, 14]), 9 \n",
      "--random samples--\n",
      "elyn=>n == tensor([ 5, 12, 25, 14]), 14 \n",
      "ylin=>. == tensor([25, 12,  9, 14]), 0 \n",
      "onan=>. == tensor([15, 14,  1, 14]), 0 \n",
      "racy=>n == tensor([18,  1,  3, 25]), 14 \n",
      "....=>t == tensor([0, 0, 0, 0]), 20 \n",
      "...a=>b == tensor([0, 0, 0, 1]), 2 \n",
      "....=>a == tensor([0, 0, 0, 0]), 1 \n",
      "...a=>r == tensor([0, 0, 0, 1]), 18 \n",
      "fend=>e == tensor([ 6,  5, 14,  4]), 5 \n",
      "....=>j == tensor([0, 0, 0, 0]), 10 \n",
      "vyol=>e == tensor([22, 25, 15, 12]), 5 \n",
      "....=>r == tensor([0, 0, 0, 0]), 18 \n",
      "lynn=>. == tensor([12, 25, 14, 14]), 0 \n",
      "pran=>i == tensor([16, 18,  1, 14]), 9 \n",
      "...p=>a == tensor([ 0,  0,  0, 16]), 1 \n",
      "..sa=>a == tensor([ 0,  0, 19,  1]), 1 \n",
      "...b=>o == tensor([0, 0, 0, 2]), 15 \n",
      "..ar=>p == tensor([ 0,  0,  1, 18]), 16 \n",
      "..ta=>r == tensor([ 0,  0, 20,  1]), 18 \n",
      ".atr=>e == tensor([ 0,  1, 20, 18]), 5 \n"
     ]
    }
   ],
   "source": [
    "def debug_samples(X, Y):\n",
    "    for inp, out in list(zip(X, Y))[25:50]:\n",
    "        print(f\"{to_word(inp)}=>{index_to_string(out.item())} == {inp}, {out} \")\n",
    "    print(\"--random samples--\")\n",
    "    for inp, out in random.sample(list(zip(X, Y)), 20):\n",
    "        print(f\"{to_word(inp)}=>{index_to_string(out.item())} == {inp}, {out} \")\n",
    "\n",
    "\n",
    "debug_samples(X, Y)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "682c2c25-3cd9-44ff-93c4-d951c3d26dca",
   "metadata": {
    "tags": []
   },
   "source": [
    "# Our arhcitecture\n",
    "\n",
    "```mermaid\n",
    "sequenceDiagram\n",
    "actor X as input<br>(letters)\n",
    "participant C as C=lookup embedding<br>(27x2)\n",
    "\n",
    "X->>C: 1 shot encoding on input charectors\n",
    "\n",
    "```\n",
    "\n",
    "Tree\n",
    "\n",
    "```mermaid\n",
    "%%{init: {\"flowchart\": {\"htmlLabels\": false}} }%%\n",
    "\n",
    "flowchart TD\n",
    "    I0[\"in[0]\"]\n",
    "    Ii[\"in[i]\"]\n",
    "    In[\"in[n]\"]\n",
    "    I0 --> C_In_0\n",
    "    Ii --> C_In_i\n",
    "    In --> C_In_n\n",
    "    C_In_0 --> N\n",
    "    C_In_i --> N\n",
    "    C_In_n --> N\n",
    "    subgraph Neuron\n",
    "    N --> SoftMax\n",
    "    end\n",
    "    SoftMax-->output\n",
    "```\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f7496a55-4e83-454f-810f-9c587d44a53c",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}