From 4ab96ce7a9841297e0f10de6512f0a410b12d295 Mon Sep 17 00:00:00 2001 From: Nipun Batra Date: Thu, 21 Dec 2023 14:37:54 +0530 Subject: [PATCH] added some initial comments --- posts/2023-12-21-towards-transformers.ipynb | 734 ++++++++++++++++++++ 1 file changed, 734 insertions(+) create mode 100644 posts/2023-12-21-towards-transformers.ipynb diff --git a/posts/2023-12-21-towards-transformers.ipynb b/posts/2023-12-21-towards-transformers.ipynb new file mode 100644 index 0000000..aac1707 --- /dev/null +++ b/posts/2023-12-21-towards-transformers.ipynb @@ -0,0 +1,734 @@ +{ + "cells": [ + { + "cell_type": "raw", + "id": "12a1fd37", + "metadata": {}, + "source": [ + "---\n", + "author: Nipun Batra\n", + "badges: true\n", + "categories:\n", + "- ML\n", + "date: '2023-12-21'\n", + "output-file: towards-transformers.html\n", + "title: Towards transformers\n", + "toc: true\n", + "\n", + "---\n", + "\n" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "id": "c576bebd", + "metadata": {}, + "source": [ + "### Basic Imports" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "8c21a02c", + "metadata": {}, + "outputs": [], + "source": [ + "import tiktoken" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "7b43cb10", + "metadata": {}, + "outputs": [], + "source": [ + "encoding = tiktoken.get_encoding(\"cl100k_base\")\n" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "9a0f13cd", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[9906, 4435, 0, 1115, 374, 264, 4382, 38266]" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "encoding.encode(\"Hello World! This is a simple notebook\")\n" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "c40ba132", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "'Hello World! This'" + ] + }, + "execution_count": 14, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "encoding.decode([9906, 4435, 0, 1115])" + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "id": "5ea21b34", + "metadata": {}, + "outputs": [], + "source": [ + "ser = {}\n", + "n =20\n", + "for i in range(n**2):\n", + " ser[i] = encoding.decode([i])" + ] + }, + { + "cell_type": "code", + "execution_count": 35, + "id": "fee884bf", + "metadata": {}, + "outputs": [], + "source": [ + "import pandas as pd" + ] + }, + { + "cell_type": "code", + "execution_count": 36, + "id": "d96ea3c4", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
012345678910111213141516171819
0!\"#$%&'()*+,-./01234
156789:;<=>?@ABCDEFGH
2IJKLMNOPQRSTUVWXYZ[\\
3]^_`abcdefghijklmnop
4qrstuvwxyz{|}~������
5��������������������
6��������������������
7��������������������
8��������������������
9��������\u0000\u0001\u0002\u0003\u0004\u0005\u0006\u0007\b\\t\\n
10\\r\u000e\u000f\u0010\u0011\u0012\u0013\u0014\u0015\u0016\u0017\u0018\u0019\u001a\u001b
11������������������
12����������������int
13eronareatstenorth\\n\\nclesitanaralthe
14;\\npfou=isingeswionedicbdetmo\\t\\tro
15aselctndinhentidnamtore--{ofom);\\nim\\r\\n
16(il//andurselexSad\"chutif**}emolth
17)\\n{\\ngigiv,\\nceodvateTagay*otusCstIun
18ulueAow'ew<ation()forabortumameispetrck�y
19ist----.\\n\\nheeloMbeersonconapubPassint>\\nlyurn
\n", + "
" + ], + "text/plain": [ + " 0 1 2 3 4 5 6 7 8 9 \\\n", + "0 ! \" # $ % & ' ( ) * \n", + "1 5 6 7 8 9 : ; < = > \n", + "2 I J K L M N O P Q R \n", + "3 ] ^ _ ` a b c d e f \n", + "4 q r s t u v w x y z \n", + "5 � � � � � � � � � � \n", + "6 � � � � � � � � � � \n", + "7 � � � � � � � � � � \n", + "8 � � � � � � � � � � \n", + "9 � � � � � � � � \u0000 \u0001 \n", + "10 \n", + " \\r \u000e \u000f \u0010 \u0011 \u0012 \u0013 \u0014 \u0015 \n", + "11  � � � � � � � � \n", + "12 � � � � � � � � � � \n", + "13 er on a re at st en or \n", + "14 ;\\n p f ou = is ing es w \n", + "15 as el ct nd in h ent id n am \n", + "16 ( il // and ur se l ex S ad \n", + "17 )\\n {\\n g ig iv ,\\n ce od v ate \n", + "18 ul ue A ow ' ew < ation () for \n", + "19 ist ---- .\\n\\n he e lo M be ers on \n", + "\n", + " 10 11 12 13 14 15 16 17 \\\n", + "0 + , - . / 0 1 2 \n", + "1 ? @ A B C D E F \n", + "2 S T U V W X Y Z \n", + "3 g h i j k l m n \n", + "4 { | } ~ � � � � \n", + "5 � � � � � � � � \n", + "6 � � � � � � � � \n", + "7 � � � � � � � � \n", + "8 � � � � � � � � \n", + "9 \u0002 \u0003 \u0004 \u0005 \u0006 \u0007 \b \\t \n", + "10 \u0016 \u0017 \u0018 \u0019 \u001a \u001b \n", + " \n", + " \n", + "11 � � � � � � � � \n", + "12 � � � � � � \n", + "13 th \\n\\n c le s it an ar \n", + "14 ion ed ic b d et m o \n", + "15 to re -- { of om );\\n \n", + "16 \" ch ut if ** } em ol \n", + "17 T ag ay * ot us C st \n", + "18 ab ort um ame is pe tr ck \n", + "19 con ap ub P ass int >\\n \n", + "\n", + " 18 19 \n", + "0 3 4 \n", + "1 G H \n", + "2 [ \\ \n", + "3 o p \n", + "4 � � \n", + "5 � � \n", + "6 � � \n", + "7 � � \n", + "8 � � \n", + "9 \\n \n", + " \n", + "10 \n", + " \u001f \n", + "11 � � \n", + "12 in t \n", + "13 al the \n", + "14 \\t\\t ro \n", + "15 im \\r\\n \n", + "16 th \n", + "17 I un \n", + "18 � y \n", + "19 ly urn " + ] + }, + "execution_count": 36, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "pd.DataFrame(pd.Series(ser).values.reshape(n,n))" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.7" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +}