d2l_learning.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "https://chat.openai.com/share/ca177a86-984f-4193-80a4-e2b8a69b62b3"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "python -m pip install --pre torch torchvision torchaudio --index-url https://download.pytorch.org/whl/nightly/cu121"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import collections\n",
    "import hashlib\n",
    "import math\n",
    "import os\n",
    "import random\n",
    "import re\n",
    "import shutil\n",
    "import sys\n",
    "import tarfile\n",
    "import time\n",
    "import zipfile\n",
    "from collections import defaultdict\n",
    "import pandas as pd\n",
    "import requests\n",
    "from IPython import display\n",
    "from matplotlib import pyplot as plt\n",
    "from matplotlib_inline import backend_inline\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.cuda.is_available()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "x = torch.tensor([[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "input_tensor = torch.tensor([[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]])\n",
    "\n",
    "# 计算每个样本的平均值和方差\n",
    "mean = input_tensor.mean(dim=1, keepdim=True)\n",
    "variance = input_tensor.var(dim=1, keepdim=True, unbiased=False)\n",
    "\n",
    "# 定义 epsilon 防止除零错误\n",
    "epsilon = 1e-5\n",
    "\n",
    "# 应用层归一化公式\n",
    "normalized_tensor = (input_tensor - mean) / torch.sqrt(variance + epsilon)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[-1.2247,  0.0000,  1.2247],\n",
       "        [-1.2247,  0.0000,  1.2247]])"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "normalized_tensor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "原始张量:\n",
      "tensor([[1.0000, 2.0000, 3.0000],\n",
      "        [4.0000, 5.0000, 6.0000],\n",
      "        [7.8000, 8.0000, 9.0000]])\n",
      "经过层归一化后的张量:\n",
      "tensor([[-1.2247,  0.0000,  1.2247],\n",
      "        [-1.2247,  0.0000,  1.2247],\n",
      "        [-0.8890, -0.5080,  1.3970]], grad_fn=<NativeLayerNormBackward0>)\n"
     ]
    }
   ],
   "source": [
    "# 创建一个简单的张量\n",
    "input_tensor = torch.tensor([[1.0, 2.0, 3.0], [4.0, 5.0, 6.0], [7.8, 8.0, 9.0]])\n",
    "\n",
    "# 定义层归一化层\n",
    "# 假设我们的特征维度是3（因为我们的输入有3个特征）\n",
    "layer_norm = nn.LayerNorm(input_tensor.size()[1])\n",
    "\n",
    "# 应用层归一化\n",
    "output_tensor = layer_norm(input_tensor)\n",
    "\n",
    "print(\"原始张量:\")\n",
    "print(input_tensor)\n",
    "print(\"经过层归一化后的张量:\")\n",
    "print(output_tensor)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "from minGPT.mingpt.utils import CfgNode as CN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "class CausalSelfAttention(nn.Module):\n",
    "    \"\"\"\n",
    "    A vanilla multi-head masked self-attention layer with a projection at the end.\n",
    "    It is possible to use torch.nn.MultiheadAttention here but I am including an\n",
    "    explicit implementation here to show that there is nothing too scary here.\n",
    "    \"\"\"\n",
    "\n",
    "    def __init__(self, config):\n",
    "        super().__init__()\n",
    "        assert config.n_embd % config.n_head == 0\n",
    "        # key, query, value projections for all heads, but in a batch\n",
    "        self.c_attn = nn.Linear(config.n_embd, 3 * config.n_embd)\n",
    "        # output projection\n",
    "        self.c_proj = nn.Linear(config.n_embd, config.n_embd)\n",
    "        # regularization\n",
    "        self.attn_dropout = nn.Dropout(config.attn_pdrop)\n",
    "        self.resid_dropout = nn.Dropout(config.resid_pdrop)\n",
    "        # causal mask to ensure that attention is only applied to the left in the input sequence\n",
    "        self.register_buffer(\"bias\", torch.tril(torch.ones(config.block_size, config.block_size))\n",
    "                                     .view(1, 1, config.block_size, config.block_size))\n",
    "        self.n_head = config.n_head\n",
    "        self.n_embd = config.n_embd\n",
    "\n",
    "    def forward(self, x):\n",
    "        B, T, C = x.size() # batch size, sequence length, embedding dimensionality (n_embd)\n",
    "\n",
    "        # calculate query, key, values for all heads in batch and move head forward to be the batch dim\n",
    "        q, k ,v  = self.c_attn(x).split(self.n_embd, dim=2)\n",
    "        k = k.view(B, T, self.n_head, C // self.n_head).transpose(1, 2) # (B, nh, T, hs)\n",
    "        q = q.view(B, T, self.n_head, C // self.n_head).transpose(1, 2) # (B, nh, T, hs)\n",
    "        v = v.view(B, T, self.n_head, C // self.n_head).transpose(1, 2) # (B, nh, T, hs)\n",
    "\n",
    "        # causal self-attention; Self-attend: (B, nh, T, hs) x (B, nh, hs, T) -> (B, nh, T, T)\n",
    "        att = (q @ k.transpose(-2, -1)) * (1.0 / math.sqrt(k.size(-1)))\n",
    "        att = att.masked_fill(self.bias[:,:,:T,:T] == 0, float('-inf'))\n",
    "        att = F.softmax(att, dim=-1)\n",
    "        att = self.attn_dropout(att)\n",
    "        y = att @ v # (B, nh, T, T) x (B, nh, T, hs) -> (B, nh, T, hs)\n",
    "        y = y.transpose(1, 2).contiguous().view(B, T, C) # re-assemble all head outputs side by side\n",
    "\n",
    "        # output projection\n",
    "        y = self.resid_dropout(self.c_proj(y))\n",
    "        return y\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "class GPT(nn.Module):\n",
    "    \"\"\" GPT Language Model \"\"\"\n",
    "\n",
    "    @staticmethod\n",
    "    def get_default_config():\n",
    "        C = CN()\n",
    "        # either model_type or (n_layer, n_head, n_embd) must be given in the config\n",
    "        C.model_type = 'gpt'\n",
    "        C.n_layer = None\n",
    "        C.n_head = None\n",
    "        C.n_embd =  None\n",
    "        # these options must be filled in externally\n",
    "        C.vocab_size = None\n",
    "        C.block_size = None\n",
    "        # dropout hyperparameters\n",
    "        C.embd_pdrop = 0.1\n",
    "        C.resid_pdrop = 0.1\n",
    "        C.attn_pdrop = 0.1\n",
    "        return C\n",
    "\n",
    "    def __init__(self, config):\n",
    "        super().__init__()\n",
    "        assert config.vocab_size is not None\n",
    "        assert config.block_size is not None\n",
    "        self.block_size = config.block_size\n",
    "\n",
    "        type_given = config.model_type is not None\n",
    "        params_given = all([config.n_layer is not None, config.n_head is not None, config.n_embd is not None])\n",
    "        assert type_given ^ params_given # exactly one of these (XOR)\n",
    "        if type_given:\n",
    "            # translate from model_type to detailed configuration\n",
    "            config.merge_from_dict({\n",
    "                # names follow the huggingface naming conventions\n",
    "                # GPT-1\n",
    "                'openai-gpt':   dict(n_layer=12, n_head=12, n_embd=768),  # 117M params\n",
    "                # GPT-2 configs\n",
    "                'gpt2':         dict(n_layer=12, n_head=12, n_embd=768),  # 124M params\n",
    "                'gpt2-medium':  dict(n_layer=24, n_head=16, n_embd=1024), # 350M params\n",
    "                'gpt2-large':   dict(n_layer=36, n_head=20, n_embd=1280), # 774M params\n",
    "                'gpt2-xl':      dict(n_layer=48, n_head=25, n_embd=1600), # 1558M params\n",
    "                # Gophers\n",
    "                'gopher-44m':   dict(n_layer=8, n_head=16, n_embd=512),\n",
    "                # (there are a number more...)\n",
    "                # I made these tiny models up\n",
    "                'gpt-mini':     dict(n_layer=6, n_head=6, n_embd=192),\n",
    "                'gpt-micro':    dict(n_layer=4, n_head=4, n_embd=128),\n",
    "                'gpt-nano':     dict(n_layer=3, n_head=3, n_embd=48),\n",
    "            }[config.model_type])\n",
    "\n",
    "        self.transformer = nn.ModuleDict(dict(\n",
    "            wte = nn.Embedding(config.vocab_size, config.n_embd),\n",
    "            wpe = nn.Embedding(config.block_size, config.n_embd),\n",
    "            drop = nn.Dropout(config.embd_pdrop),\n",
    "            h = nn.ModuleList([Block(config) for _ in range(config.n_layer)]),\n",
    "            ln_f = nn.LayerNorm(config.n_embd),\n",
    "        ))\n",
    "        self.lm_head = nn.Linear(config.n_embd, config.vocab_size, bias=False)\n",
    "\n",
    "        # init all weights, and apply a special scaled init to the residual projections, per GPT-2 paper\n",
    "        self.apply(self._init_weights)\n",
    "        for pn, p in self.named_parameters():\n",
    "            if pn.endswith('c_proj.weight'):\n",
    "                torch.nn.init.normal_(p, mean=0.0, std=0.02/math.sqrt(2 * config.n_layer))\n",
    "\n",
    "        # report number of parameters (note we don't count the decoder parameters in lm_head)\n",
    "        n_params = sum(p.numel() for p in self.transformer.parameters())\n",
    "        print(\"number of parameters: %.2fM\" % (n_params/1e6,))\n",
    "\n",
    "    def _init_weights(self, module):\n",
    "        if isinstance(module, nn.Linear):\n",
    "            torch.nn.init.normal_(module.weight, mean=0.0, std=0.02)\n",
    "            if module.bias is not None:\n",
    "                torch.nn.init.zeros_(module.bias)\n",
    "        elif isinstance(module, nn.Embedding):\n",
    "            torch.nn.init.normal_(module.weight, mean=0.0, std=0.02)\n",
    "        elif isinstance(module, nn.LayerNorm):\n",
    "            torch.nn.init.zeros_(module.bias)\n",
    "            torch.nn.init.ones_(module.weight)\n",
    "\n",
    "    @classmethod\n",
    "    def from_pretrained(cls, model_type):\n",
    "        \"\"\"\n",
    "        Initialize a pretrained GPT model by copying over the weights\n",
    "        from a huggingface/transformers checkpoint.\n",
    "        \"\"\"\n",
    "        assert model_type in {'gpt2', 'gpt2-medium', 'gpt2-large', 'gpt2-xl'}\n",
    "        from transformers import GPT2LMHeadModel\n",
    "\n",
    "        # create a from-scratch initialized minGPT model\n",
    "        config = cls.get_default_config()\n",
    "        config.model_type = model_type\n",
    "        config.vocab_size = 50257 # openai's model vocabulary\n",
    "        config.block_size = 1024  # openai's model block_size\n",
    "        model = GPT(config)\n",
    "        sd = model.state_dict()\n",
    "\n",
    "        # init a huggingface/transformers model\n",
    "        model_hf = GPT2LMHeadModel.from_pretrained(model_type)\n",
    "        sd_hf = model_hf.state_dict()\n",
    "\n",
    "        # copy while ensuring all of the parameters are aligned and match in names and shapes\n",
    "        keys = [k for k in sd_hf if not k.endswith('attn.masked_bias')] # ignore these\n",
    "        transposed = ['attn.c_attn.weight', 'attn.c_proj.weight', 'mlp.c_fc.weight', 'mlp.c_proj.weight']\n",
    "        # basically the openai checkpoints use a \"Conv1D\" module, but we only want to use a vanilla nn.Linear.\n",
    "        # this means that we have to transpose these weights when we import them\n",
    "        assert len(keys) == len(sd)\n",
    "        for k in keys:\n",
    "            if any(k.endswith(w) for w in transposed):\n",
    "                # special treatment for the Conv1D weights we need to transpose\n",
    "                assert sd_hf[k].shape[::-1] == sd[k].shape\n",
    "                with torch.no_grad():\n",
    "                    sd[k].copy_(sd_hf[k].t())\n",
    "            else:\n",
    "                # vanilla copy over the other parameters\n",
    "                assert sd_hf[k].shape == sd[k].shape\n",
    "                with torch.no_grad():\n",
    "                    sd[k].copy_(sd_hf[k])\n",
    "\n",
    "        return model\n",
    "\n",
    "    def configure_optimizers(self, train_config):\n",
    "        \"\"\"\n",
    "        This long function is unfortunately doing something very simple and is being very defensive:\n",
    "        We are separating out all parameters of the model into two buckets: those that will experience\n",
    "        weight decay for regularization and those that won't (biases, and layernorm/embedding weights).\n",
    "        We are then returning the PyTorch optimizer object.\n",
    "        \"\"\"\n",
    "\n",
    "        # separate out all parameters to those that will and won't experience regularizing weight decay\n",
    "        decay = set()\n",
    "        no_decay = set()\n",
    "        whitelist_weight_modules = (torch.nn.Linear, )\n",
    "        blacklist_weight_modules = (torch.nn.LayerNorm, torch.nn.Embedding)\n",
    "        for mn, m in self.named_modules():\n",
    "            for pn, p in m.named_parameters():\n",
    "                fpn = '%s.%s' % (mn, pn) if mn else pn # full param name\n",
    "                # random note: because named_modules and named_parameters are recursive\n",
    "                # we will see the same tensors p many many times. but doing it this way\n",
    "                # allows us to know which parent module any tensor p belongs to...\n",
    "                if pn.endswith('bias'):\n",
    "                    # all biases will not be decayed\n",
    "                    no_decay.add(fpn)\n",
    "                elif pn.endswith('weight') and isinstance(m, whitelist_weight_modules):\n",
    "                    # weights of whitelist modules will be weight decayed\n",
    "                    decay.add(fpn)\n",
    "                elif pn.endswith('weight') and isinstance(m, blacklist_weight_modules):\n",
    "                    # weights of blacklist modules will NOT be weight decayed\n",
    "                    no_decay.add(fpn)\n",
    "\n",
    "        # validate that we considered every parameter\n",
    "        param_dict = {pn: p for pn, p in self.named_parameters()}\n",
    "        inter_params = decay & no_decay\n",
    "        union_params = decay | no_decay\n",
    "        assert len(inter_params) == 0, \"parameters %s made it into both decay/no_decay sets!\" % (str(inter_params), )\n",
    "        assert len(param_dict.keys() - union_params) == 0, \"parameters %s were not separated into either decay/no_decay set!\" \\\n",
    "                                                    % (str(param_dict.keys() - union_params), )\n",
    "\n",
    "        # create the pytorch optimizer object\n",
    "        optim_groups = [\n",
    "            {\"params\": [param_dict[pn] for pn in sorted(list(decay))], \"weight_decay\": train_config.weight_decay},\n",
    "            {\"params\": [param_dict[pn] for pn in sorted(list(no_decay))], \"weight_decay\": 0.0},\n",
    "        ]\n",
    "        optimizer = torch.optim.AdamW(optim_groups, lr=train_config.learning_rate, betas=train_config.betas)\n",
    "        return optimizer\n",
    "\n",
    "    def forward(self, idx, targets=None):\n",
    "        device = idx.device\n",
    "        b, t = idx.size()\n",
    "        assert t <= self.block_size, f\"Cannot forward sequence of length {t}, block size is only {self.block_size}\"\n",
    "        pos = torch.arange(0, t, dtype=torch.long, device=device).unsqueeze(0) # shape (1, t)\n",
    "\n",
    "        # forward the GPT model itself\n",
    "        tok_emb = self.transformer.wte(idx) # token embeddings of shape (b, t, n_embd)\n",
    "        pos_emb = self.transformer.wpe(pos) # position embeddings of shape (1, t, n_embd)\n",
    "        x = self.transformer.drop(tok_emb + pos_emb)\n",
    "        for block in self.transformer.h:\n",
    "            x = block(x)\n",
    "        x = self.transformer.ln_f(x)\n",
    "        logits = self.lm_head(x)\n",
    "\n",
    "        # if we are given some desired targets also calculate the loss\n",
    "        loss = None\n",
    "        if targets is not None:\n",
    "            loss = F.cross_entropy(logits.view(-1, logits.size(-1)), targets.view(-1), ignore_index=-1)\n",
    "\n",
    "        return logits, loss\n",
    "\n",
    "    @torch.no_grad()\n",
    "    def generate(self, idx, max_new_tokens, temperature=1.0, do_sample=False, top_k=None):\n",
    "        \"\"\"\n",
    "        Take a conditioning sequence of indices idx (LongTensor of shape (b,t)) and complete\n",
    "        the sequence max_new_tokens times, feeding the predictions back into the model each time.\n",
    "        Most likely you'll want to make sure to be in model.eval() mode of operation for this.\n",
    "        \"\"\"\n",
    "        for _ in range(max_new_tokens):\n",
    "            # if the sequence context is growing too long we must crop it at block_size\n",
    "            idx_cond = idx if idx.size(1) <= self.block_size else idx[:, -self.block_size:]\n",
    "            # forward the model to get the logits for the index in the sequence\n",
    "            logits, _ = self(idx_cond)\n",
    "            # pluck the logits at the final step and scale by desired temperature\n",
    "            logits = logits[:, -1, :] / temperature\n",
    "            # optionally crop the logits to only the top k options\n",
    "            if top_k is not None:\n",
    "                v, _ = torch.topk(logits, top_k)\n",
    "                logits[logits < v[:, [-1]]] = -float('Inf')\n",
    "            # apply softmax to convert logits to (normalized) probabilities\n",
    "            probs = F.softmax(logits, dim=-1)\n",
    "            # either sample from the distribution or take the most likely element\n",
    "            if do_sample:\n",
    "                idx_next = torch.multinomial(probs, num_samples=1)\n",
    "            else:\n",
    "                _, idx_next = torch.topk(probs, k=1, dim=-1)\n",
    "            # append sampled index to the running sequence and continue\n",
    "            idx = torch.cat((idx, idx_next), dim=1)\n",
    "\n",
    "        return idx"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "config = GPT.get_default_config()\n",
    "config.model_type = 'gpt-nano'\n",
    "config.vocab_size = 3\n",
    "config.block_size = 11\n",
    "config.n_embd = 48\n",
    "config.n_head = 3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "attn = CausalSelfAttention(config)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'gpt'"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "config.model_type"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch.nn.functional as F\n",
    "\n",
    "input_tensor = torch.randn(1, 10, config.n_embd)\n",
    "\n",
    "# 通过 CausalSelfAttention 获取输出\n",
    "output_tensor = attn(input_tensor)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[[-4.5397e-02, -5.3180e-01,  1.0115e+00, -9.7004e-02, -0.0000e+00,\n",
       "           8.9517e-02,  0.0000e+00, -1.4924e-01, -4.0716e-01, -6.1810e-01,\n",
       "          -3.7041e-01,  3.6248e-02, -1.7717e-01,  9.4238e-01, -6.0726e-01,\n",
       "           1.4189e-01,  7.5382e-02,  0.0000e+00, -4.3820e-01,  2.8809e-01,\n",
       "           6.3346e-02,  1.0342e-01,  8.3648e-01, -1.3487e-01, -3.6065e-02,\n",
       "          -3.6872e-01,  2.6944e-01, -8.6270e-02, -7.2617e-01,  3.1154e-01,\n",
       "          -4.6656e-01,  3.2325e-01, -3.8224e-01, -1.0432e+00,  2.8136e-02,\n",
       "           2.2506e-01, -2.4952e-01,  1.3780e-01,  2.5083e-01, -2.5613e-01,\n",
       "           5.2677e-01,  1.4351e-01,  8.5896e-02,  8.5359e-02,  6.2343e-01,\n",
       "           5.9525e-01, -1.1253e+00, -1.3406e+00],\n",
       "         [-6.1220e-01, -3.9249e-01,  6.7203e-01, -2.7848e-02, -0.0000e+00,\n",
       "          -1.9633e-01,  2.7571e-01,  2.4278e-01, -7.7353e-01, -3.4798e-01,\n",
       "           3.9674e-01, -1.1360e-01, -1.0206e-01,  4.3668e-01,  1.9027e-01,\n",
       "           2.5527e-01, -0.0000e+00,  4.4270e-01, -5.5822e-01,  3.0087e-01,\n",
       "          -0.0000e+00,  9.4634e-02,  2.8913e-01,  1.5082e-01,  2.6064e-01,\n",
       "           7.7332e-02,  3.8149e-02,  8.9360e-02, -3.8837e-01, -8.0226e-03,\n",
       "          -3.5700e-01,  3.6478e-01, -6.1415e-01, -7.5760e-01, -1.1040e-01,\n",
       "          -8.8849e-02, -1.6653e-01,  2.3891e-02,  3.9512e-01, -0.0000e+00,\n",
       "           4.4439e-01, -8.3242e-02,  2.2314e-01,  9.1955e-02,  2.5205e-01,\n",
       "           2.2067e-01, -5.8332e-01, -1.0980e+00],\n",
       "         [-3.8488e-01, -3.2684e-01,  2.1912e-01, -1.1540e-01,  0.0000e+00,\n",
       "          -2.0820e-01,  3.0995e-01,  3.9844e-01, -5.3082e-01, -1.7244e-01,\n",
       "           5.0194e-01, -1.4091e-01,  1.0112e-01,  2.1120e-01,  5.7983e-02,\n",
       "           5.8660e-02,  2.1678e-02,  3.5085e-01, -3.1743e-01,  7.1281e-03,\n",
       "          -4.1265e-02, -2.0980e-02,  5.5583e-02,  0.0000e+00,  2.6266e-01,\n",
       "           1.9248e-01,  3.4318e-02, -1.3102e-01, -5.7373e-02, -1.4020e-01,\n",
       "          -0.0000e+00,  0.0000e+00, -3.4166e-01, -5.0512e-01, -3.1899e-03,\n",
       "          -0.0000e+00,  3.0000e-02, -2.1373e-02,  2.4907e-01, -5.9518e-02,\n",
       "           5.0456e-01, -1.1055e-01,  7.8027e-02,  3.5496e-02,  1.2346e-01,\n",
       "           1.9694e-01, -1.8067e-01, -6.8091e-01],\n",
       "         [-0.0000e+00, -4.3444e-01,  2.3484e-01, -1.0882e-01, -2.9105e-02,\n",
       "          -1.4878e-01,  3.8063e-01,  0.0000e+00, -3.3498e-01, -2.4761e-01,\n",
       "           2.9600e-01, -2.6432e-01,  1.3049e-01,  3.8613e-01, -3.7084e-02,\n",
       "          -9.4654e-02,  1.6748e-02,  3.9814e-01, -2.5128e-01,  1.2454e-02,\n",
       "          -0.0000e+00,  1.1845e-01,  1.2501e-01,  5.6887e-02,  4.6162e-01,\n",
       "           1.8095e-01,  8.6931e-02,  4.8516e-02, -1.5873e-01, -7.9823e-02,\n",
       "          -3.3894e-01,  2.4036e-01, -4.4252e-01, -4.2578e-01, -1.4687e-01,\n",
       "          -3.6790e-02,  8.0902e-02,  5.1001e-02,  3.4513e-01, -3.1802e-01,\n",
       "           4.0948e-01, -1.4886e-01, -2.4792e-02, -1.3626e-02,  2.7694e-01,\n",
       "           2.0750e-01, -0.0000e+00, -8.2572e-01],\n",
       "         [-2.4831e-01, -2.7892e-01,  2.3956e-01,  4.7967e-02, -8.3553e-02,\n",
       "          -0.0000e+00,  1.1379e-01,  0.0000e+00, -2.4154e-01, -2.9144e-01,\n",
       "           2.0849e-01, -1.6326e-02, -2.1853e-02,  3.0058e-01, -0.0000e+00,\n",
       "           5.1987e-02,  6.8024e-04, -2.5565e-02, -1.9010e-01,  4.5910e-02,\n",
       "          -0.0000e+00, -1.5914e-01,  2.3150e-01,  9.8600e-02,  4.8514e-01,\n",
       "           0.0000e+00,  7.4410e-02,  1.1450e-01, -1.0815e-01, -2.6611e-01,\n",
       "          -1.3632e-01,  0.0000e+00, -2.1105e-01, -5.5595e-01, -1.3142e-01,\n",
       "          -1.5836e-02,  1.0889e-01, -1.2139e-01,  1.2301e-01, -2.1962e-01,\n",
       "           3.4866e-01,  0.0000e+00, -7.8917e-02, -9.8205e-02, -5.1338e-03,\n",
       "           2.3858e-01, -6.9944e-02, -6.6610e-01],\n",
       "         [-1.8104e-01, -5.0641e-01, -2.6277e-02,  4.9235e-03,  1.0552e-01,\n",
       "          -1.1605e-02,  3.4256e-01,  3.9390e-01, -3.7869e-01, -1.1453e-03,\n",
       "           3.8693e-01, -2.5318e-01,  9.0518e-02,  3.0900e-01, -1.6309e-02,\n",
       "          -1.2560e-01,  4.2420e-03,  9.2856e-02, -0.0000e+00, -6.2535e-02,\n",
       "          -1.0138e-01, -8.6558e-02,  8.9269e-02,  3.6842e-02,  7.0265e-01,\n",
       "           0.0000e+00,  9.7545e-02, -8.4013e-04,  8.7708e-02, -2.0180e-01,\n",
       "          -1.5331e-01,  2.1663e-01, -2.0646e-01, -4.4138e-01, -1.0615e-01,\n",
       "           2.3368e-02,  1.7216e-01, -1.5805e-01,  1.0354e-01, -3.2537e-01,\n",
       "           3.7765e-01, -6.9477e-02, -1.0636e-01, -1.0570e-01,  1.8353e-01,\n",
       "           2.0322e-01,  7.0181e-02, -6.8746e-01],\n",
       "         [ 9.5016e-03, -0.0000e+00,  0.0000e+00, -0.0000e+00, -1.4589e-01,\n",
       "          -2.1536e-01,  1.8569e-01,  4.3101e-01, -3.6593e-01, -1.8492e-01,\n",
       "           2.5835e-01,  6.9768e-04,  4.7452e-02,  3.5466e-01, -2.1794e-01,\n",
       "          -0.0000e+00,  6.5707e-02,  1.5579e-01, -1.1428e-01,  5.4263e-02,\n",
       "          -1.4897e-01,  1.5929e-02,  1.6581e-01, -3.3457e-03,  4.7846e-01,\n",
       "          -2.4010e-03,  1.2453e-01, -1.9778e-02, -3.1477e-02, -8.0348e-02,\n",
       "          -2.2942e-01,  0.0000e+00, -3.6826e-01, -3.8366e-01, -4.3384e-02,\n",
       "           1.5689e-03,  4.0334e-02, -1.6838e-01,  1.5448e-01, -3.7062e-01,\n",
       "           2.2243e-01,  1.2068e-02, -6.6247e-02,  6.6588e-02,  3.6845e-01,\n",
       "           2.2947e-01, -2.3366e-01, -5.9571e-01],\n",
       "         [-2.6978e-01, -3.0347e-01,  1.1017e-02, -7.8706e-02, -7.3509e-02,\n",
       "          -1.1676e-01,  1.5923e-01,  3.5323e-01, -3.5712e-01, -7.9637e-02,\n",
       "           0.0000e+00, -8.0780e-02, -3.7666e-02,  2.5747e-01,  5.2313e-02,\n",
       "           4.6670e-02,  7.9725e-02,  1.5254e-01, -1.6127e-01, -6.6775e-02,\n",
       "          -1.2567e-01,  4.7924e-03,  0.0000e+00,  9.7295e-02,  5.3097e-01,\n",
       "           2.6835e-01, -1.4080e-01,  2.5898e-02,  6.3532e-02,  7.2195e-02,\n",
       "          -0.0000e+00,  3.1926e-02,  2.7379e-02, -2.4840e-01, -0.0000e+00,\n",
       "          -6.9993e-02, -0.0000e+00, -1.3524e-01, -4.7648e-02, -2.2588e-01,\n",
       "           8.3314e-02, -7.1992e-02, -1.6082e-01, -0.0000e+00,  2.8795e-01,\n",
       "           1.2070e-01,  2.9527e-02, -5.3445e-01],\n",
       "         [-1.4931e-01, -2.9188e-01,  2.2657e-01, -0.0000e+00, -4.9461e-02,\n",
       "          -6.4083e-02,  0.0000e+00,  2.5275e-01, -3.8373e-01, -0.0000e+00,\n",
       "           0.0000e+00, -3.6900e-02, -0.0000e+00,  3.6297e-01, -4.3359e-02,\n",
       "           1.7201e-02,  8.1981e-02,  6.7941e-02, -0.0000e+00, -2.6920e-02,\n",
       "          -0.0000e+00, -0.0000e+00,  1.7150e-01,  7.4059e-02,  4.8336e-01,\n",
       "           1.1102e-01,  4.0637e-02, -0.0000e+00,  9.4615e-03, -3.3554e-02,\n",
       "          -2.0208e-01,  9.1798e-02, -5.9825e-02, -4.2943e-01, -1.4634e-01,\n",
       "          -1.3892e-02, -1.0100e-01,  1.7786e-02, -0.0000e+00, -2.4876e-01,\n",
       "           1.8009e-01,  7.0948e-02, -1.2800e-01, -0.0000e+00,  1.7718e-01,\n",
       "           0.0000e+00, -2.1859e-01, -6.2701e-01],\n",
       "         [-4.2690e-03, -2.2568e-01,  1.2972e-01, -1.9811e-01, -7.0844e-02,\n",
       "          -0.0000e+00,  1.5389e-01,  3.0752e-01, -1.6950e-01, -1.5626e-01,\n",
       "           1.9447e-01,  1.6650e-02,  1.6449e-01,  0.0000e+00, -0.0000e+00,\n",
       "          -5.4387e-02,  1.5075e-01,  1.7713e-01, -1.4163e-01, -2.8235e-02,\n",
       "          -4.9231e-02, -1.0525e-01,  1.6442e-01,  2.0680e-03,  4.9996e-01,\n",
       "           2.5367e-01,  1.5797e-02, -0.0000e+00,  7.9753e-02,  1.0215e-01,\n",
       "          -1.4967e-01, -6.6354e-03, -3.5351e-02, -2.7417e-01, -2.2432e-01,\n",
       "           5.2392e-03, -0.0000e+00, -7.1060e-02, -0.0000e+00, -2.5173e-01,\n",
       "           0.0000e+00, -2.2308e-02, -2.4559e-01, -1.0642e-02,  0.0000e+00,\n",
       "           3.0486e-01, -1.3706e-01, -5.0340e-01]]], grad_fn=<MulBackward0>)"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "output_tensor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[[-1.1434e+00,  8.5085e-01, -1.1020e-01, -4.1483e-02, -3.7499e-01,\n",
       "          -2.7098e-01, -2.0127e+00, -1.0807e+00, -2.5766e+00, -1.5124e+00,\n",
       "          -8.5082e-02, -2.0072e+00, -1.3589e+00,  7.1515e-01,  1.2694e+00,\n",
       "          -5.0828e-01,  3.2971e-01,  4.7442e-01,  1.7500e-01, -4.7310e-03,\n",
       "          -2.4749e-01, -1.6138e-01,  6.3162e-01,  2.0431e-01,  1.9941e-01,\n",
       "          -1.5073e+00, -7.6510e-01, -7.7724e-01, -9.5968e-02,  2.2442e+00,\n",
       "          -1.0663e+00,  1.0552e+00,  7.8778e-01,  3.9780e-01,  4.6046e-01,\n",
       "          -3.2837e-01,  1.4538e-01, -1.2118e+00, -1.6320e+00, -1.1839e+00,\n",
       "          -1.1512e+00, -3.9916e-01, -9.2560e-02,  9.2976e-01, -3.6033e-01,\n",
       "          -7.4420e-01,  2.7736e-01,  1.4367e+00],\n",
       "         [-1.4534e+00,  1.6543e+00, -8.8243e-01, -3.2915e-01,  3.4075e-01,\n",
       "           8.5306e-01, -9.4084e-01, -4.1994e-01,  6.8491e-02,  1.8351e-01,\n",
       "          -1.3080e+00,  8.6825e-02,  1.0545e+00, -3.8472e-01,  1.4881e-01,\n",
       "          -1.0184e+00,  1.5137e+00,  3.0889e-01,  5.3831e-01, -1.5614e+00,\n",
       "          -7.8515e-01,  1.6958e+00,  8.0554e-01,  1.1069e+00,  6.6714e-01,\n",
       "          -1.0801e+00, -9.2673e-01, -1.9656e+00,  1.3499e+00,  1.0012e+00,\n",
       "          -1.2812e+00, -1.6254e-01,  1.5929e-01,  7.8916e-01, -3.5783e-01,\n",
       "          -3.6528e-01,  7.2547e-01,  1.1862e+00,  3.7372e-01, -8.7888e-02,\n",
       "           1.7513e-01, -1.2960e-01, -1.1300e+00,  9.1096e-01,  9.5678e-01,\n",
       "          -8.7917e-01,  2.4772e+00,  8.4328e-01],\n",
       "         [-8.7361e-01,  4.0306e-01,  5.2471e-01,  3.7373e-01,  2.9216e-02,\n",
       "           1.2829e+00,  1.0796e+00, -3.0546e-01,  1.5253e+00,  1.8433e+00,\n",
       "          -3.0525e-01,  1.6778e+00, -2.7514e-01,  1.6879e+00, -5.4629e-01,\n",
       "          -1.1710e+00,  3.4183e-01, -6.6930e-01,  3.4736e-01,  4.6994e-01,\n",
       "          -4.8847e-01,  8.6471e-01,  3.7201e-01,  1.5366e+00,  1.6299e+00,\n",
       "          -1.4901e+00, -1.9117e-01, -9.2807e-01,  2.9711e-01,  1.0801e+00,\n",
       "           8.2508e-01,  5.4650e-01,  3.9237e-01,  1.4640e-02, -1.1098e+00,\n",
       "          -1.1655e+00, -5.2240e-01, -7.7694e-01, -1.0742e+00, -2.3852e-01,\n",
       "           1.5061e-01,  9.9854e-01, -9.2698e-02,  4.0248e-01,  4.5017e-02,\n",
       "           1.1444e+00, -8.6052e-01,  2.1536e+00],\n",
       "         [ 3.9498e-01,  1.1613e+00, -1.5373e+00, -5.4678e-01, -5.7828e-02,\n",
       "           1.3939e+00, -3.1711e-01,  6.8659e-01, -1.0385e+00, -8.0295e-01,\n",
       "           1.0725e+00,  2.8451e-01,  1.0745e-02,  3.5312e-01, -1.4314e+00,\n",
       "          -1.3363e+00,  1.0561e-01, -2.8341e-02, -1.6111e+00, -1.6195e+00,\n",
       "          -2.1762e+00,  5.9010e-01,  1.6446e+00,  1.2247e+00, -5.9051e-01,\n",
       "          -6.2419e-01,  1.5539e+00,  4.8451e-01,  3.2464e-01,  3.0261e-01,\n",
       "           2.6816e-02, -2.7960e-01, -6.3899e-01, -4.2294e-01, -3.3422e-02,\n",
       "           1.8919e+00, -4.1144e+00, -6.5074e-01,  3.0570e-01,  9.0090e-01,\n",
       "          -4.2457e-01,  2.7447e-01, -1.7000e+00,  2.3819e-01,  1.5980e-01,\n",
       "          -3.1805e-01, -1.4858e-01, -1.3560e+00],\n",
       "         [-9.8411e-02, -2.0102e-01,  1.6958e-01, -2.7152e-02,  2.5303e-02,\n",
       "          -4.0571e-01,  1.8623e+00, -2.8069e-01,  1.3529e+00, -2.4444e-01,\n",
       "           4.2503e-01,  8.7443e-01,  5.9614e-01,  1.1975e+00,  1.5065e-02,\n",
       "           4.7391e-01,  2.7016e-01,  9.6371e-01, -1.2558e+00, -3.3603e-01,\n",
       "           6.7874e-01,  6.8459e-01,  8.5965e-01,  1.4208e+00, -1.4414e+00,\n",
       "           1.9753e+00,  1.6753e-01, -5.3356e-01, -2.9623e-01, -2.8344e-01,\n",
       "          -4.3410e-01,  9.6698e-02, -3.2847e-01,  2.9730e+00,  1.8583e-02,\n",
       "          -1.3789e+00,  1.1741e+00, -7.9806e-02,  1.7671e+00,  7.7114e-02,\n",
       "          -1.5070e+00,  2.3046e-01, -5.3982e-01, -1.9726e-01,  7.2343e-01,\n",
       "           1.8310e+00,  9.0547e-01,  3.5836e-01],\n",
       "         [-2.5764e+00,  1.5270e-01, -1.2980e+00,  5.5908e-01, -1.1126e+00,\n",
       "          -9.6348e-01, -3.1478e-01,  7.3223e-01,  8.8727e-01, -4.5802e-02,\n",
       "           6.9842e-01, -1.3673e+00, -9.7420e-01,  8.5191e-01,  1.4166e+00,\n",
       "          -3.6877e-01,  1.1560e+00,  1.3871e+00,  2.1663e+00,  3.1883e-01,\n",
       "          -5.4797e-01,  1.1466e-01,  4.1294e-01, -9.9513e-01, -2.1201e-01,\n",
       "           1.5804e-01,  4.4674e-01, -1.0932e+00,  9.5539e-01, -7.9709e-01,\n",
       "          -2.3910e-01,  8.4866e-01, -3.2989e-01,  4.6623e-01,  1.7549e+00,\n",
       "          -1.7213e+00, -1.9637e+00, -9.7443e-01,  7.5692e-01,  7.3340e-01,\n",
       "          -2.3796e-01,  4.8667e-01, -6.1085e-01, -2.2212e-01, -5.9743e-01,\n",
       "          -1.9234e+00, -1.4403e+00,  1.2222e-01],\n",
       "         [ 6.3687e-01, -3.3216e-01, -3.8322e-01,  7.1767e-02,  1.7554e+00,\n",
       "          -1.2136e-01, -1.1447e+00,  1.1113e+00, -1.6653e+00, -8.0835e-02,\n",
       "           1.3145e+00, -5.8859e-02, -1.0046e+00,  3.7602e-01, -4.9904e-02,\n",
       "           1.7367e+00, -9.2469e-01, -9.1193e-01, -4.6526e-01, -1.1928e+00,\n",
       "          -1.1158e+00,  1.7080e+00, -9.7813e-02,  1.2550e+00,  2.7831e-01,\n",
       "           1.2749e+00,  2.7420e-01, -2.0610e+00,  5.2992e-01,  1.5049e+00,\n",
       "           1.8209e+00, -4.8626e-01,  1.6263e+00,  1.9268e+00, -1.8979e+00,\n",
       "           6.8125e-01,  3.3977e-01, -8.7828e-01,  1.5015e+00,  1.8623e-01,\n",
       "           9.1128e-01, -5.2096e-01,  3.2448e-02, -6.8498e-01, -8.5964e-01,\n",
       "           1.0872e+00, -9.9665e-01,  9.6223e-01],\n",
       "         [ 2.1168e+00,  1.0746e-01, -1.5483e+00, -2.9808e-02,  5.3820e-01,\n",
       "          -1.0248e+00,  2.4584e+00,  1.2024e+00,  7.5152e-01,  1.0762e+00,\n",
       "          -3.3181e-01, -9.1365e-01,  9.0658e-02, -1.0557e-01,  1.2233e+00,\n",
       "          -1.9447e+00, -1.1812e+00,  3.4699e-01, -1.5185e+00, -1.4696e+00,\n",
       "          -6.3665e-01,  1.1429e+00, -2.5816e-01,  7.9025e-01,  8.9829e-02,\n",
       "          -3.7640e-01, -1.7054e+00, -3.3809e-01, -1.0076e+00,  8.9098e-01,\n",
       "          -2.8997e-01,  1.1069e+00, -1.0840e+00, -5.1686e-01, -6.5695e-01,\n",
       "          -1.4667e+00,  1.5653e+00,  1.1391e-01,  2.2616e-01,  9.5715e-02,\n",
       "           5.3691e-01,  1.2869e+00, -2.0798e-01,  1.1513e+00,  6.6572e-01,\n",
       "           1.2450e-01,  1.1799e+00,  2.2515e-01],\n",
       "         [ 1.1050e+00, -2.2360e-01, -1.5840e+00,  1.4358e+00, -8.6698e-01,\n",
       "          -5.9488e-01, -2.3602e-01,  4.7475e-01, -1.0276e+00, -7.7849e-01,\n",
       "           1.4210e+00, -3.9582e-01,  1.3097e-01, -8.3848e-02,  8.0584e-02,\n",
       "          -8.0667e-01, -6.9115e-01,  1.3053e+00, -1.4175e-01, -1.2383e+00,\n",
       "           7.3872e-01, -2.2646e+00, -2.6141e-01, -4.1944e-01,  6.9246e-01,\n",
       "          -1.2355e+00,  2.4458e+00, -3.0642e-01, -1.6433e-01,  1.0878e+00,\n",
       "           3.4285e-01,  1.6735e+00, -3.8404e-01,  2.2263e+00, -1.2771e+00,\n",
       "          -3.8126e-01,  1.7411e+00, -8.2584e-01,  2.2320e+00, -2.7444e+00,\n",
       "           2.3554e-01,  8.3516e-01,  1.0938e-01, -8.3741e-02, -8.5888e-01,\n",
       "           1.4087e+00, -1.5748e+00,  4.9317e-01],\n",
       "         [ 2.0076e-01, -1.4694e+00, -8.1092e-01,  1.2562e+00,  5.3629e-01,\n",
       "          -1.5711e+00, -1.0957e+00,  8.2712e-01,  1.1066e+00,  2.2725e-02,\n",
       "           5.8961e-01,  4.3694e-02, -6.5706e-01, -8.3588e-01, -1.0679e-01,\n",
       "          -6.4319e-01,  1.3377e+00,  1.0571e+00,  2.0298e-01,  1.5887e-01,\n",
       "           1.5626e-01,  1.3144e+00, -8.1007e-01,  6.3025e-01,  2.0072e+00,\n",
       "           1.8663e+00,  1.9144e+00, -1.1573e+00,  6.4554e-01,  2.3373e+00,\n",
       "           5.2706e-01,  8.3998e-03,  3.2776e-02,  4.1992e-01,  2.2707e+00,\n",
       "          -1.1137e+00,  1.4160e-01,  1.3474e+00,  8.4999e-01,  8.1013e-01,\n",
       "           4.2577e-01,  7.1229e-01, -1.1968e-01, -3.9787e-01, -3.5811e-01,\n",
       "          -1.1000e+00, -1.8399e-03, -1.6822e+00]]])"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "input_tensor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}