Levels_prepare_data.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/calisolo/Levels_image_captioning_NICE/blob/master/Levels_prepare_data.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "%cd /content\n",
        "\n",
        "!git clone https://github.com/calisolo/Levels_image_captioning_NICE.git\n",
        "!git clone https://huggingface.co/datasets/calisolo/NICE_tsv/  #dataset\n",
        "\n",
        "\n",
        "%cd ./Levels_image_captioning_NICE"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "3wHp6S8axZJW",
        "outputId": "2f445f5c-515c-4586-e558-3e043e7362f9"
      },
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "/content\n",
            "Cloning into 'Levels_image_captioning_NICE'...\n",
            "remote: Enumerating objects: 689, done.\u001b[K\n",
            "remote: Counting objects: 100% (159/159), done.\u001b[K\n",
            "remote: Compressing objects: 100% (113/113), done.\u001b[K\n",
            "remote: Total 689 (delta 77), reused 121 (delta 46), pack-reused 530\u001b[K\n",
            "Receiving objects: 100% (689/689), 89.37 MiB | 27.74 MiB/s, done.\n",
            "Resolving deltas: 100% (308/308), done.\n",
            "Cloning into 'NICE_tsv'...\n",
            "remote: Enumerating objects: 9, done.\u001b[K\n",
            "remote: Counting objects: 100% (9/9), done.\u001b[K\n",
            "remote: Compressing objects: 100% (8/8), done.\u001b[K\n",
            "remote: Total 9 (delta 1), reused 0 (delta 0), pack-reused 0\u001b[K\n",
            "Unpacking objects: 100% (9/9), 1.25 KiB | 1.25 MiB/s, done.\n",
            "Filtering content: 100% (2/2), 890.00 MiB | 137.31 MiB/s, done.\n",
            "/content/Levels_image_captioning_NICE\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install transformers==4.20.0\n",
        "!pip install loguru"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "vt-2NVwOqXYB",
        "outputId": "d34c8e24-ef4e-4645-a435-31cdc5608e34"
      },
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting transformers==4.20.0\n",
            "  Downloading transformers-4.20.0-py3-none-any.whl (4.4 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m4.4/4.4 MB\u001b[0m \u001b[31m55.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from transformers==4.20.0) (3.12.0)\n",
            "Collecting huggingface-hub<1.0,>=0.1.0 (from transformers==4.20.0)\n",
            "  Downloading huggingface_hub-0.14.1-py3-none-any.whl (224 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m224.5/224.5 kB\u001b[0m \u001b[31m26.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.10/dist-packages (from transformers==4.20.0) (1.22.4)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/dist-packages (from transformers==4.20.0) (23.1)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from transformers==4.20.0) (6.0)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.10/dist-packages (from transformers==4.20.0) (2022.10.31)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from transformers==4.20.0) (2.27.1)\n",
            "Collecting tokenizers!=0.11.3,<0.13,>=0.11.1 (from transformers==4.20.0)\n",
            "  Downloading tokenizers-0.12.1-cp310-cp310-manylinux_2_12_x86_64.manylinux2010_x86_64.whl (6.6 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m6.6/6.6 MB\u001b[0m \u001b[31m119.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.10/dist-packages (from transformers==4.20.0) (4.65.0)\n",
            "Requirement already satisfied: fsspec in /usr/local/lib/python3.10/dist-packages (from huggingface-hub<1.0,>=0.1.0->transformers==4.20.0) (2023.4.0)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub<1.0,>=0.1.0->transformers==4.20.0) (4.5.0)\n",
            "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->transformers==4.20.0) (1.26.15)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->transformers==4.20.0) (2022.12.7)\n",
            "Requirement already satisfied: charset-normalizer~=2.0.0 in /usr/local/lib/python3.10/dist-packages (from requests->transformers==4.20.0) (2.0.12)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->transformers==4.20.0) (3.4)\n",
            "Installing collected packages: tokenizers, huggingface-hub, transformers\n",
            "Successfully installed huggingface-hub-0.14.1 tokenizers-0.12.1 transformers-4.20.0\n",
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting loguru\n",
            "  Downloading loguru-0.7.0-py3-none-any.whl (59 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m60.0/60.0 kB\u001b[0m \u001b[31m6.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hInstalling collected packages: loguru\n",
            "Successfully installed loguru-0.7.0\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import pandas as pd \n",
        "val_set = pd.read_csv(f'data/nice-val-5k.csv')\n",
        "pic_path = '../drive/MyDrive/NICE/cvpr-nice-test' ## you need to put test images to here! (in my case it was my google drive)"
      ],
      "metadata": {
        "id": "Oek7JyrC8CU_"
      },
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "YJMIuT_qBV6S"
      },
      "source": [
        "##image data, tsv format(img_id, '\\t', img_content)(base64 encoding):"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "background_save": true,
          "base_uri": "https://localhost:8080/"
        },
        "id": "1fNYmfIXBMQ_",
        "outputId": "058fd022-96b1-46df-cec9-e86872ab3428"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 21377/21377 [1:32:48<00:00,  3.84it/s]\n"
          ]
        }
      ],
      "source": [
        "from PIL import Image\n",
        "from io import BytesIO\n",
        "from tqdm import tqdm\n",
        "import base64\n",
        "\n",
        "result = []\n",
        "for pic_id in  tqdm(val_set['public_id'], total = len(val_set)):\n",
        "  img = Image.open(f'{pic_path}cvpr-nice-test/{pic_id}.jpg') # path to file\n",
        "  img_buffer = BytesIO()\n",
        "  img.save(img_buffer, format=img.format)\n",
        "  byte_data = img_buffer.getvalue()\n",
        "  base64_str = base64.b64encode(byte_data) # bytes\n",
        "  base64_str = base64_str.decode(\"utf-8\") # str\n",
        "\n",
        "  resultStr = str(pic_id) + '\\t' + base64_str\n",
        "  result.append(resultStr)\n",
        "  "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "fFFI3X9IEiyD"
      },
      "outputs": [],
      "source": [
        "import csv\n",
        "resultDF = pd.DataFrame(result)\n",
        "resultDF.to_csv(f'./train_image(need to delete backslash).tsv',sep='\\t', encoding = 'utf-8', index = False, header=None, quoting=csv.QUOTE_NONE,escapechar='\\\\' ) \n",
        "#A phenomenon in which double quotation marks are saved back and forth in str.. Need to re-upload after saving"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## make cosine similarities map"
      ],
      "metadata": {
        "id": "JdUfYNvRoZst"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "#load model to get cosine similiarities from encoderOutput\n",
        "from component.ofa.modeling_ofa import OFAModelForCaption, OFAModel\n",
        "from component.ofa.tokenization_ofa import OFATokenizer \n",
        "from torchvision import transforms\n",
        "import torch\n",
        "\n",
        "model_name_or_path = 'calisolo/OFA_huge_image_captioning'\n",
        "csv_path = '../NICE_tsv/'\n",
        "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
        "\n",
        "model = OFAModelForCaption.from_pretrained(model_name_or_path).to(device)\n",
        "tokenizer = OFATokenizer.from_pretrained('./vocab')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 155,
          "referenced_widgets": [
            "a62407f111334701915f6196034c7e02",
            "94800865747f496190bf10a4db1b104f",
            "09103135b5564852a480a7cfd5896a31",
            "653ef3edd71a4516843f07c01229f44d",
            "3e6a6486f631409d828d2017cf7e8a80",
            "2b5722de304d4f76a25e1a5525779d0a",
            "d4165e05cd37446fb6336cc3c4d93c1c",
            "9258b52ed9ae4ceaa8f2afb6034d626c",
            "372205eb1022400abfff7f91de248715",
            "2b7b69cc4bd947efb9fe306749bb2b35",
            "40484af65bf34135a236f6ec025a3118",
            "7dbfce2745cc4a0281512c5268b972c8",
            "595737ef22604248a55a26417238de21",
            "2e746cfcf71346c48236d3d8a1f8003c",
            "5f1119c391ba46158dd7e1115b06af40",
            "df704d5a494c41a5bee0ec809bd6e2f9",
            "afec0f8e19b441c69041e34e7db5ab92",
            "be69190efd534edda52d58e2e6241759",
            "af5ee55933c142ff87e5234ea51dc0d9",
            "8da9130dbb2e42dfbe5d0de0fd13a293",
            "f95d08e6610a49759d1ee7030d1434ae",
            "365c4b2e9c004a46bb23f77420511325"
          ]
        },
        "id": "UN-XLSW6qJHh",
        "outputId": "21acb648-601f-4ef0-fef1-6627d628ecfd"
      },
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading:   0%|          | 0.00/1.36k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "a62407f111334701915f6196034c7e02"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading:   0%|          | 0.00/4.09G [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "7dbfce2745cc4a0281512c5268b972c8"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/torch/functional.py:504: UserWarning: torch.meshgrid: in an upcoming release, it will be required to pass the indexing argument. (Triggered internally at ../aten/src/ATen/native/TensorShape.cpp:3483.)\n",
            "  return _VF.meshgrid(tensors, **kwargs)  # type: ignore[attr-defined]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Calculate the cosine similarity of **valid set** values"
      ],
      "metadata": {
        "id": "butwnt8zwN8b"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from PIL import Image\n",
        "from tqdm import tqdm\n",
        "from io import BytesIO\n",
        "import base64\n",
        "\n",
        "model.eval()\n",
        "\n",
        "mean, std = [0.5, 0.5, 0.5], [0.5, 0.5, 0.5]\n",
        "resolution = 256\n",
        "patch_resize_transform = transforms.Compose([\n",
        "        lambda image: image.convert(\"RGB\"),\n",
        "        transforms.Resize((resolution, resolution), interpolation=Image.BICUBIC),\n",
        "        transforms.ToTensor(),\n",
        "        transforms.Normalize(mean=mean, std=std)\n",
        "    ])\n",
        "\n",
        "\n",
        "txt = \" what does the image describe?\"\n",
        "inputs = tokenizer([txt], return_tensors=\"pt\").input_ids.to(device)\n",
        "encoderDict = {}\n",
        "image_file = f'{csv_path}train_image.tsv'\n",
        "\n",
        "with open(image_file, 'r', encoding='utf8') as f:\n",
        "  lines = f.readlines()\n",
        "\n",
        "  with torch.no_grad():\n",
        "    for line in tqdm(lines):\n",
        "      pic_id, image_content = line.split('\\t')\n",
        "      img = Image.open(BytesIO(base64.urlsafe_b64decode(image_content)))\n",
        "      patch_img = patch_resize_transform(img).unsqueeze(0).to(device)\n",
        "\n",
        "\n",
        "\n",
        "      encodedOutput = model.encoder(input_ids = inputs, patch_images = patch_img)\n",
        "      encoderDict[pic_id] = encodedOutput.last_hidden_state.cpu()\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Nj2jUvHFoe9q",
        "outputId": "94384b8e-2ff9-4a23-cb57-1deabc4110fd"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 5000/5000 [05:48<00:00, 14.35it/s]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def get_cosine_similarity(x1, x2):\n",
        "    return (x1 * x2).sum() / ((x1**2).sum()**.5 * (x2**2).sum()**.5)\n",
        "\n",
        "public_id_list = []\n",
        "for i in encoderDict:\n",
        "  public_id_list.append(i)"
      ],
      "metadata": {
        "id": "l6TnpDpapUA2"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "import numpy as np\n",
        "csMap = []\n",
        "\n",
        "for index_i,i in tqdm(enumerate(public_id_list), total = len(public_id_list)):\n",
        "  imsi = []\n",
        "  for index_j,j in enumerate(public_id_list):\n",
        "    if index_i == index_j:\n",
        "      imsi.append(0)\n",
        "      continue \n",
        "    cs = get_cosine_similarity(encoderDict[i], encoderDict[j]).item()\n",
        "    imsi.append(cs)\n",
        "\n",
        "  np_numbers = np.array(imsi)\n",
        "  top_8_indices = np.argpartition(np_numbers, -8)[-8:]\n",
        "  sorted_top_8_indices = top_8_indices[np.argsort(np_numbers[top_8_indices])[::-1]]\n",
        "\n",
        "  num_8= {}\n",
        "  for num in sorted_top_8_indices:\n",
        "    num_8[val_set['public_id'][num]]= imsi[num]\n",
        "\n",
        "  fin = [i, num_8]\n",
        "\n",
        "  csMap.append(fin)\n",
        "\n",
        "cspanda = pd.DataFrame(csMap)\n",
        "cspanda.to_csv('./valid_encoderMap.csv',index= False)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 394
        },
        "id": "4iR-XOAOpVSe",
        "outputId": "9e532eeb-bf19-413b-a9c6-03ebf4043940"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "  0%|          | 5/5000 [00:06<1:46:13,  1.28s/it]\n"
          ]
        },
        {
          "output_type": "error",
          "ename": "KeyboardInterrupt",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-10-8d5a7be00687>\u001b[0m in \u001b[0;36m<cell line: 4>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      8\u001b[0m       \u001b[0mimsi\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      9\u001b[0m       \u001b[0;32mcontinue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 10\u001b[0;31m     \u001b[0mcs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mget_cosine_similarity\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mencoderDict\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mencoderDict\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mj\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mitem\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     11\u001b[0m     \u001b[0mimsi\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     12\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-9-8c699a5f15c3>\u001b[0m in \u001b[0;36mget_cosine_similarity\u001b[0;34m(x1, x2)\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mget_cosine_similarity\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mx2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mx1\u001b[0m \u001b[0;34m*\u001b[0m \u001b[0mx2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msum\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m/\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx1\u001b[0m\u001b[0;34m**\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msum\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m**\u001b[0m\u001b[0;36m.5\u001b[0m \u001b[0;34m*\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mx2\u001b[0m\u001b[0;34m**\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msum\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m**\u001b[0m\u001b[0;36m.5\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      3\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0mpublic_id_list\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mencoderDict\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Calculate the cosine similarity value of **test set x valid set**"
      ],
      "metadata": {
        "id": "3DH-w6pvwXXf"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from io import BytesIO\n",
        "import base64\n",
        "#train dataset도 encoding 뽑아보기\n",
        "\n",
        "model.eval()\n",
        "mean, std = [0.5, 0.5, 0.5], [0.5, 0.5, 0.5]\n",
        "resolution = 256\n",
        "patch_resize_transform = transforms.Compose([\n",
        "        lambda image: image.convert(\"RGB\"),\n",
        "        transforms.Resize((resolution, resolution), interpolation=Image.BICUBIC),\n",
        "        transforms.ToTensor(),\n",
        "        transforms.Normalize(mean=mean, std=std)\n",
        "    ])\n",
        "\n",
        "\n",
        "txt = \" what does the image describe?\"\n",
        "inputs = tokenizer([txt], return_tensors=\"pt\").input_ids.to(device)\n",
        "encoderDict_test = {}\n",
        "\n",
        "image_file = f'{csv_path}test_image(need to delete backslash).tsv'\n",
        "\n",
        "with open(image_file, 'r', encoding='utf8') as f:\n",
        "  lines = f.readlines()\n",
        "\n",
        "  with torch.no_grad():\n",
        "    for line in tqdm(lines):\n",
        "      pic_id, image_content = line.split('\\t')\n",
        "      \n",
        "      pic_id = pic_id[:-1]##tsv에서 \\\\ 를 삭제안했을경우에..\n",
        "      img = Image.open(BytesIO(base64.urlsafe_b64decode(image_content)))\n",
        "      patch_img = patch_resize_transform(img).unsqueeze(0).to(device)\n",
        "\n",
        "\n",
        "\n",
        "      encodedOutput = model.encoder(input_ids = inputs, patch_images = patch_img)\n",
        "      encoderDict_test[pic_id] = encodedOutput.last_hidden_state.cpu()\n",
        "\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 418
        },
        "id": "mJAE9YMfpjp_",
        "outputId": "2c2020d9-626c-4e9e-d1fd-cf3f704bf938"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "  1%|          | 136/21377 [00:09<25:11, 14.05it/s]\n"
          ]
        },
        {
          "output_type": "error",
          "ename": "KeyboardInterrupt",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-11-6eeebc070274>\u001b[0m in \u001b[0;36m<cell line: 22>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     33\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     34\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 35\u001b[0;31m       \u001b[0mencodedOutput\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mencoder\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput_ids\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0minputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpatch_images\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpatch_img\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     36\u001b[0m       \u001b[0mencoderDict_test\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mpic_id\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mencodedOutput\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlast_hidden_state\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcpu\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     37\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m_call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1499\u001b[0m                 \u001b[0;32mor\u001b[0m \u001b[0m_global_backward_pre_hooks\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0m_global_backward_hooks\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1500\u001b[0m                 or _global_forward_hooks or _global_forward_pre_hooks):\n\u001b[0;32m-> 1501\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mforward_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1502\u001b[0m         \u001b[0;31m# Do not call functions when jit is used\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1503\u001b[0m         \u001b[0mfull_backward_hooks\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnon_full_backward_hooks\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/content/Levels_image_captioning_NICE/component/ofa/modeling_ofa.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, input_ids, patch_images, patch_images_2, patch_masks, output_attentions, output_hidden_states, token_embeddings, sample_patch_num)\u001b[0m\n\u001b[1;32m   1274\u001b[0m             \u001b[0mself_attn_bias\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself_attn_bias\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreshape\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1275\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1276\u001b[0;31m             \u001b[0mhidden_outputs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlayer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mattention_mask\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mhas_pads\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mattn_bias\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself_attn_bias\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moutput_attentions\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0moutput_attentions\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1277\u001b[0m             \u001b[0mx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhidden_outputs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1278\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m_call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1499\u001b[0m                 \u001b[0;32mor\u001b[0m \u001b[0m_global_backward_pre_hooks\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0m_global_backward_hooks\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1500\u001b[0m                 or _global_forward_hooks or _global_forward_pre_hooks):\n\u001b[0;32m-> 1501\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mforward_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1502\u001b[0m         \u001b[0;31m# Do not call functions when jit is used\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1503\u001b[0m         \u001b[0mfull_backward_hooks\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnon_full_backward_hooks\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/content/Levels_image_captioning_NICE/component/ofa/modeling_ofa.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, hidden_states, attention_mask, output_attentions, attn_bias)\u001b[0m\n\u001b[1;32m    505\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnormalize_before\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    506\u001b[0m             \u001b[0mhidden_states\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mself_attn_layer_norm\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mhidden_states\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 507\u001b[0;31m         hidden_states, attn_weights, _ = self.self_attn(\n\u001b[0m\u001b[1;32m    508\u001b[0m             \u001b[0mhidden_states\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mhidden_states\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    509\u001b[0m             \u001b[0mattention_mask\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mattention_mask\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m_call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1499\u001b[0m                 \u001b[0;32mor\u001b[0m \u001b[0m_global_backward_pre_hooks\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0m_global_backward_hooks\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1500\u001b[0m                 or _global_forward_hooks or _global_forward_pre_hooks):\n\u001b[0;32m-> 1501\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mforward_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1502\u001b[0m         \u001b[0;31m# Do not call functions when jit is used\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1503\u001b[0m         \u001b[0mfull_backward_hooks\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnon_full_backward_hooks\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/content/Levels_image_captioning_NICE/component/ofa/modeling_ofa.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, hidden_states, key_value_states, past_key_value, attention_mask, output_attentions, attn_bias)\u001b[0m\n\u001b[1;32m    395\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    396\u001b[0m         \u001b[0msrc_len\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mkey_states\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 397\u001b[0;31m         \u001b[0mattn_weights\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbmm\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mquery_states\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkey_states\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtranspose\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    398\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    399\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mattn_weights\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m!=\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mbsz\u001b[0m \u001b[0;34m*\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnum_heads\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtgt_len\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msrc_len\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "public_id_list_test = []\n",
        "for i in encoderDict_test:\n",
        "  public_id_list_test.append(i)\n"
      ],
      "metadata": {
        "id": "GP0YD7kipmUP"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "import numpy as np\n",
        "csMap_test = []\n",
        "\n",
        "for index_i,i in tqdm(enumerate(public_id_list_test), total = len(public_id_list_test)):\n",
        "  imsi = []\n",
        "  for index_j,j in enumerate(public_id_list):\n",
        "    # if index_i >= index_j:\n",
        "    #   imsi.append(0)\n",
        "    #   continue \n",
        "    cs = get_cosine_similarity(encoderDict_test[i], encoderDict[j]).item()\n",
        "    imsi.append(cs)\n",
        "\n",
        "  np_numbers = np.array(imsi)\n",
        "  top_8_indices = np.argpartition(np_numbers, -16)[-16:]\n",
        "  sorted_top_8_indices = top_8_indices[np.argsort(np_numbers[top_8_indices])[::-1]]\n",
        "\n",
        "  num_8= {}\n",
        "  for num in sorted_top_8_indices:\n",
        "    num_8[val_set['public_id'][num]]= imsi[num]\n",
        "\n",
        "  fin = [i, num_8]\n",
        "\n",
        "  csMap_test.append(fin)\n",
        "\n",
        "  if len(csMap_test)% 2000==1:\n",
        "    cspanda2 = pd.DataFrame(csMap_test)\n",
        "    cspanda2.to_csv('./test_encoderMap.csv',index= False)\n",
        "\n",
        "\n",
        "cspanda2 = pd.DataFrame(csMap_test)\n",
        "cspanda2.to_csv('./test_encoderMap.csv',index= False)"
      ],
      "metadata": {
        "id": "h6NrN7Vvpn2p"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "##add location, shotstyle, category to encoder hint"
      ],
      "metadata": {
        "id": "XuDLPNTcrknr"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "tag_added = pd.read_csv('./data/shotstyle_location worked.csv')\n",
        "train_added = pd.read_csv('./data/encoder/test_cosine_parsed.csv')"
      ],
      "metadata": {
        "id": "6zuPJ_y3gV2Z"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "valid = pd.read_csv('./data/encoder/test_encoderMap.csv')\n",
        "cosine = pd.read_csv('./data/encoder/test_fully_parsed_final.csv')"
      ],
      "metadata": {
        "id": "2u1ZOz0yDcUD"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "train_added"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 641
        },
        "id": "7WJcVFxUhvxE",
        "outputId": "80d72b48-61ec-4c7c-d841-0830c6ca08fb"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                0                                                  1  \\\n",
              "0      1587040490  {1586724557: 0.2432771772146225, 1654577039: 0...   \n",
              "1      1859182859  {1859182757: 0.3702859580516815, 1859181131: 0...   \n",
              "2      1766908167  {1670342087: 0.35281312465667725, 1586672558: ...   \n",
              "3      1587024551  {1572535982: 0.30572494864463806, 1817411052: ...   \n",
              "4       216584981  {216586736: 0.3301239013671875, 216589541: 0.3...   \n",
              "...           ...                                                ...   \n",
              "21372  1570572905  {1570573439: 0.4303014874458313, 1572527732: 0...   \n",
              "21373  1578224348  {1578935525: 0.35002365708351135, 1590067799: ...   \n",
              "21374  1586691827  {1586691842: 0.3243945837020874, 1586690789: 0...   \n",
              "21375  1869041693  {1869041636: 0.4087737202644348, 1843607069: 0...   \n",
              "21376  1590201398  {1590201410: 0.3639160692691803, 1590201377: 0...   \n",
              "\n",
              "             key1        key2        key3        key4           cos1  \\\n",
              "0      1586724557  1654577039  1590216599  1873340771  [cosHint lv1]   \n",
              "1      1859182757  1859181131  1859182766  1587148232  [cosHint lv3]   \n",
              "2      1670342087  1586672558  1722076475  1586673437  [cosHint lv3]   \n",
              "3      1572535982  1817411052  1297774409  1576774418  [cosHint lv2]   \n",
              "4       216586736   216589541  1590350849   216582401  [cosHint lv3]   \n",
              "...           ...         ...         ...         ...            ...   \n",
              "21372  1570573439  1572527732  1570161023  1570572920  [cosHint lv4]   \n",
              "21373  1578935525  1590067799  1578215051  1590059978  [cosHint lv3]   \n",
              "21374  1586691842  1586690789  1586690921  1586691857  [cosHint lv3]   \n",
              "21375  1869041636  1843607069  1866001004  1586666987  [cosHint lv4]   \n",
              "21376  1590201410  1590201377  1588019078  1588016237  [cosHint lv3]   \n",
              "\n",
              "                cos2           cos3           cos4           diff1  \\\n",
              "0      [cosHint lv1]  [cosHint lv1]  [cosHint lv1]  [diffHint lv1]   \n",
              "1      [cosHint lv3]  [cosHint lv3]  [cosHint lv3]  [diffHint lv2]   \n",
              "2      [cosHint lv3]  [cosHint lv3]  [cosHint lv2]  [diffHint lv1]   \n",
              "3      [cosHint lv2]  [cosHint lv1]  [cosHint lv1]  [diffHint lv1]   \n",
              "4      [cosHint lv2]  [cosHint lv2]  [cosHint lv2]  [diffHint lv2]   \n",
              "...              ...            ...            ...             ...   \n",
              "21372  [cosHint lv4]  [cosHint lv3]  [cosHint lv3]  [diffHint lv2]   \n",
              "21373  [cosHint lv3]  [cosHint lv3]  [cosHint lv3]  [diffHint lv1]   \n",
              "21374  [cosHint lv2]  [cosHint lv2]  [cosHint lv1]  [diffHint lv3]   \n",
              "21375  [cosHint lv3]  [cosHint lv2]  [cosHint lv1]  [diffHint lv3]   \n",
              "21376  [cosHint lv1]  [cosHint lv1]  [cosHint lv1]  [diffHint lv3]   \n",
              "\n",
              "                diff2           diff3           diff4  \n",
              "0      [diffHint lv1]  [diffHint lv1]  [diffHint lv1]  \n",
              "1      [diffHint lv2]  [diffHint lv3]  [diffHint lv1]  \n",
              "2      [diffHint lv1]  [diffHint lv1]  [diffHint lv1]  \n",
              "3      [diffHint lv1]  [diffHint lv1]  [diffHint lv1]  \n",
              "4      [diffHint lv2]  [diffHint lv1]  [diffHint lv2]  \n",
              "...               ...             ...             ...  \n",
              "21372  [diffHint lv1]  [diffHint lv1]  [diffHint lv3]  \n",
              "21373  [diffHint lv1]  [diffHint lv2]  [diffHint lv1]  \n",
              "21374  [diffHint lv2]  [diffHint lv2]  [diffHint lv3]  \n",
              "21375  [diffHint lv1]  [diffHint lv1]  [diffHint lv1]  \n",
              "21376  [diffHint lv3]  [diffHint lv1]  [diffHint lv1]  \n",
              "\n",
              "[21377 rows x 14 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-c403681c-b369-4284-bce4-f5c3dc48e4db\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>0</th>\n",
              "      <th>1</th>\n",
              "      <th>key1</th>\n",
              "      <th>key2</th>\n",
              "      <th>key3</th>\n",
              "      <th>key4</th>\n",
              "      <th>cos1</th>\n",
              "      <th>cos2</th>\n",
              "      <th>cos3</th>\n",
              "      <th>cos4</th>\n",
              "      <th>diff1</th>\n",
              "      <th>diff2</th>\n",
              "      <th>diff3</th>\n",
              "      <th>diff4</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>1587040490</td>\n",
              "      <td>{1586724557: 0.2432771772146225, 1654577039: 0...</td>\n",
              "      <td>1586724557</td>\n",
              "      <td>1654577039</td>\n",
              "      <td>1590216599</td>\n",
              "      <td>1873340771</td>\n",
              "      <td>[cosHint lv1]</td>\n",
              "      <td>[cosHint lv1]</td>\n",
              "      <td>[cosHint lv1]</td>\n",
              "      <td>[cosHint lv1]</td>\n",
              "      <td>[diffHint lv1]</td>\n",
              "      <td>[diffHint lv1]</td>\n",
              "      <td>[diffHint lv1]</td>\n",
              "      <td>[diffHint lv1]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>1859182859</td>\n",
              "      <td>{1859182757: 0.3702859580516815, 1859181131: 0...</td>\n",
              "      <td>1859182757</td>\n",
              "      <td>1859181131</td>\n",
              "      <td>1859182766</td>\n",
              "      <td>1587148232</td>\n",
              "      <td>[cosHint lv3]</td>\n",
              "      <td>[cosHint lv3]</td>\n",
              "      <td>[cosHint lv3]</td>\n",
              "      <td>[cosHint lv3]</td>\n",
              "      <td>[diffHint lv2]</td>\n",
              "      <td>[diffHint lv2]</td>\n",
              "      <td>[diffHint lv3]</td>\n",
              "      <td>[diffHint lv1]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>1766908167</td>\n",
              "      <td>{1670342087: 0.35281312465667725, 1586672558: ...</td>\n",
              "      <td>1670342087</td>\n",
              "      <td>1586672558</td>\n",
              "      <td>1722076475</td>\n",
              "      <td>1586673437</td>\n",
              "      <td>[cosHint lv3]</td>\n",
              "      <td>[cosHint lv3]</td>\n",
              "      <td>[cosHint lv3]</td>\n",
              "      <td>[cosHint lv2]</td>\n",
              "      <td>[diffHint lv1]</td>\n",
              "      <td>[diffHint lv1]</td>\n",
              "      <td>[diffHint lv1]</td>\n",
              "      <td>[diffHint lv1]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>1587024551</td>\n",
              "      <td>{1572535982: 0.30572494864463806, 1817411052: ...</td>\n",
              "      <td>1572535982</td>\n",
              "      <td>1817411052</td>\n",
              "      <td>1297774409</td>\n",
              "      <td>1576774418</td>\n",
              "      <td>[cosHint lv2]</td>\n",
              "      <td>[cosHint lv2]</td>\n",
              "      <td>[cosHint lv1]</td>\n",
              "      <td>[cosHint lv1]</td>\n",
              "      <td>[diffHint lv1]</td>\n",
              "      <td>[diffHint lv1]</td>\n",
              "      <td>[diffHint lv1]</td>\n",
              "      <td>[diffHint lv1]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>216584981</td>\n",
              "      <td>{216586736: 0.3301239013671875, 216589541: 0.3...</td>\n",
              "      <td>216586736</td>\n",
              "      <td>216589541</td>\n",
              "      <td>1590350849</td>\n",
              "      <td>216582401</td>\n",
              "      <td>[cosHint lv3]</td>\n",
              "      <td>[cosHint lv2]</td>\n",
              "      <td>[cosHint lv2]</td>\n",
              "      <td>[cosHint lv2]</td>\n",
              "      <td>[diffHint lv2]</td>\n",
              "      <td>[diffHint lv2]</td>\n",
              "      <td>[diffHint lv1]</td>\n",
              "      <td>[diffHint lv2]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>21372</th>\n",
              "      <td>1570572905</td>\n",
              "      <td>{1570573439: 0.4303014874458313, 1572527732: 0...</td>\n",
              "      <td>1570573439</td>\n",
              "      <td>1572527732</td>\n",
              "      <td>1570161023</td>\n",
              "      <td>1570572920</td>\n",
              "      <td>[cosHint lv4]</td>\n",
              "      <td>[cosHint lv4]</td>\n",
              "      <td>[cosHint lv3]</td>\n",
              "      <td>[cosHint lv3]</td>\n",
              "      <td>[diffHint lv2]</td>\n",
              "      <td>[diffHint lv1]</td>\n",
              "      <td>[diffHint lv1]</td>\n",
              "      <td>[diffHint lv3]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>21373</th>\n",
              "      <td>1578224348</td>\n",
              "      <td>{1578935525: 0.35002365708351135, 1590067799: ...</td>\n",
              "      <td>1578935525</td>\n",
              "      <td>1590067799</td>\n",
              "      <td>1578215051</td>\n",
              "      <td>1590059978</td>\n",
              "      <td>[cosHint lv3]</td>\n",
              "      <td>[cosHint lv3]</td>\n",
              "      <td>[cosHint lv3]</td>\n",
              "      <td>[cosHint lv3]</td>\n",
              "      <td>[diffHint lv1]</td>\n",
              "      <td>[diffHint lv1]</td>\n",
              "      <td>[diffHint lv2]</td>\n",
              "      <td>[diffHint lv1]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>21374</th>\n",
              "      <td>1586691827</td>\n",
              "      <td>{1586691842: 0.3243945837020874, 1586690789: 0...</td>\n",
              "      <td>1586691842</td>\n",
              "      <td>1586690789</td>\n",
              "      <td>1586690921</td>\n",
              "      <td>1586691857</td>\n",
              "      <td>[cosHint lv3]</td>\n",
              "      <td>[cosHint lv2]</td>\n",
              "      <td>[cosHint lv2]</td>\n",
              "      <td>[cosHint lv1]</td>\n",
              "      <td>[diffHint lv3]</td>\n",
              "      <td>[diffHint lv2]</td>\n",
              "      <td>[diffHint lv2]</td>\n",
              "      <td>[diffHint lv3]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>21375</th>\n",
              "      <td>1869041693</td>\n",
              "      <td>{1869041636: 0.4087737202644348, 1843607069: 0...</td>\n",
              "      <td>1869041636</td>\n",
              "      <td>1843607069</td>\n",
              "      <td>1866001004</td>\n",
              "      <td>1586666987</td>\n",
              "      <td>[cosHint lv4]</td>\n",
              "      <td>[cosHint lv3]</td>\n",
              "      <td>[cosHint lv2]</td>\n",
              "      <td>[cosHint lv1]</td>\n",
              "      <td>[diffHint lv3]</td>\n",
              "      <td>[diffHint lv1]</td>\n",
              "      <td>[diffHint lv1]</td>\n",
              "      <td>[diffHint lv1]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>21376</th>\n",
              "      <td>1590201398</td>\n",
              "      <td>{1590201410: 0.3639160692691803, 1590201377: 0...</td>\n",
              "      <td>1590201410</td>\n",
              "      <td>1590201377</td>\n",
              "      <td>1588019078</td>\n",
              "      <td>1588016237</td>\n",
              "      <td>[cosHint lv3]</td>\n",
              "      <td>[cosHint lv1]</td>\n",
              "      <td>[cosHint lv1]</td>\n",
              "      <td>[cosHint lv1]</td>\n",
              "      <td>[diffHint lv3]</td>\n",
              "      <td>[diffHint lv3]</td>\n",
              "      <td>[diffHint lv1]</td>\n",
              "      <td>[diffHint lv1]</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>21377 rows × 14 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-c403681c-b369-4284-bce4-f5c3dc48e4db')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-c403681c-b369-4284-bce4-f5c3dc48e4db button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-c403681c-b369-4284-bce4-f5c3dc48e4db');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 47
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "id_list = []\n",
        "cat_list = []\n",
        "val_dict = {}\n",
        "val_cat = {}\n",
        "for index, i in enumerate(tag_added['public_id']):\n",
        "  val_dict[i] = tag_added['location'][index]\n",
        "  id_list.append(i)\n",
        "\n",
        "for index, i in enumerate(tag_added['public_id']):\n",
        "  val_cat[i] = tag_added['category'][index]\n",
        "  cat_list.append(i)"
      ],
      "metadata": {
        "id": "3RS7jtGHjCQ6"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def find_closest_numbers(target, num_list, k):\n",
        "    num_list.sort(key=lambda x: abs(x - target))\n",
        "    if num_list[0] == target:\n",
        "      return num_list[1:k+1]\n",
        "    else:\n",
        "      return num_list[:k]\n"
      ],
      "metadata": {
        "id": "1AzIDy4Wj2nu"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import math\n",
        "shots = []\n",
        "locations = []\n",
        "for id in train_added['0']:\n",
        "  closest_numbers = find_closest_numbers(id, id_list, 6)\n",
        "  imsishot = []\n",
        "  imsilocation = []\n",
        "  for shot in closest_numbers:\n",
        "    if  type(val_dict[shot])!= float:\n",
        "      difference = abs(int(id) - int(shot))\n",
        "      if difference < 100:\n",
        "        diff = '[diffHint lv3]'+'['+ val_cat[shot]+ ']'\n",
        "      elif difference < 10000:\n",
        "        diff = '[diffHint lv2]'+'['+ val_cat[shot]+ ']'\n",
        "      else:\n",
        "        diff = '[diffHint lv1]'+'['+ val_cat[shot]+ ']'\n",
        "      imsilocation.append(diff + val_dict[shot])\n",
        "  locations.append(imsilocation)"
      ],
      "metadata": {
        "id": "AVRZwueKjv3r"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "cosine.to_csv('./test_fully_parsed_final_2.csv', index=False)"
      ],
      "metadata": {
        "id": "J7gTUHjTHLbc"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "cosine"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "wOEXxX-5G7pl",
        "outputId": "88d9d938-f371-4180-b2e7-2f5e311ee3f3"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                0                                                  1  \\\n",
              "0      1587040490  {1586724557: 0.2432771772146225, 1654577039: 0...   \n",
              "1      1859182859  {1859182757: 0.3702859580516815, 1859181131: 0...   \n",
              "2      1766908167  {1670342087: 0.35281312465667725, 1586672558: ...   \n",
              "3      1587024551  {1572535982: 0.30572494864463806, 1817411052: ...   \n",
              "4       216584981  {216586736: 0.3301239013671875, 216589541: 0.3...   \n",
              "...           ...                                                ...   \n",
              "21372  1570572905  {1570573439: 0.4303014874458313, 1572527732: 0...   \n",
              "21373  1578224348  {1578935525: 0.35002365708351135, 1590067799: ...   \n",
              "21374  1586691827  {1586691842: 0.3243945837020874, 1586690789: 0...   \n",
              "21375  1869041693  {1869041636: 0.4087737202644348, 1843607069: 0...   \n",
              "21376  1590201398  {1590201410: 0.3639160692691803, 1590201377: 0...   \n",
              "\n",
              "             key1        key2        key3        key4           cos1  \\\n",
              "0      1586724557  1654577039  1590216599  1873340771  [cosHint lv1]   \n",
              "1      1859182757  1859181131  1859182766  1587148232  [cosHint lv3]   \n",
              "2      1670342087  1586672558  1722076475  1586673437  [cosHint lv3]   \n",
              "3      1572535982  1817411052  1297774409  1576774418  [cosHint lv2]   \n",
              "4       216586736   216589541  1590350849   216582401  [cosHint lv3]   \n",
              "...           ...         ...         ...         ...            ...   \n",
              "21372  1570573439  1572527732  1570161023  1570572920  [cosHint lv4]   \n",
              "21373  1578935525  1590067799  1578215051  1590059978  [cosHint lv3]   \n",
              "21374  1586691842  1586690789  1586690921  1586691857  [cosHint lv3]   \n",
              "21375  1869041636  1843607069  1866001004  1586666987  [cosHint lv4]   \n",
              "21376  1590201410  1590201377  1588019078  1588016237  [cosHint lv3]   \n",
              "\n",
              "                cos2           cos3           cos4           diff1  \\\n",
              "0      [cosHint lv1]  [cosHint lv1]  [cosHint lv1]  [diffHint lv1]   \n",
              "1      [cosHint lv3]  [cosHint lv3]  [cosHint lv3]  [diffHint lv2]   \n",
              "2      [cosHint lv3]  [cosHint lv3]  [cosHint lv2]  [diffHint lv1]   \n",
              "3      [cosHint lv2]  [cosHint lv1]  [cosHint lv1]  [diffHint lv1]   \n",
              "4      [cosHint lv2]  [cosHint lv2]  [cosHint lv2]  [diffHint lv2]   \n",
              "...              ...            ...            ...             ...   \n",
              "21372  [cosHint lv4]  [cosHint lv3]  [cosHint lv3]  [diffHint lv2]   \n",
              "21373  [cosHint lv3]  [cosHint lv3]  [cosHint lv3]  [diffHint lv1]   \n",
              "21374  [cosHint lv2]  [cosHint lv2]  [cosHint lv1]  [diffHint lv3]   \n",
              "21375  [cosHint lv3]  [cosHint lv2]  [cosHint lv1]  [diffHint lv3]   \n",
              "21376  [cosHint lv1]  [cosHint lv1]  [cosHint lv1]  [diffHint lv3]   \n",
              "\n",
              "                diff2           diff3           diff4  \\\n",
              "0      [diffHint lv1]  [diffHint lv1]  [diffHint lv1]   \n",
              "1      [diffHint lv2]  [diffHint lv3]  [diffHint lv1]   \n",
              "2      [diffHint lv1]  [diffHint lv1]  [diffHint lv1]   \n",
              "3      [diffHint lv1]  [diffHint lv1]  [diffHint lv1]   \n",
              "4      [diffHint lv2]  [diffHint lv1]  [diffHint lv2]   \n",
              "...               ...             ...             ...   \n",
              "21372  [diffHint lv1]  [diffHint lv1]  [diffHint lv3]   \n",
              "21373  [diffHint lv1]  [diffHint lv2]  [diffHint lv1]   \n",
              "21374  [diffHint lv2]  [diffHint lv2]  [diffHint lv3]   \n",
              "21375  [diffHint lv1]  [diffHint lv1]  [diffHint lv1]   \n",
              "21376  [diffHint lv3]  [diffHint lv1]  [diffHint lv1]   \n",
              "\n",
              "                                                   shots  \\\n",
              "0                            ['Close up', 'Close up of']   \n",
              "1      ['Close up shot of', 'Low angle vertical shot ...   \n",
              "2                                  ['Low angle view of']   \n",
              "3      ['Blurred view of', 'View of', 'View of', 'Vie...   \n",
              "4                                        ['Close up of']   \n",
              "...                                                  ...   \n",
              "21372                                 ['Aerial view of']   \n",
              "21373                                                 []   \n",
              "21374                                                 []   \n",
              "21375  ['Profile of', 'Front view portrait of', 'Elev...   \n",
              "21376                         ['Blurred motion shot of']   \n",
              "\n",
              "                                               locations        cat1  \\\n",
              "0                                                     []  [outdoors]   \n",
              "1                                                     []  [outdoors]   \n",
              "2                                                     []  [outdoors]   \n",
              "3       [[diffHint lv2][outdoors]Cape Town South Africa]  [outdoors]   \n",
              "4      [[diffHint lv2][outdoors]Alcazar Seville Spain...  [outdoors]   \n",
              "...                                                  ...         ...   \n",
              "21372  [[diffHint lv3][outdoors]Cuxiu Muni Amazon Riv...  [outdoors]   \n",
              "21373                                                 []      [misc]   \n",
              "21374                                                 []      [misc]   \n",
              "21375  [[diffHint lv3][outdoors]western cape in south...  [outdoors]   \n",
              "21376  [[diffHint lv3][outdoors]Chicago Illinois Unit...  [outdoors]   \n",
              "\n",
              "             cat2        cat3        cat4  \n",
              "0          [misc]  [outdoors]    [office]  \n",
              "1      [outdoors]  [outdoors]  [outdoors]  \n",
              "2      [outdoors]  [outdoors]  [outdoors]  \n",
              "3      [outdoors]  [outdoors]  [outdoors]  \n",
              "4      [outdoors]  [outdoors]  [outdoors]  \n",
              "...           ...         ...         ...  \n",
              "21372  [outdoors]  [outdoors]  [outdoors]  \n",
              "21373   [indoors]      [misc]   [indoors]  \n",
              "21374      [misc]      [misc]      [misc]  \n",
              "21375  [outdoors]  [outdoors]      [misc]  \n",
              "21376   [medical]   [indoors]  [outdoors]  \n",
              "\n",
              "[21377 rows x 20 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-55d23769-4488-43e9-b734-a5e55eed9853\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>0</th>\n",
              "      <th>1</th>\n",
              "      <th>key1</th>\n",
              "      <th>key2</th>\n",
              "      <th>key3</th>\n",
              "      <th>key4</th>\n",
              "      <th>cos1</th>\n",
              "      <th>cos2</th>\n",
              "      <th>cos3</th>\n",
              "      <th>cos4</th>\n",
              "      <th>diff1</th>\n",
              "      <th>diff2</th>\n",
              "      <th>diff3</th>\n",
              "      <th>diff4</th>\n",
              "      <th>shots</th>\n",
              "      <th>locations</th>\n",
              "      <th>cat1</th>\n",
              "      <th>cat2</th>\n",
              "      <th>cat3</th>\n",
              "      <th>cat4</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>1587040490</td>\n",
              "      <td>{1586724557: 0.2432771772146225, 1654577039: 0...</td>\n",
              "      <td>1586724557</td>\n",
              "      <td>1654577039</td>\n",
              "      <td>1590216599</td>\n",
              "      <td>1873340771</td>\n",
              "      <td>[cosHint lv1]</td>\n",
              "      <td>[cosHint lv1]</td>\n",
              "      <td>[cosHint lv1]</td>\n",
              "      <td>[cosHint lv1]</td>\n",
              "      <td>[diffHint lv1]</td>\n",
              "      <td>[diffHint lv1]</td>\n",
              "      <td>[diffHint lv1]</td>\n",
              "      <td>[diffHint lv1]</td>\n",
              "      <td>['Close up', 'Close up of']</td>\n",
              "      <td>[]</td>\n",
              "      <td>[outdoors]</td>\n",
              "      <td>[misc]</td>\n",
              "      <td>[outdoors]</td>\n",
              "      <td>[office]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>1859182859</td>\n",
              "      <td>{1859182757: 0.3702859580516815, 1859181131: 0...</td>\n",
              "      <td>1859182757</td>\n",
              "      <td>1859181131</td>\n",
              "      <td>1859182766</td>\n",
              "      <td>1587148232</td>\n",
              "      <td>[cosHint lv3]</td>\n",
              "      <td>[cosHint lv3]</td>\n",
              "      <td>[cosHint lv3]</td>\n",
              "      <td>[cosHint lv3]</td>\n",
              "      <td>[diffHint lv2]</td>\n",
              "      <td>[diffHint lv2]</td>\n",
              "      <td>[diffHint lv3]</td>\n",
              "      <td>[diffHint lv1]</td>\n",
              "      <td>['Close up shot of', 'Low angle vertical shot ...</td>\n",
              "      <td>[]</td>\n",
              "      <td>[outdoors]</td>\n",
              "      <td>[outdoors]</td>\n",
              "      <td>[outdoors]</td>\n",
              "      <td>[outdoors]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>1766908167</td>\n",
              "      <td>{1670342087: 0.35281312465667725, 1586672558: ...</td>\n",
              "      <td>1670342087</td>\n",
              "      <td>1586672558</td>\n",
              "      <td>1722076475</td>\n",
              "      <td>1586673437</td>\n",
              "      <td>[cosHint lv3]</td>\n",
              "      <td>[cosHint lv3]</td>\n",
              "      <td>[cosHint lv3]</td>\n",
              "      <td>[cosHint lv2]</td>\n",
              "      <td>[diffHint lv1]</td>\n",
              "      <td>[diffHint lv1]</td>\n",
              "      <td>[diffHint lv1]</td>\n",
              "      <td>[diffHint lv1]</td>\n",
              "      <td>['Low angle view of']</td>\n",
              "      <td>[]</td>\n",
              "      <td>[outdoors]</td>\n",
              "      <td>[outdoors]</td>\n",
              "      <td>[outdoors]</td>\n",
              "      <td>[outdoors]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>1587024551</td>\n",
              "      <td>{1572535982: 0.30572494864463806, 1817411052: ...</td>\n",
              "      <td>1572535982</td>\n",
              "      <td>1817411052</td>\n",
              "      <td>1297774409</td>\n",
              "      <td>1576774418</td>\n",
              "      <td>[cosHint lv2]</td>\n",
              "      <td>[cosHint lv2]</td>\n",
              "      <td>[cosHint lv1]</td>\n",
              "      <td>[cosHint lv1]</td>\n",
              "      <td>[diffHint lv1]</td>\n",
              "      <td>[diffHint lv1]</td>\n",
              "      <td>[diffHint lv1]</td>\n",
              "      <td>[diffHint lv1]</td>\n",
              "      <td>['Blurred view of', 'View of', 'View of', 'Vie...</td>\n",
              "      <td>[[diffHint lv2][outdoors]Cape Town South Africa]</td>\n",
              "      <td>[outdoors]</td>\n",
              "      <td>[outdoors]</td>\n",
              "      <td>[outdoors]</td>\n",
              "      <td>[outdoors]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>216584981</td>\n",
              "      <td>{216586736: 0.3301239013671875, 216589541: 0.3...</td>\n",
              "      <td>216586736</td>\n",
              "      <td>216589541</td>\n",
              "      <td>1590350849</td>\n",
              "      <td>216582401</td>\n",
              "      <td>[cosHint lv3]</td>\n",
              "      <td>[cosHint lv2]</td>\n",
              "      <td>[cosHint lv2]</td>\n",
              "      <td>[cosHint lv2]</td>\n",
              "      <td>[diffHint lv2]</td>\n",
              "      <td>[diffHint lv2]</td>\n",
              "      <td>[diffHint lv1]</td>\n",
              "      <td>[diffHint lv2]</td>\n",
              "      <td>['Close up of']</td>\n",
              "      <td>[[diffHint lv2][outdoors]Alcazar Seville Spain...</td>\n",
              "      <td>[outdoors]</td>\n",
              "      <td>[outdoors]</td>\n",
              "      <td>[outdoors]</td>\n",
              "      <td>[outdoors]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>21372</th>\n",
              "      <td>1570572905</td>\n",
              "      <td>{1570573439: 0.4303014874458313, 1572527732: 0...</td>\n",
              "      <td>1570573439</td>\n",
              "      <td>1572527732</td>\n",
              "      <td>1570161023</td>\n",
              "      <td>1570572920</td>\n",
              "      <td>[cosHint lv4]</td>\n",
              "      <td>[cosHint lv4]</td>\n",
              "      <td>[cosHint lv3]</td>\n",
              "      <td>[cosHint lv3]</td>\n",
              "      <td>[diffHint lv2]</td>\n",
              "      <td>[diffHint lv1]</td>\n",
              "      <td>[diffHint lv1]</td>\n",
              "      <td>[diffHint lv3]</td>\n",
              "      <td>['Aerial view of']</td>\n",
              "      <td>[[diffHint lv3][outdoors]Cuxiu Muni Amazon Riv...</td>\n",
              "      <td>[outdoors]</td>\n",
              "      <td>[outdoors]</td>\n",
              "      <td>[outdoors]</td>\n",
              "      <td>[outdoors]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>21373</th>\n",
              "      <td>1578224348</td>\n",
              "      <td>{1578935525: 0.35002365708351135, 1590067799: ...</td>\n",
              "      <td>1578935525</td>\n",
              "      <td>1590067799</td>\n",
              "      <td>1578215051</td>\n",
              "      <td>1590059978</td>\n",
              "      <td>[cosHint lv3]</td>\n",
              "      <td>[cosHint lv3]</td>\n",
              "      <td>[cosHint lv3]</td>\n",
              "      <td>[cosHint lv3]</td>\n",
              "      <td>[diffHint lv1]</td>\n",
              "      <td>[diffHint lv1]</td>\n",
              "      <td>[diffHint lv2]</td>\n",
              "      <td>[diffHint lv1]</td>\n",
              "      <td>[]</td>\n",
              "      <td>[]</td>\n",
              "      <td>[misc]</td>\n",
              "      <td>[indoors]</td>\n",
              "      <td>[misc]</td>\n",
              "      <td>[indoors]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>21374</th>\n",
              "      <td>1586691827</td>\n",
              "      <td>{1586691842: 0.3243945837020874, 1586690789: 0...</td>\n",
              "      <td>1586691842</td>\n",
              "      <td>1586690789</td>\n",
              "      <td>1586690921</td>\n",
              "      <td>1586691857</td>\n",
              "      <td>[cosHint lv3]</td>\n",
              "      <td>[cosHint lv2]</td>\n",
              "      <td>[cosHint lv2]</td>\n",
              "      <td>[cosHint lv1]</td>\n",
              "      <td>[diffHint lv3]</td>\n",
              "      <td>[diffHint lv2]</td>\n",
              "      <td>[diffHint lv2]</td>\n",
              "      <td>[diffHint lv3]</td>\n",
              "      <td>[]</td>\n",
              "      <td>[]</td>\n",
              "      <td>[misc]</td>\n",
              "      <td>[misc]</td>\n",
              "      <td>[misc]</td>\n",
              "      <td>[misc]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>21375</th>\n",
              "      <td>1869041693</td>\n",
              "      <td>{1869041636: 0.4087737202644348, 1843607069: 0...</td>\n",
              "      <td>1869041636</td>\n",
              "      <td>1843607069</td>\n",
              "      <td>1866001004</td>\n",
              "      <td>1586666987</td>\n",
              "      <td>[cosHint lv4]</td>\n",
              "      <td>[cosHint lv3]</td>\n",
              "      <td>[cosHint lv2]</td>\n",
              "      <td>[cosHint lv1]</td>\n",
              "      <td>[diffHint lv3]</td>\n",
              "      <td>[diffHint lv1]</td>\n",
              "      <td>[diffHint lv1]</td>\n",
              "      <td>[diffHint lv1]</td>\n",
              "      <td>['Profile of', 'Front view portrait of', 'Elev...</td>\n",
              "      <td>[[diffHint lv3][outdoors]western cape in south...</td>\n",
              "      <td>[outdoors]</td>\n",
              "      <td>[outdoors]</td>\n",
              "      <td>[outdoors]</td>\n",
              "      <td>[misc]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>21376</th>\n",
              "      <td>1590201398</td>\n",
              "      <td>{1590201410: 0.3639160692691803, 1590201377: 0...</td>\n",
              "      <td>1590201410</td>\n",
              "      <td>1590201377</td>\n",
              "      <td>1588019078</td>\n",
              "      <td>1588016237</td>\n",
              "      <td>[cosHint lv3]</td>\n",
              "      <td>[cosHint lv1]</td>\n",
              "      <td>[cosHint lv1]</td>\n",
              "      <td>[cosHint lv1]</td>\n",
              "      <td>[diffHint lv3]</td>\n",
              "      <td>[diffHint lv3]</td>\n",
              "      <td>[diffHint lv1]</td>\n",
              "      <td>[diffHint lv1]</td>\n",
              "      <td>['Blurred motion shot of']</td>\n",
              "      <td>[[diffHint lv3][outdoors]Chicago Illinois Unit...</td>\n",
              "      <td>[outdoors]</td>\n",
              "      <td>[medical]</td>\n",
              "      <td>[indoors]</td>\n",
              "      <td>[outdoors]</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>21377 rows × 20 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-55d23769-4488-43e9-b734-a5e55eed9853')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-55d23769-4488-43e9-b734-a5e55eed9853 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-55d23769-4488-43e9-b734-a5e55eed9853');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 53
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2v-xKpefJfgO"
      },
      "source": [
        "##caption data, jsonl format:\n",
        "{\"image_id\": \"1813180760\", \"text\": [\"A vertical shot of sunset on a beach\"], \"encoder_prefix\": \"[cosHint lv3][diffHint lv1]A landscape shot of sunset at horizon over ocean[cosHint lv3][diffHint lv1]Sun beach and ocean at Gerrans Bay Cornwall United Kingdom[cosHint lv3][diffHint lv1]Vertical shot of a beautiful sunset over the sea[cosHint lv3][diffHint lv1]Sunrise near Los Islotes Baja California Sur Mexico\"}\n",
        "\n",
        "form of a dictionary so that it can be called with a caption suitable for the public id when desired."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "l9slTJQSYr-4",
        "outputId": "f0dec67d-2ba9-4e7b-bd42-83f410afa1c2"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting jsonlines\n",
            "  Downloading jsonlines-3.1.0-py3-none-any.whl (8.6 kB)\n",
            "Requirement already satisfied: attrs>=19.2.0 in /usr/local/lib/python3.10/dist-packages (from jsonlines) (23.1.0)\n",
            "Installing collected packages: jsonlines\n",
            "Successfully installed jsonlines-3.1.0\n"
          ]
        }
      ],
      "source": [
        "!pip install jsonlines"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "val_set"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 424
        },
        "id": "oXJZXd9tuBxA",
        "outputId": "4ab7cacd-c7da-44cd-ccd3-005c2c50d14f"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "       public_id                                         caption_gt  category\n",
              "0      216582038                       View of snowy mountain range  outdoors\n",
              "1     1590061574                       A young woman lying on a bed      misc\n",
              "2     1868714447  A horizontal view of a scientist in lab suit e...    office\n",
              "3     1813180760               A vertical shot of sunset on a beach  outdoors\n",
              "4     1578946151                   A woman relaxing in a deck chair  outdoors\n",
              "...          ...                                                ...       ...\n",
              "4995  1587836972       Father and young children cooking in kitchen   indoors\n",
              "4996   217367066                     Mountains and forest in Canada  outdoors\n",
              "4997  1571687297              View to Ronne Bornholm Island Denmark  outdoors\n",
              "4998  1297775183  Young businessman in busy office looking at ca...    office\n",
              "4999   217368161               Country road and single tree Belgium  outdoors\n",
              "\n",
              "[5000 rows x 3 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-dfc9d0eb-4b92-4749-8065-4230a0bf2f92\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>public_id</th>\n",
              "      <th>caption_gt</th>\n",
              "      <th>category</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>216582038</td>\n",
              "      <td>View of snowy mountain range</td>\n",
              "      <td>outdoors</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>1590061574</td>\n",
              "      <td>A young woman lying on a bed</td>\n",
              "      <td>misc</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>1868714447</td>\n",
              "      <td>A horizontal view of a scientist in lab suit e...</td>\n",
              "      <td>office</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>1813180760</td>\n",
              "      <td>A vertical shot of sunset on a beach</td>\n",
              "      <td>outdoors</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>1578946151</td>\n",
              "      <td>A woman relaxing in a deck chair</td>\n",
              "      <td>outdoors</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4995</th>\n",
              "      <td>1587836972</td>\n",
              "      <td>Father and young children cooking in kitchen</td>\n",
              "      <td>indoors</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4996</th>\n",
              "      <td>217367066</td>\n",
              "      <td>Mountains and forest in Canada</td>\n",
              "      <td>outdoors</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4997</th>\n",
              "      <td>1571687297</td>\n",
              "      <td>View to Ronne Bornholm Island Denmark</td>\n",
              "      <td>outdoors</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4998</th>\n",
              "      <td>1297775183</td>\n",
              "      <td>Young businessman in busy office looking at ca...</td>\n",
              "      <td>office</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4999</th>\n",
              "      <td>217368161</td>\n",
              "      <td>Country road and single tree Belgium</td>\n",
              "      <td>outdoors</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>5000 rows × 3 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-dfc9d0eb-4b92-4749-8065-4230a0bf2f92')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-dfc9d0eb-4b92-4749-8065-4230a0bf2f92 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-dfc9d0eb-4b92-4749-8065-4230a0bf2f92');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "val_dict = {}\n",
        "for index, i in enumerate(val_set['public_id']):\n",
        "  val_dict[i] = val_set['caption_gt'][index]"
      ],
      "metadata": {
        "id": "barIdQDhtnk1"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "pwd"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "id": "HE51org8SsgB",
        "outputId": "c0e73374-0833-4313-a9fd-6a3a437a8c61"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'/content/Levels_image_captioning_NICE'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "valid = pd.read_csv('./data/encoder/valid_encoderMap.csv')  ## get cosine similarities map \n",
        "cosine = pd.read_csv('./data/encoder/train_fully_parsed_final_2.csv') \n",
        "valid"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 424
        },
        "id": "VRf6VJnXxyM3",
        "outputId": "d128d498-9a12-4605-fb6c-e9a146f4d722"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "               0                                                  1\n",
              "0      216582038  {216581957: 0.5075774192810059, 216578222: 0.4...\n",
              "1     1590061574  {1590061697: 0.4025574028491974, 1590061787: 0...\n",
              "2     1868714447  {1587990881: 0.3821038007736206, 1868714480: 0...\n",
              "3     1813180760  {1811160932: 0.3904225826263428, 1859201003: 0...\n",
              "4     1578946151  {1578947159: 0.36091023683547974, 1817411052: ...\n",
              "...          ...                                                ...\n",
              "4995  1587836972  {1576745393: 0.34038618206977844, 1813179776: ...\n",
              "4996   217367066  {1572381332: 0.37447425723075867, 1570300802: ...\n",
              "4997  1571687297  {1590338072: 0.4009000360965729, 1571335145: 0...\n",
              "4998  1297775183  {1839580703: 0.38302579522132874, 1590061580: ...\n",
              "4999   217368161  {216581375: 0.3430343270301819, 1588016909: 0....\n",
              "\n",
              "[5000 rows x 2 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-e2522996-9789-4efe-a7fd-35cd33a48732\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>0</th>\n",
              "      <th>1</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>216582038</td>\n",
              "      <td>{216581957: 0.5075774192810059, 216578222: 0.4...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>1590061574</td>\n",
              "      <td>{1590061697: 0.4025574028491974, 1590061787: 0...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>1868714447</td>\n",
              "      <td>{1587990881: 0.3821038007736206, 1868714480: 0...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>1813180760</td>\n",
              "      <td>{1811160932: 0.3904225826263428, 1859201003: 0...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>1578946151</td>\n",
              "      <td>{1578947159: 0.36091023683547974, 1817411052: ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4995</th>\n",
              "      <td>1587836972</td>\n",
              "      <td>{1576745393: 0.34038618206977844, 1813179776: ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4996</th>\n",
              "      <td>217367066</td>\n",
              "      <td>{1572381332: 0.37447425723075867, 1570300802: ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4997</th>\n",
              "      <td>1571687297</td>\n",
              "      <td>{1590338072: 0.4009000360965729, 1571335145: 0...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4998</th>\n",
              "      <td>1297775183</td>\n",
              "      <td>{1839580703: 0.38302579522132874, 1590061580: ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4999</th>\n",
              "      <td>217368161</td>\n",
              "      <td>{216581375: 0.3430343270301819, 1588016909: 0....</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>5000 rows × 2 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-e2522996-9789-4efe-a7fd-35cd33a48732')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-e2522996-9789-4efe-a7fd-35cd33a48732 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-e2522996-9789-4efe-a7fd-35cd33a48732');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "cosine"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 922
        },
        "id": "63YtMj69r1Ri",
        "outputId": "69428210-6565-4b22-bb98-9aa43a0c9947"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "               0                                                  1  \\\n",
              "0      216582038  {216581957: 0.5075774192810059, 216578222: 0.4...   \n",
              "1     1590061574  {1590061697: 0.4025574028491974, 1590061787: 0...   \n",
              "2     1868714447  {1587990881: 0.3821038007736206, 1868714480: 0...   \n",
              "3     1813180760  {1811160932: 0.3904225826263428, 1859201003: 0...   \n",
              "4     1578946151  {1578947159: 0.36091023683547974, 1817411052: ...   \n",
              "...          ...                                                ...   \n",
              "4995  1587836972  {1576745393: 0.34038618206977844, 1813179776: ...   \n",
              "4996   217367066  {1572381332: 0.37447425723075867, 1570300802: ...   \n",
              "4997  1571687297  {1590338072: 0.4009000360965729, 1571335145: 0...   \n",
              "4998  1297775183  {1839580703: 0.38302579522132874, 1590061580: ...   \n",
              "4999   217368161  {216581375: 0.3430343270301819, 1588016909: 0....   \n",
              "\n",
              "            key1        key2        key3        key4           cos1  \\\n",
              "0      216581957   216578222   216581945  1586673371  [cosHint lv4]   \n",
              "1     1590061697  1590061787  1576745306  1588012475  [cosHint lv4]   \n",
              "2     1587990881  1868714480  1872077435  1868714486  [cosHint lv3]   \n",
              "3     1811160932  1859201003  1572527741   216779531  [cosHint lv3]   \n",
              "4     1578947159  1817411052  1571580875  1578942776  [cosHint lv3]   \n",
              "...          ...         ...         ...         ...            ...   \n",
              "4995  1576745393  1813179776  1665809777  1859336927  [cosHint lv3]   \n",
              "4996  1572381332  1570300802  1567877678  1567878608  [cosHint lv3]   \n",
              "4997  1590338072  1571335145  1571686673  1587842159  [cosHint lv4]   \n",
              "4998  1839580703  1590061580  1839584162  1587139106  [cosHint lv3]   \n",
              "4999   216581375  1588016909  1846400639   216570866  [cosHint lv3]   \n",
              "\n",
              "               cos2           cos3           cos4           diff1  \\\n",
              "0     [cosHint lv4]  [cosHint lv4]  [cosHint lv4]  [diffHint lv3]   \n",
              "1     [cosHint lv3]  [cosHint lv2]  [cosHint lv2]  [diffHint lv2]   \n",
              "2     [cosHint lv3]  [cosHint lv3]  [cosHint lv3]  [diffHint lv1]   \n",
              "3     [cosHint lv3]  [cosHint lv3]  [cosHint lv3]  [diffHint lv1]   \n",
              "4     [cosHint lv3]  [cosHint lv2]  [cosHint lv2]  [diffHint lv2]   \n",
              "...             ...            ...            ...             ...   \n",
              "4995  [cosHint lv3]  [cosHint lv2]  [cosHint lv2]  [diffHint lv1]   \n",
              "4996  [cosHint lv3]  [cosHint lv3]  [cosHint lv3]  [diffHint lv1]   \n",
              "4997  [cosHint lv3]  [cosHint lv3]  [cosHint lv3]  [diffHint lv1]   \n",
              "4998  [cosHint lv3]  [cosHint lv3]  [cosHint lv3]  [diffHint lv1]   \n",
              "4999  [cosHint lv3]  [cosHint lv2]  [cosHint lv2]  [diffHint lv1]   \n",
              "\n",
              "               diff2           diff3           diff4  \\\n",
              "0     [diffHint lv2]  [diffHint lv3]  [diffHint lv1]   \n",
              "1     [diffHint lv2]  [diffHint lv1]  [diffHint lv1]   \n",
              "2     [diffHint lv3]  [diffHint lv1]  [diffHint lv3]   \n",
              "3     [diffHint lv1]  [diffHint lv1]  [diffHint lv1]   \n",
              "4     [diffHint lv1]  [diffHint lv1]  [diffHint lv2]   \n",
              "...              ...             ...             ...   \n",
              "4995  [diffHint lv1]  [diffHint lv1]  [diffHint lv1]   \n",
              "4996  [diffHint lv1]  [diffHint lv1]  [diffHint lv1]   \n",
              "4997  [diffHint lv1]  [diffHint lv2]  [diffHint lv1]   \n",
              "4998  [diffHint lv1]  [diffHint lv1]  [diffHint lv1]   \n",
              "4999  [diffHint lv1]  [diffHint lv1]  [diffHint lv1]   \n",
              "\n",
              "                                                  shots  \\\n",
              "0                     ['View of', 'View of', 'View of']   \n",
              "1                                                    []   \n",
              "2     ['A portrait of', 'A horizontal view of', 'A f...   \n",
              "3                 ['Close up shot of', 'A portrait of']   \n",
              "4                                       ['Portrait of']   \n",
              "...                                                 ...   \n",
              "4995                                                 []   \n",
              "4996                                                 []   \n",
              "4997                                                 []   \n",
              "4998                                                 []   \n",
              "4999                                                 []   \n",
              "\n",
              "                                              locations        cat1  \\\n",
              "0     ['[diffHint lv2][outdoors]Jarnac and the Chare...  [outdoors]   \n",
              "1                                                    []      [misc]   \n",
              "2                                                    []   [indoors]   \n",
              "3                                                    []      [misc]   \n",
              "4                                                    []  [outdoors]   \n",
              "...                                                 ...         ...   \n",
              "4995  [\"[diffHint lv2][outdoors]Saint Peter's Basili...      [misc]   \n",
              "4996  ['[diffHint lv3][outdoors]Lauterbrunnen Switze...  [outdoors]   \n",
              "4997  ['[diffHint lv3][outdoors]Ravensburg Baden Wur...  [outdoors]   \n",
              "4998                                                 []    [office]   \n",
              "4999  ['[diffHint lv3][outdoors]South Shetland Islan...  [outdoors]   \n",
              "\n",
              "               cat2        cat3        cat4  \n",
              "0        [outdoors]  [outdoors]  [outdoors]  \n",
              "1            [misc]  [outdoors]      [misc]  \n",
              "2          [office]    [office]    [office]  \n",
              "3        [outdoors]  [outdoors]  [outdoors]  \n",
              "4        [outdoors]  [outdoors]  [outdoors]  \n",
              "...             ...         ...         ...  \n",
              "4995      [indoors]      [food]      [misc]  \n",
              "4996     [outdoors]  [outdoors]  [outdoors]  \n",
              "4997     [outdoors]  [outdoors]  [outdoors]  \n",
              "4998  [electronics]    [office]    [office]  \n",
              "4999     [outdoors]  [outdoors]  [outdoors]  \n",
              "\n",
              "[5000 rows x 20 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-8bc9183a-9958-44c7-84ee-e806478461f7\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>0</th>\n",
              "      <th>1</th>\n",
              "      <th>key1</th>\n",
              "      <th>key2</th>\n",
              "      <th>key3</th>\n",
              "      <th>key4</th>\n",
              "      <th>cos1</th>\n",
              "      <th>cos2</th>\n",
              "      <th>cos3</th>\n",
              "      <th>cos4</th>\n",
              "      <th>diff1</th>\n",
              "      <th>diff2</th>\n",
              "      <th>diff3</th>\n",
              "      <th>diff4</th>\n",
              "      <th>shots</th>\n",
              "      <th>locations</th>\n",
              "      <th>cat1</th>\n",
              "      <th>cat2</th>\n",
              "      <th>cat3</th>\n",
              "      <th>cat4</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>216582038</td>\n",
              "      <td>{216581957: 0.5075774192810059, 216578222: 0.4...</td>\n",
              "      <td>216581957</td>\n",
              "      <td>216578222</td>\n",
              "      <td>216581945</td>\n",
              "      <td>1586673371</td>\n",
              "      <td>[cosHint lv4]</td>\n",
              "      <td>[cosHint lv4]</td>\n",
              "      <td>[cosHint lv4]</td>\n",
              "      <td>[cosHint lv4]</td>\n",
              "      <td>[diffHint lv3]</td>\n",
              "      <td>[diffHint lv2]</td>\n",
              "      <td>[diffHint lv3]</td>\n",
              "      <td>[diffHint lv1]</td>\n",
              "      <td>['View of', 'View of', 'View of']</td>\n",
              "      <td>['[diffHint lv2][outdoors]Jarnac and the Chare...</td>\n",
              "      <td>[outdoors]</td>\n",
              "      <td>[outdoors]</td>\n",
              "      <td>[outdoors]</td>\n",
              "      <td>[outdoors]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>1590061574</td>\n",
              "      <td>{1590061697: 0.4025574028491974, 1590061787: 0...</td>\n",
              "      <td>1590061697</td>\n",
              "      <td>1590061787</td>\n",
              "      <td>1576745306</td>\n",
              "      <td>1588012475</td>\n",
              "      <td>[cosHint lv4]</td>\n",
              "      <td>[cosHint lv3]</td>\n",
              "      <td>[cosHint lv2]</td>\n",
              "      <td>[cosHint lv2]</td>\n",
              "      <td>[diffHint lv2]</td>\n",
              "      <td>[diffHint lv2]</td>\n",
              "      <td>[diffHint lv1]</td>\n",
              "      <td>[diffHint lv1]</td>\n",
              "      <td>[]</td>\n",
              "      <td>[]</td>\n",
              "      <td>[misc]</td>\n",
              "      <td>[misc]</td>\n",
              "      <td>[outdoors]</td>\n",
              "      <td>[misc]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>1868714447</td>\n",
              "      <td>{1587990881: 0.3821038007736206, 1868714480: 0...</td>\n",
              "      <td>1587990881</td>\n",
              "      <td>1868714480</td>\n",
              "      <td>1872077435</td>\n",
              "      <td>1868714486</td>\n",
              "      <td>[cosHint lv3]</td>\n",
              "      <td>[cosHint lv3]</td>\n",
              "      <td>[cosHint lv3]</td>\n",
              "      <td>[cosHint lv3]</td>\n",
              "      <td>[diffHint lv1]</td>\n",
              "      <td>[diffHint lv3]</td>\n",
              "      <td>[diffHint lv1]</td>\n",
              "      <td>[diffHint lv3]</td>\n",
              "      <td>['A portrait of', 'A horizontal view of', 'A f...</td>\n",
              "      <td>[]</td>\n",
              "      <td>[indoors]</td>\n",
              "      <td>[office]</td>\n",
              "      <td>[office]</td>\n",
              "      <td>[office]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>1813180760</td>\n",
              "      <td>{1811160932: 0.3904225826263428, 1859201003: 0...</td>\n",
              "      <td>1811160932</td>\n",
              "      <td>1859201003</td>\n",
              "      <td>1572527741</td>\n",
              "      <td>216779531</td>\n",
              "      <td>[cosHint lv3]</td>\n",
              "      <td>[cosHint lv3]</td>\n",
              "      <td>[cosHint lv3]</td>\n",
              "      <td>[cosHint lv3]</td>\n",
              "      <td>[diffHint lv1]</td>\n",
              "      <td>[diffHint lv1]</td>\n",
              "      <td>[diffHint lv1]</td>\n",
              "      <td>[diffHint lv1]</td>\n",
              "      <td>['Close up shot of', 'A portrait of']</td>\n",
              "      <td>[]</td>\n",
              "      <td>[misc]</td>\n",
              "      <td>[outdoors]</td>\n",
              "      <td>[outdoors]</td>\n",
              "      <td>[outdoors]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>1578946151</td>\n",
              "      <td>{1578947159: 0.36091023683547974, 1817411052: ...</td>\n",
              "      <td>1578947159</td>\n",
              "      <td>1817411052</td>\n",
              "      <td>1571580875</td>\n",
              "      <td>1578942776</td>\n",
              "      <td>[cosHint lv3]</td>\n",
              "      <td>[cosHint lv3]</td>\n",
              "      <td>[cosHint lv2]</td>\n",
              "      <td>[cosHint lv2]</td>\n",
              "      <td>[diffHint lv2]</td>\n",
              "      <td>[diffHint lv1]</td>\n",
              "      <td>[diffHint lv1]</td>\n",
              "      <td>[diffHint lv2]</td>\n",
              "      <td>['Portrait of']</td>\n",
              "      <td>[]</td>\n",
              "      <td>[outdoors]</td>\n",
              "      <td>[outdoors]</td>\n",
              "      <td>[outdoors]</td>\n",
              "      <td>[outdoors]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4995</th>\n",
              "      <td>1587836972</td>\n",
              "      <td>{1576745393: 0.34038618206977844, 1813179776: ...</td>\n",
              "      <td>1576745393</td>\n",
              "      <td>1813179776</td>\n",
              "      <td>1665809777</td>\n",
              "      <td>1859336927</td>\n",
              "      <td>[cosHint lv3]</td>\n",
              "      <td>[cosHint lv3]</td>\n",
              "      <td>[cosHint lv2]</td>\n",
              "      <td>[cosHint lv2]</td>\n",
              "      <td>[diffHint lv1]</td>\n",
              "      <td>[diffHint lv1]</td>\n",
              "      <td>[diffHint lv1]</td>\n",
              "      <td>[diffHint lv1]</td>\n",
              "      <td>[]</td>\n",
              "      <td>[\"[diffHint lv2][outdoors]Saint Peter's Basili...</td>\n",
              "      <td>[misc]</td>\n",
              "      <td>[indoors]</td>\n",
              "      <td>[food]</td>\n",
              "      <td>[misc]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4996</th>\n",
              "      <td>217367066</td>\n",
              "      <td>{1572381332: 0.37447425723075867, 1570300802: ...</td>\n",
              "      <td>1572381332</td>\n",
              "      <td>1570300802</td>\n",
              "      <td>1567877678</td>\n",
              "      <td>1567878608</td>\n",
              "      <td>[cosHint lv3]</td>\n",
              "      <td>[cosHint lv3]</td>\n",
              "      <td>[cosHint lv3]</td>\n",
              "      <td>[cosHint lv3]</td>\n",
              "      <td>[diffHint lv1]</td>\n",
              "      <td>[diffHint lv1]</td>\n",
              "      <td>[diffHint lv1]</td>\n",
              "      <td>[diffHint lv1]</td>\n",
              "      <td>[]</td>\n",
              "      <td>['[diffHint lv3][outdoors]Lauterbrunnen Switze...</td>\n",
              "      <td>[outdoors]</td>\n",
              "      <td>[outdoors]</td>\n",
              "      <td>[outdoors]</td>\n",
              "      <td>[outdoors]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4997</th>\n",
              "      <td>1571687297</td>\n",
              "      <td>{1590338072: 0.4009000360965729, 1571335145: 0...</td>\n",
              "      <td>1590338072</td>\n",
              "      <td>1571335145</td>\n",
              "      <td>1571686673</td>\n",
              "      <td>1587842159</td>\n",
              "      <td>[cosHint lv4]</td>\n",
              "      <td>[cosHint lv3]</td>\n",
              "      <td>[cosHint lv3]</td>\n",
              "      <td>[cosHint lv3]</td>\n",
              "      <td>[diffHint lv1]</td>\n",
              "      <td>[diffHint lv1]</td>\n",
              "      <td>[diffHint lv2]</td>\n",
              "      <td>[diffHint lv1]</td>\n",
              "      <td>[]</td>\n",
              "      <td>['[diffHint lv3][outdoors]Ravensburg Baden Wur...</td>\n",
              "      <td>[outdoors]</td>\n",
              "      <td>[outdoors]</td>\n",
              "      <td>[outdoors]</td>\n",
              "      <td>[outdoors]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4998</th>\n",
              "      <td>1297775183</td>\n",
              "      <td>{1839580703: 0.38302579522132874, 1590061580: ...</td>\n",
              "      <td>1839580703</td>\n",
              "      <td>1590061580</td>\n",
              "      <td>1839584162</td>\n",
              "      <td>1587139106</td>\n",
              "      <td>[cosHint lv3]</td>\n",
              "      <td>[cosHint lv3]</td>\n",
              "      <td>[cosHint lv3]</td>\n",
              "      <td>[cosHint lv3]</td>\n",
              "      <td>[diffHint lv1]</td>\n",
              "      <td>[diffHint lv1]</td>\n",
              "      <td>[diffHint lv1]</td>\n",
              "      <td>[diffHint lv1]</td>\n",
              "      <td>[]</td>\n",
              "      <td>[]</td>\n",
              "      <td>[office]</td>\n",
              "      <td>[electronics]</td>\n",
              "      <td>[office]</td>\n",
              "      <td>[office]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4999</th>\n",
              "      <td>217368161</td>\n",
              "      <td>{216581375: 0.3430343270301819, 1588016909: 0....</td>\n",
              "      <td>216581375</td>\n",
              "      <td>1588016909</td>\n",
              "      <td>1846400639</td>\n",
              "      <td>216570866</td>\n",
              "      <td>[cosHint lv3]</td>\n",
              "      <td>[cosHint lv3]</td>\n",
              "      <td>[cosHint lv2]</td>\n",
              "      <td>[cosHint lv2]</td>\n",
              "      <td>[diffHint lv1]</td>\n",
              "      <td>[diffHint lv1]</td>\n",
              "      <td>[diffHint lv1]</td>\n",
              "      <td>[diffHint lv1]</td>\n",
              "      <td>[]</td>\n",
              "      <td>['[diffHint lv3][outdoors]South Shetland Islan...</td>\n",
              "      <td>[outdoors]</td>\n",
              "      <td>[outdoors]</td>\n",
              "      <td>[outdoors]</td>\n",
              "      <td>[outdoors]</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>5000 rows × 20 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-8bc9183a-9958-44c7-84ee-e806478461f7')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-8bc9183a-9958-44c7-84ee-e806478461f7 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-8bc9183a-9958-44c7-84ee-e806478461f7');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "rLrWOmSoYk2K"
      },
      "outputs": [],
      "source": [
        "import jsonlines\n",
        "import ast\n",
        "\n",
        "# 예시 데이터\n",
        "caption_result = []\n",
        "imsi_result = {}\n",
        "for index, public_id in enumerate(cosine['0']):\n",
        "  processed_data = ''\n",
        "  imsi_result['image_id'] = str(public_id)\n",
        "  imsi_result['text'] = [val_dict[public_id]]\n",
        "\n",
        "  processed_data = cosine['cos1'][index] + cosine['diff1'][index] + cosine['cat1'][index]+ val_dict[cosine['key1'][index]]\n",
        "  processed_data += cosine['cos2'][index] + cosine['diff2'][index] + cosine['cat2'][index]+val_dict[cosine['key2'][index]]\n",
        "  processed_data += cosine['cos3'][index] + cosine['diff3'][index] + cosine['cat3'][index]+val_dict[cosine['key3'][index]]\n",
        "  processed_data += cosine['cos4'][index] + cosine['diff4'][index] + cosine['cat4'][index]+val_dict[cosine['key4'][index]]\n",
        "\n",
        "  processed_data += '[shot_style]'\n",
        "\n",
        "  \n",
        "  shotlist = ast.literal_eval(cosine['shots'][index])\n",
        "  locationlist = ast.literal_eval(cosine['locations'][index])\n",
        "  if shotlist == []:\n",
        "    processed_data += '[NULL]'\n",
        "  else:\n",
        "    for shot in shotlist:\n",
        "      processed_data += shot + ' | '\n",
        "  processed_data += '[Location]'\n",
        "  if locationlist == []:\n",
        "    processed_data += '[NULL]'\n",
        "  else:\n",
        "    for location in locationlist:\n",
        "      processed_data += location + ' | '\n",
        "  imsi_result['encoder_prefix'] = processed_data\n",
        "  \n",
        "  \n",
        "  a = imsi_result.copy()\n",
        "  caption_result.append(a)\n"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "# 파일에 JSONNL 형식으로 데이터 저장\n",
        "with jsonlines.open(f'data/train_caption_catAdded_2.jsonl', mode='w') as writer:\n",
        "    for item in caption_result:\n",
        "        writer.write(item)"
      ],
      "metadata": {
        "id": "DJ_kccKvxmxt"
      },
      "execution_count": null,
      "outputs": []
    }
  ],
  "metadata": {
    "colab": {
      "provenance": [],
      "machine_shape": "hm",
      "collapsed_sections": [
        "YJMIuT_qBV6S",
        "JdUfYNvRoZst",
        "XuDLPNTcrknr",
        "2v-xKpefJfgO"
      ],
      "gpuType": "V100",
      "authorship_tag": "ABX9TyNAaT8U4s2kYbZGIbIe71iD",
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    },
    "gpuClass": "standard",
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "a62407f111334701915f6196034c7e02": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_94800865747f496190bf10a4db1b104f",
              "IPY_MODEL_09103135b5564852a480a7cfd5896a31",
              "IPY_MODEL_653ef3edd71a4516843f07c01229f44d"
            ],
            "layout": "IPY_MODEL_3e6a6486f631409d828d2017cf7e8a80"
          }
        },
        "94800865747f496190bf10a4db1b104f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_2b5722de304d4f76a25e1a5525779d0a",
            "placeholder": "​",
            "style": "IPY_MODEL_d4165e05cd37446fb6336cc3c4d93c1c",
            "value": "Downloading: 100%"
          }
        },
        "09103135b5564852a480a7cfd5896a31": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_9258b52ed9ae4ceaa8f2afb6034d626c",
            "max": 1396,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_372205eb1022400abfff7f91de248715",
            "value": 1396
          }
        },
        "653ef3edd71a4516843f07c01229f44d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_2b7b69cc4bd947efb9fe306749bb2b35",
            "placeholder": "​",
            "style": "IPY_MODEL_40484af65bf34135a236f6ec025a3118",
            "value": " 1.36k/1.36k [00:00&lt;00:00, 70.6kB/s]"
          }
        },
        "3e6a6486f631409d828d2017cf7e8a80": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "2b5722de304d4f76a25e1a5525779d0a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "d4165e05cd37446fb6336cc3c4d93c1c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "9258b52ed9ae4ceaa8f2afb6034d626c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "372205eb1022400abfff7f91de248715": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "2b7b69cc4bd947efb9fe306749bb2b35": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "40484af65bf34135a236f6ec025a3118": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "7dbfce2745cc4a0281512c5268b972c8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_595737ef22604248a55a26417238de21",
              "IPY_MODEL_2e746cfcf71346c48236d3d8a1f8003c",
              "IPY_MODEL_5f1119c391ba46158dd7e1115b06af40"
            ],
            "layout": "IPY_MODEL_df704d5a494c41a5bee0ec809bd6e2f9"
          }
        },
        "595737ef22604248a55a26417238de21": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_afec0f8e19b441c69041e34e7db5ab92",
            "placeholder": "​",
            "style": "IPY_MODEL_be69190efd534edda52d58e2e6241759",
            "value": "Downloading: 100%"
          }
        },
        "2e746cfcf71346c48236d3d8a1f8003c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_af5ee55933c142ff87e5234ea51dc0d9",
            "max": 4394603544,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_8da9130dbb2e42dfbe5d0de0fd13a293",
            "value": 4394603544
          }
        },
        "5f1119c391ba46158dd7e1115b06af40": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_f95d08e6610a49759d1ee7030d1434ae",
            "placeholder": "​",
            "style": "IPY_MODEL_365c4b2e9c004a46bb23f77420511325",
            "value": " 4.09G/4.09G [01:45&lt;00:00, 38.8MB/s]"
          }
        },
        "df704d5a494c41a5bee0ec809bd6e2f9": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "afec0f8e19b441c69041e34e7db5ab92": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "be69190efd534edda52d58e2e6241759": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "af5ee55933c142ff87e5234ea51dc0d9": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "8da9130dbb2e42dfbe5d0de0fd13a293": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "f95d08e6610a49759d1ee7030d1434ae": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "365c4b2e9c004a46bb23f77420511325": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}