transformer_models.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from copy import deepcopy\n",
    "from pathlib import Path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from PIL import Image\n",
    "import requests\n",
    "from transformers import AutoProcessor, AutoModel\n",
    "import torch\n",
    "\n",
    "model = AutoModel.from_pretrained(\"google/siglip-base-patch16-224\")\n",
    "processor = AutoProcessor.from_pretrained(\"google/siglip-base-patch16-224\")\n",
    "\n",
    "url = \"http://images.cocodataset.org/val2017/000000039769.jpg\"\n",
    "image = Image.open(requests.get(url, stream=True).raw)\n",
    "\n",
    "texts = [\"a photo of 2 dsadsadcats\",'dsadasd']\n",
    "# important: we pass `padding=max_length` since the model was trained with this\n",
    "inputs = processor(text=texts, images=[image,image], padding=\"max_length\", return_tensors=\"pt\")\n",
    "\n",
    "with torch.no_grad():\n",
    "    outputs = model(**inputs)\n",
    "\n",
    "logits_per_image = outputs.logits_per_image\n",
    "probs = torch.sigmoid(logits_per_image) # these are the probabilities\n",
    "print(f\"{probs[0][0]:.1%} that image 0 is '{texts[0]}'\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from transformers import SiglipModel, SiglipConfig\n",
    "from typing import Optional, Tuple, Union\n",
    "import torch\n",
    "class Siglip(SiglipModel):\n",
    "    config_class = SiglipConfig\n",
    "    def __init__(self, config: SiglipConfig):\n",
    "        super().__init__(config)\n",
    "    def forward(\n",
    "        self,\n",
    "        input_ids: Optional[torch.LongTensor] = None,\n",
    "        pixel_values: Optional[torch.FloatTensor] = None,\n",
    "        attention_mask: Optional[torch.Tensor] = None,\n",
    "        position_ids: Optional[torch.LongTensor] = None,\n",
    "        return_loss: Optional[bool] = None,\n",
    "        output_attentions: Optional[bool] = None,\n",
    "        output_hidden_states: Optional[bool] = None,\n",
    "        return_dict: Optional[bool] = None,\n",
    "    ):\n",
    "        vision_outputs = self.vision_model(\n",
    "            pixel_values=pixel_values,\n",
    "            output_attentions=output_attentions,\n",
    "            output_hidden_states=output_hidden_states,\n",
    "            return_dict=return_dict,\n",
    "        )\n",
    "        text_outputs = self.text_model(\n",
    "            input_ids=input_ids,\n",
    "            attention_mask=attention_mask,\n",
    "            position_ids=position_ids,\n",
    "            output_attentions=output_attentions,\n",
    "            output_hidden_states=output_hidden_states,\n",
    "            return_dict=return_dict,\n",
    "        )\n",
    "        image_embeds = vision_outputs[1]\n",
    "        text_embeds = text_outputs[1]\n",
    "        # normalized features\n",
    "        image_embeds = image_embeds / image_embeds.norm(p=2, dim=-1, keepdim=True)\n",
    "        text_embeds = text_embeds / text_embeds.norm(p=2, dim=-1, keepdim=True)\n",
    "        return image_embeds, text_embeds, self.logit_scale.exp(), self.logit_bias\n",
    "wrapper_model = Siglip.from_pretrained(\"/home/mila/l/le.zhang/scratch/github_clone/Enhance-FineGrained/src/Outputs/test_07-Apr-2024-22-52-56/checkpoints/epoch_1.pt\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "config = SiglipConfig.from_pretrained(\"google/siglip-base-patch16-224\")\n",
    "model = Siglip(config)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_state_dict(checkpoint_path: str, map_location='cpu'):\n",
    "    checkpoint = torch.load(checkpoint_path, map_location=map_location)\n",
    "    if isinstance(checkpoint, dict) and 'state_dict' in checkpoint:\n",
    "        state_dict = checkpoint['state_dict']\n",
    "    else:\n",
    "        state_dict = checkpoint\n",
    "    if next(iter(state_dict.items()))[0].startswith('module'):\n",
    "        state_dict = {k[7:]: v for k, v in state_dict.items()}\n",
    "    return state_dict\n",
    "\n",
    "state_dict = load_state_dict(\"/home/mila/l/le.zhang/scratch/github_clone/Enhance-FineGrained/src/Outputs/test_07-Apr-2024-22-52-56/checkpoints/epoch_1.pt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<All keys matched successfully>"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.load_state_dict(state_dict, strict=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.load_state_dict(checkpoint)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "device =torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
    "n =128\n",
    "logits = torch.randn(n, n)\n",
    "labels = 2 * torch.eye(n, device=device) - torch.ones(n, device = device) \n",
    "labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch.nn.functional as F\n",
    "-torch.mean(F.logsigmoid(labels * logits))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "torchvision.transforms("
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "openflamingo",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.16"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}