siamese-contrastive-loss.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "siamese-contrastive-loss.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.8.0"
    },
    "toc": {
      "base_numbering": 1,
      "nav_menu": {},
      "number_sections": true,
      "sideBar": true,
      "skip_h1_title": false,
      "title_cell": "Table of Contents",
      "title_sidebar": "Contents",
      "toc_cell": true,
      "toc_position": {},
      "toc_section_display": true,
      "toc_window_display": false
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "koty8IglfEJi"
      },
      "source": [
        "<h1>Table of Contents<span class=\"tocSkip\"></span></h1>\n",
        "<div class=\"toc\"><ul class=\"toc-item\"><li><span><a href=\"#Data-Cleaning\" data-toc-modified-id=\"Data-Cleaning-1\"><span class=\"toc-item-num\">1&nbsp;&nbsp;</span>Data Cleaning</a></span></li><li><span><a href=\"#Data-Split\" data-toc-modified-id=\"Data-Split-2\"><span class=\"toc-item-num\">2&nbsp;&nbsp;</span>Data Split</a></span></li></ul></div>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "wxcoNPQ1fEJq",
        "outputId": "9f855755-ab45-40cd-bb32-10afc646007d"
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Mounted at /content/drive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "H3uvUz3Qfh1i",
        "outputId": "852b63c9-4b0b-4994-cc70-8d6f84569c10"
      },
      "source": [
        "nltk.download('stopwords')\n",
        "nltk.download('punkt')\n",
        "nltk.download('averaged_perceptron_tagger')\n",
        "nltk.download('wordnet')"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[nltk_data] Downloading package stopwords to /root/nltk_data...\n",
            "[nltk_data]   Unzipping corpora/stopwords.zip.\n",
            "[nltk_data] Downloading package punkt to /root/nltk_data...\n",
            "[nltk_data]   Unzipping tokenizers/punkt.zip.\n",
            "[nltk_data] Downloading package averaged_perceptron_tagger to\n",
            "[nltk_data]     /root/nltk_data...\n",
            "[nltk_data]   Unzipping taggers/averaged_perceptron_tagger.zip.\n",
            "[nltk_data] Downloading package wordnet to /root/nltk_data...\n",
            "[nltk_data]   Unzipping corpora/wordnet.zip.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "True"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 2
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "N3nfdBdofEJr"
      },
      "source": [
        "import pandas as pd\n",
        "import numpy as np\n",
        "import random\n",
        "import warnings\n",
        "import time\n",
        "import datetime\n",
        "import re\n",
        "import string\n",
        "import itertools\n",
        "import pickle\n",
        "import joblib\n",
        "import nltk\n",
        "import csv\n",
        "\n",
        "from nltk.corpus import stopwords, wordnet\n",
        "stop = set(stopwords.words('english'))\n",
        "from wordcloud import WordCloud, STOPWORDS\n",
        "from nltk.stem import WordNetLemmatizer\n",
        "from nltk.tokenize import word_tokenize\n",
        "from nltk.probability import FreqDist\n",
        "from collections import Counter, defaultdict\n",
        "\n",
        "import tensorflow as tf\n",
        "import keras\n",
        "import keras.backend as K\n",
        "from keras.preprocessing.text import Tokenizer\n",
        "from keras.preprocessing.sequence import pad_sequences\n",
        "from keras.layers import Input, Concatenate, Conv2D, Flatten, Dense, Embedding, LSTM\n",
        "from keras.models import Model\n",
        "\n",
        "from sklearn.model_selection import train_test_split\n",
        "from sklearn.metrics import roc_auc_score\n",
        "\n",
        "from keras.regularizers import l2\n",
        "from keras.models import Sequential\n",
        "from keras.optimizers import Adam\n",
        "from keras.layers import Conv2D, ZeroPadding2D, Activation, Input, concatenate\n",
        "from keras.models import Model\n",
        "from keras.layers.normalization import BatchNormalization\n",
        "from keras.layers.pooling import MaxPooling2D\n",
        "from keras.layers.merge import Concatenate\n",
        "from keras.layers.core import Lambda, Flatten, Dense\n",
        "from keras.initializers import glorot_uniform\n",
        "from keras.layers import Input, Dense, Flatten, GlobalMaxPool2D, GlobalAvgPool2D, Concatenate, Multiply, Dropout, Subtract, Add, Conv2D\n",
        "\n",
        "import matplotlib.pyplot as plt\n",
        "import seaborn as sns\n",
        "%matplotlib inline\n",
        "\n",
        "import warnings\n",
        "warnings.filterwarnings('ignore')"
      ],
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 195
        },
        "id": "5BlS2QUJfEJr",
        "outputId": "672f58c4-1f65-4c77-f606-921172517abe"
      },
      "source": [
        "df = pd.read_csv('tweets.csv', encoding='utf-8', error_bad_lines=False, sep=',')\n",
        "df.drop(['keyword', 'location', 'id'], axis=1, inplace=True)\n",
        "display(df.sample(5))"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>text</th>\n",
              "      <th>target</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>7814</th>\n",
              "      <td>I say this is as if wraith WOULDNT absolutely ...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3095</th>\n",
              "      <td>Titan: My whole life is a lie! I just found ou...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3952</th>\n",
              "      <td>The streets of Iran are filled with thankful p...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6208</th>\n",
              "      <td>Goals this season👑 Rashford 19 Martial 11 Gree...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5900</th>\n",
              "      <td>🛑 Democrats are flooding our country with ILLE...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                                   text  target\n",
              "7814  I say this is as if wraith WOULDNT absolutely ...       0\n",
              "3095  Titan: My whole life is a lie! I just found ou...       0\n",
              "3952  The streets of Iran are filled with thankful p...       0\n",
              "6208  Goals this season👑 Rashford 19 Martial 11 Gree...       0\n",
              "5900  🛑 Democrats are flooding our country with ILLE...       0"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "immmR1yYfEJs",
        "outputId": "920348d5-cfe6-4e7b-e60b-96beeeb00cad"
      },
      "source": [
        "df.shape"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(11370, 2)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "fbSWwM-mfEJs",
        "outputId": "d5053600-0c30-475d-d566-368671bde93d"
      },
      "source": [
        "df['target'].value_counts()"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0    9256\n",
              "1    2114\n",
              "Name: target, dtype: int64"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "GwGBCrznfEJs"
      },
      "source": [
        "## Data Cleaning"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "poC0Gk90fEJt"
      },
      "source": [
        "def remove_url(text):\n",
        "    url = re.compile(r'https?://\\S+|www\\.\\S+')\n",
        "    return url.sub(r'', text)\n",
        "\n",
        "def remove_emoji(text):\n",
        "    emoji_pattern = re.compile(\n",
        "        '['\n",
        "        u'\\U0001F600-\\U0001F64F'  # emoticons\n",
        "        u'\\U0001F300-\\U0001F5FF'  # symbols & pictographs\n",
        "        u'\\U0001F680-\\U0001F6FF'  # transport & map symbols\n",
        "        u'\\U0001F1E0-\\U0001F1FF'  # flags (iOS)\n",
        "        u'\\U00002702-\\U000027B0'\n",
        "        u'\\U000024C2-\\U0001F251'\n",
        "        ']+',\n",
        "        flags=re.UNICODE)\n",
        "    return emoji_pattern.sub(r'', text)\n",
        "\n",
        "def remove_html(text):\n",
        "    html = re.compile(r'<.*?>|&([a-z0-9]+|#[0-9]{1,6}|#x[0-9a-f]{1,6});')\n",
        "    return re.sub(html, '', text)\n",
        "\n",
        "def remove_punct(text):\n",
        "    table = str.maketrans('', '', string.punctuation)\n",
        "    return text.translate(table)\n",
        "\n",
        "df['clean_text'] = df['text'].apply(lambda x: remove_url(str(x)))\n",
        "df['clean_text'] = df['clean_text'].apply(lambda x: remove_emoji(str(x)))\n",
        "df['clean_text'] = df['clean_text'].apply(lambda x: remove_html(str(x)))\n",
        "df['clean_text'] = df['clean_text'].apply(lambda x: remove_punct(str(x)))\n",
        "df['clean_text'] = df['clean_text'].apply(word_tokenize)\n",
        "df['clean_text'] = df['clean_text'].apply(lambda x: [word.lower() for word in x])\n",
        "df['clean_text'] = df['clean_text'].apply(lambda x: [word for word in x if word not in stop])\n",
        "df['clean_text'] = df['clean_text'].apply(nltk.tag.pos_tag)\n",
        "\n",
        "def get_wordnet_pos(tag):\n",
        "    if tag.startswith('J'):\n",
        "        return wordnet.ADJ\n",
        "    elif tag.startswith('V'):\n",
        "        return wordnet.VERB\n",
        "    elif tag.startswith('N'):\n",
        "        return wordnet.NOUN\n",
        "    elif tag.startswith('R'):\n",
        "        return wordnet.ADV\n",
        "    else:\n",
        "        return wordnet.NOUN\n",
        "    \n",
        "df['clean_text'] = df['clean_text'].apply(\n",
        "    lambda x: [(word, get_wordnet_pos(pos_tag)) for (word, pos_tag) in x])\n",
        "wnl = WordNetLemmatizer()\n",
        "df['clean_text'] = df['clean_text'].apply(\n",
        "    lambda x: [wnl.lemmatize(word, tag) for word, tag in x])\n",
        "df['clean_text'] = df['clean_text'].apply(\n",
        "    lambda x: [word for word in x if word not in stop])\n",
        "df['clean_text'] = [' '.join(map(str, l)) for l in df['clean_text']]"
      ],
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 106
        },
        "id": "qVQaKcaIfEJt",
        "outputId": "e6bf6f48-d98a-4e98-b6e5-031586e1190a"
      },
      "source": [
        "display(df.sample(2))"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>text</th>\n",
              "      <th>target</th>\n",
              "      <th>clean_text</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>5202</th>\n",
              "      <td>If you convert one kilogram of mass into energ...</td>\n",
              "      <td>0</td>\n",
              "      <td>convert one kilogram mass energy create 000009...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9357</th>\n",
              "      <td>Bushfire smoke plume destined to reach Austral...</td>\n",
              "      <td>1</td>\n",
              "      <td>bushfire smoke plume destine reach australia c...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                                   text  ...                                         clean_text\n",
              "5202  If you convert one kilogram of mass into energ...  ...  convert one kilogram mass energy create 000009...\n",
              "9357  Bushfire smoke plume destined to reach Austral...  ...  bushfire smoke plume destine reach australia c...\n",
              "\n",
              "[2 rows x 3 columns]"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "oUTDMAjxIEy_"
      },
      "source": [
        "# df.to_csv('tweets_processed.csv')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "SjdsQ4-oeY3m"
      },
      "source": [
        "## Data Preparation"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wn6npGixedaN"
      },
      "source": [
        "disaster_initial = list(df[df['target'] == 1]['clean_text'])\n",
        "non_disaster_initial = list(df[df['target'] == 0]['clean_text'])\n",
        "\n",
        "# Selecting only 20 samples for siamese model\n",
        "disaster = disaster_initial[:20]\n",
        "non_disaster = non_disaster_initial[:20]\n",
        "\n",
        "# Creating pairs of data for siamese training => label 1 if pairs from same class otherwise 0\n",
        "df2 = pd.DataFrame(columns=['text1', 'text2', 'label'])\n",
        "\n",
        "for data in disaster:\n",
        "  data1 = data\n",
        "  data2 = random.choice(disaster)\n",
        "  data3 = random.choice(non_disaster)\n",
        "\n",
        "  df2.loc[len(df2)] = [data1, data2, 1]\n",
        "  df2.loc[len(df2)] = [data1, data3, 0]\n",
        "\n",
        "\n",
        "for data in non_disaster:\n",
        "  data1 = data\n",
        "  data2 = random.choice(non_disaster)\n",
        "  data3 = random.choice(disaster)\n",
        "  \n",
        "  df2.loc[len(df2)] = [data1, data2, 1]\n",
        "  df2.loc[len(df2)] = [data1, data3, 0]\n"
      ],
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "EvVkio8nm6DK",
        "outputId": "63f1a553-d14d-4128-c8eb-7cb1576951c3"
      },
      "source": [
        "df2.shape"
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(80, 3)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 11
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 195
        },
        "id": "vMaxZO6Tec61",
        "outputId": "49c4fdf2-ad2e-49f5-d2ee-ea83e560b007"
      },
      "source": [
        "display(df2.sample(5))"
      ],
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>text1</th>\n",
              "      <th>text2</th>\n",
              "      <th>label</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>41</th>\n",
              "      <td>lord jesus love brings freedom pardon fill hol...</td>\n",
              "      <td>turn blind eye icident set ablaze 50 house h…</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>31</th>\n",
              "      <td>another arson njikomboyonwr ambazombies yester...</td>\n",
              "      <td>amen set whole system ablaze man</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>telangana section 144 impose bhainsa january 1...</td>\n",
              "      <td>arsonist set car ablaze dealership</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>62</th>\n",
              "      <td>kind true sadly</td>\n",
              "      <td>back neck still fuck accident</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>arsonist set car ablaze dealership</td>\n",
              "      <td>national security minister kan dapaahs side ch...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                                text1  ... label\n",
              "41  lord jesus love brings freedom pardon fill hol...  ...     0\n",
              "31  another arson njikomboyonwr ambazombies yester...  ...     0\n",
              "2   telangana section 144 impose bhainsa january 1...  ...     1\n",
              "62                                    kind true sadly  ...     1\n",
              "7                  arsonist set car ablaze dealership  ...     0\n",
              "\n",
              "[5 rows x 3 columns]"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "WcxiqrHCfEJu"
      },
      "source": [
        "## Data Split"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "plFj4OyEfEJu",
        "outputId": "94266728-6a10-42d5-866b-4ac51aedccba"
      },
      "source": [
        "X_train, X_val, y_train, y_val = train_test_split(df2[['text1', 'text2']], df2['label'], test_size=0.2, random_state=0)\n",
        "print(X_train.shape, X_val.shape, y_train.shape, y_val.shape)"
      ],
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "(64, 2) (16, 2) (64,) (16,)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8sj77LIvfEJu"
      },
      "source": [
        "X_train['text'] = X_train[['text1', 'text2']].apply(lambda x: str(x[0])+\" \"+str(x[1]), axis=1)"
      ],
      "execution_count": 14,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0D8iSVUxxbos"
      },
      "source": [
        "## Glove Embeddings"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rngd1uXGfEJu"
      },
      "source": [
        "t = Tokenizer()\n",
        "t.fit_on_texts(X_train['text'].values)\n",
        "\n",
        "X_train['text1'] = X_train['text1'].astype(str)\n",
        "X_train['text2'] = X_train['text2'].astype(str)\n",
        "X_val['text1'] = X_val['text1'].astype(str)\n",
        "X_val['text2'] = X_val['text2'].astype(str)\n",
        "\n",
        "train_q1_seq = t.texts_to_sequences(X_train['text1'].values)\n",
        "train_q2_seq = t.texts_to_sequences(X_train['text2'].values)\n",
        "val_q1_seq = t.texts_to_sequences(X_val['text1'].values)\n",
        "val_q2_seq = t.texts_to_sequences(X_val['text2'].values)\n",
        "\n",
        "max_len = 200\n",
        "train_q1_seq = pad_sequences(train_q1_seq, maxlen=max_len, padding='post')\n",
        "train_q2_seq = pad_sequences(train_q2_seq, maxlen=max_len, padding='post')\n",
        "val_q1_seq = pad_sequences(val_q1_seq, maxlen=max_len, padding='post')\n",
        "val_q2_seq = pad_sequences(val_q2_seq, maxlen=max_len, padding='post')"
      ],
      "execution_count": 15,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "zPz4Swc2fEJv",
        "outputId": "8b16433c-74d9-4efb-b6f6-cd6b3fa40bb7"
      },
      "source": [
        "#https://blog.keras.io/using-pre-trained-word-embeddings-in-a-keras-model.html\n",
        "embeddings_index = {}\n",
        "f = open('drive/My Drive/Glove/glove.6B.300d.txt')\n",
        "for line in f:\n",
        "    values = line.split()\n",
        "    word = values[0]\n",
        "    coefs = np.asarray(values[1:], dtype='float32')\n",
        "    embeddings_index[word] = coefs\n",
        "f.close()\n",
        "\n",
        "print('Found %s word vectors.' % len(embeddings_index))"
      ],
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Found 400000 word vectors.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "LTdyVvzrfEJv",
        "outputId": "92b256f5-a4a7-4917-9d35-73b7d448c83b"
      },
      "source": [
        "not_present_list = []\n",
        "vocab_size = len(t.word_index) + 1\n",
        "print('Loaded %s word vectors.' % len(embeddings_index))\n",
        "embedding_matrix = np.zeros((vocab_size, len(embeddings_index['no'])))\n",
        "for word, i in t.word_index.items():\n",
        "    if word in embeddings_index.keys():\n",
        "        embedding_vector = embeddings_index.get(word)\n",
        "    else:\n",
        "        not_present_list.append(word)\n",
        "    if embedding_vector is not None:\n",
        "        embedding_matrix[i] = embedding_vector\n",
        "    else:\n",
        "        embedding_matrix[i] = np.zeros(300)"
      ],
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Loaded 400000 word vectors.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "5q9WJ8GhfEJv",
        "outputId": "529d1ad3-affd-4cf8-b4de-2815c5d91b69"
      },
      "source": [
        "print(embedding_matrix.shape)"
      ],
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "(317, 300)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "EIuQ9I6PxT-6"
      },
      "source": [
        "## Siamese Model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YAhYNYXbfEJw"
      },
      "source": [
        "def euclidean_distance(vectors):\n",
        "    # unpack the vectors into separate lists\n",
        "    (featsA, featsB) = vectors\n",
        "    # compute the sum of squared distances between the vectors\n",
        "    sumSquared = K.sum(K.square(featsA - featsB), axis=1, keepdims=True)\n",
        "    # return the euclidean distance between the vectors\n",
        "    return K.sqrt(K.maximum(sumSquared, K.epsilon()))\n",
        "\n",
        "def contrastive_loss(y, preds, margin=1):\n",
        "    # explicitly cast the true class label data type to the predicted\n",
        "    # class label data type (otherwise we run the risk of having two\n",
        "    # separate data types, causing TensorFlow to error out)\n",
        "    y = tf.cast(y, preds.dtype)\n",
        "    # calculate the contrastive loss between the true labels and\n",
        "    # the predicted labels\n",
        "    squaredPreds = K.square(preds)\n",
        "    squaredMargin = K.square(K.maximum(margin - preds, 0))\n",
        "    loss = K.mean(y * squaredPreds + (1 - y) * squaredMargin)\n",
        "    # return the computed contrastive loss to the calling function\n",
        "    return loss"
      ],
      "execution_count": 19,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "N2FZDhh-Q5ii"
      },
      "source": [
        "def build_network():\n",
        "\n",
        "  network = Sequential()\n",
        "  network.add(Embedding(name=\"synopsis_embedd\",input_dim =len(t.word_index)+1, \n",
        "                       output_dim=len(embeddings_index['no']),weights=[embedding_matrix], \n",
        "                       input_length=train_q1_seq.shape[1],trainable=False))\n",
        "  network.add(LSTM(64,return_sequences=True, activation=\"relu\"))\n",
        "  network.add(Flatten())\n",
        "  network.add(Dense(4096, activation='relu',\n",
        "                  kernel_regularizer=l2(1e-3),\n",
        "                  kernel_initializer='he_uniform'))\n",
        "  \n",
        "  network.add(Dense(317, activation=None,\n",
        "                  kernel_regularizer=l2(1e-3),\n",
        "                  kernel_initializer='he_uniform'))\n",
        "  \n",
        "  #Force the encoding to live on the d-dimentional hypershpere\n",
        "  network.add(Lambda(lambda x: K.l2_normalize(x,axis=-1)))\n",
        "\n",
        "  return network"
      ],
      "execution_count": 59,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4kR00lIaR1Q7"
      },
      "source": [
        "input_1 = Input(shape=(train_q1_seq.shape[1],))\n",
        "input_2 = Input(shape=(train_q2_seq.shape[1],))\n",
        "\n",
        "network = build_network()\n",
        "\n",
        "encoded_input_1 = network(input_1)\n",
        "encoded_input_2 = network(input_2)\n",
        "\n",
        "distance = Lambda(euclidean_distance)([encoded_input_1, encoded_input_2])\n",
        "\n",
        "# Connect the inputs with the outputs\n",
        "model = Model([input_1, input_2], distance)\n",
        "\n",
        "model.compile(loss=contrastive_loss, optimizer=Adam(0.001))"
      ],
      "execution_count": 60,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JKfZKgGQpTVk"
      },
      "source": [
        "y_train = np.asarray(y_train).astype('float32')\n",
        "y_val = np.asarray(y_val).astype('float32')"
      ],
      "execution_count": 47,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "H5pi0g8kfEJw",
        "outputId": "d2474cc1-8c5b-416e-d6b2-aa430c6aa2c8"
      },
      "source": [
        "model.fit([train_q1_seq,train_q2_seq],y_train.reshape(-1,1), epochs = 5, \n",
        "          batch_size=64,validation_data=([val_q1_seq, val_q2_seq],y_val.reshape(-1,1)))"
      ],
      "execution_count": 48,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch 1/5\n",
            "1/1 [==============================] - 5s 5s/step - loss: 9.1177 - val_loss: 8.2265\n",
            "Epoch 2/5\n",
            "1/1 [==============================] - 2s 2s/step - loss: 8.3272 - val_loss: 7.3562\n",
            "Epoch 3/5\n",
            "1/1 [==============================] - 2s 2s/step - loss: 7.4295 - val_loss: 6.5293\n",
            "Epoch 4/5\n",
            "1/1 [==============================] - 2s 2s/step - loss: 6.5605 - val_loss: 5.7697\n",
            "Epoch 5/5\n",
            "1/1 [==============================] - 2s 2s/step - loss: 5.7392 - val_loss: 5.1012\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tensorflow.python.keras.callbacks.History at 0x7f4ab5289590>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 48
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "WtbfF-TAgg6y",
        "outputId": "fa87c78d-fd6c-4286-f110-41ab13d02106"
      },
      "source": [
        "# Save model for further use\n",
        "# serialize model to JSON\n",
        "model_json = model.to_json()\n",
        "with open(\"siamesemodel-contrastive-loss.json\", \"w\") as json_file:\n",
        "    json_file.write(model_json)\n",
        "#serialize weights to HDF5\n",
        "model.save_weights(\"siamesemodel-contrastive-loss.h5\")\n",
        "print(\"Saved model to disk\")\n",
        "\n",
        "# load json and create model\n",
        "# json_file = open('siamesemodel-contrastive-loss.json', 'r')\n",
        "# loaded_model_json = json_file.read()\n",
        "# json_file.close()\n",
        "# loaded_model = model_from_json(loaded_model_json)\n",
        "# # load weights into new model\n",
        "# loaded_model.load_weights(\"siamesemodel-contrastive-loss.h5\")\n",
        "# print(\"Loaded model from disk\")"
      ],
      "execution_count": 49,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Saved model to disk\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "DwpkqVdzz_Rt"
      },
      "source": [
        "## Prediction"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "BMsExu7ifjIi"
      },
      "source": [
        "### Non Disaster Tweet"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "id": "L6D4KUQTg-hh",
        "outputId": "73006f41-d252-4ab9-a1dd-5225c15a2cac"
      },
      "source": [
        "non_disaster_initial[122]"
      ],
      "execution_count": 50,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'usdjpy despite richter scale overbought fxstreet line swissquote bullish long jason sen mr ambulance chaser b…'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 50
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "knG_To6i0CMq",
        "outputId": "000eeb25-f3b1-4157-a62a-902bb184afa6"
      },
      "source": [
        "prediction_data = \"usdjpy despite richter scale overbought fxstreet line swissquote bullish long jason sen mr ambulance chaser b…\"\n",
        "prediction_vector = t.texts_to_sequences([prediction_data])\n",
        "prediction_vector = pad_sequences(prediction_vector,maxlen=200)\n",
        "\n",
        "assistant_data = disaster[11]\n",
        "assistant_vector = t.texts_to_sequences([assistant_data])\n",
        "assistant_vector = pad_sequences(assistant_vector,maxlen=200)\n",
        "\n",
        "model.predict([prediction_vector, assistant_vector])"
      ],
      "execution_count": 51,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([[0.49869883]], dtype=float32)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 51
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "4Jo-NwC94mw4",
        "outputId": "66cf0c30-35c8-4540-b3d7-c9d831580677"
      },
      "source": [
        "prediction_data = \"usdjpy despite richter scale overbought fxstreet line swissquote bullish long jason sen mr ambulance chaser b…\"\n",
        "prediction_vector = t.texts_to_sequences([prediction_data])\n",
        "prediction_vector = pad_sequences(prediction_vector,maxlen=200)\n",
        "\n",
        "assistant_data = non_disaster[11]\n",
        "assistant_vector = t.texts_to_sequences([assistant_data])\n",
        "assistant_vector = pad_sequences(assistant_vector,maxlen=200)\n",
        "\n",
        "model.predict([prediction_vector, assistant_vector])"
      ],
      "execution_count": 52,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([[0.36637673]], dtype=float32)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 52
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "BLTP3mr-fv7y"
      },
      "source": [
        "### Disaster Tweet"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "id": "aF9S_pjs5hFa",
        "outputId": "7adaed8d-7bdd-45ba-cd98-2baff7a12e69"
      },
      "source": [
        "disaster_initial[12]"
      ],
      "execution_count": 53,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'cameroon bir soldier 05012020 invaded southerncameroons village kimar set ablaze total of…'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 53
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Yvbfdu5Rf1iK",
        "outputId": "34136a77-4b1f-4b75-f02f-5d2e34f73bed"
      },
      "source": [
        "prediction_data = \"cameroon bir soldier 05012020 invaded southerncameroons village kimar set ablaze total of\"\n",
        "prediction_vector = t.texts_to_sequences([prediction_data])\n",
        "prediction_vector = pad_sequences(prediction_vector,maxlen=200)\n",
        "\n",
        "assistant_data = disaster[11]\n",
        "assistant_vector = t.texts_to_sequences([assistant_data])\n",
        "assistant_vector = pad_sequences(assistant_vector,maxlen=200)\n",
        "\n",
        "model.predict([prediction_vector, assistant_vector])"
      ],
      "execution_count": 54,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([[0.38583857]], dtype=float32)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 54
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Dn3xt_bDf1aa",
        "outputId": "a71b6a0c-7932-4f53-afc7-6a06bd4c1dc8"
      },
      "source": [
        "prediction_data = \"cameroon bir soldier 05012020 invaded southerncameroons village kimar set ablaze total of\"\n",
        "prediction_vector = t.texts_to_sequences([prediction_data])\n",
        "prediction_vector = pad_sequences(prediction_vector,maxlen=200)\n",
        "\n",
        "assistant_data = non_disaster[11]\n",
        "assistant_vector = t.texts_to_sequences([assistant_data])\n",
        "assistant_vector = pad_sequences(assistant_vector,maxlen=200)\n",
        "\n",
        "model.predict([prediction_vector, assistant_vector])"
      ],
      "execution_count": 55,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([[0.46985233]], dtype=float32)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 55
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "h-1uxiHPf-tb"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}