get_relevant_documents.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zZ5gIEli6c1C"
      },
      "source": [
        "This notebook aims at retrieving documents relevant for the example use case. <br>\n",
        "Usage of Corex based on following tutorial: https://github.com/gregversteeg/corex_topic/blob/master/corextopic/example/corex_topic_example.ipynb"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 58,
      "metadata": {
        "id": "80W2SSh-3DhK"
      },
      "outputs": [],
      "source": [
        "import pandas as pd\n",
        "import sklearn\n",
        "from langdetect import detect\n",
        "import numpy as np\n",
        "import scipy.sparse as ss\n",
        "import matplotlib.pyplot as plt\n",
        "import seaborn as sns\n",
        "import corextopic.corextopic as ct\n",
        "import corextopic.vis_topic as vt\n",
        "from sklearn.feature_extraction.text import CountVectorizer\n",
        "%matplotlib inline\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "V312WP4HInP8"
      },
      "source": [
        "##I. Prepare dataset"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 8,
      "metadata": {
        "id": "Kuh5_CVf3XzP"
      },
      "outputs": [],
      "source": [
        "# path to csv dataset created with dataset_preparation.ipynb\n",
        "df = pd.read_csv(' ')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 12,
      "metadata": {
        "id": "EHzBExNh4A8N"
      },
      "outputs": [],
      "source": [
        "df['language'] = df['texts'].apply(lambda x: detect(x))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 13,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 387
        },
        "id": "Svobgsec4NOr",
        "outputId": "8a3996cf-9d5a-48f6-ff26-61b4572a200c"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "<seaborn.axisgrid.FacetGrid at 0x7f6bbf3fd890>"
            ]
          },
          "execution_count": 13,
          "metadata": {},
          "output_type": "execute_result"
        },
        {
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAWAAAAFgCAYAAACFYaNMAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAZD0lEQVR4nO3de5SlVX3m8e8DDUK8NUjbg00zoLK8R8QW74nKMiOoARMEJyqtg7YT8RZHMzg6K86MmYVjEm9xULwENEZAlEWLBkVEjDdiI3cQ6UFId4vQKhIRjYK/+ePslkNbl1PVfWpXV30/a9U677vfvd93n7fqPPXWPufdlapCkjT3durdAUlarAxgSerEAJakTgxgSerEAJakTgxgSepkrAGcZGmSM5J8J8nVSZ6YZM8k5ya5tj3u0eomyXuSrE9yWZKDxtk3Sept3FfA7wbOqaqHAo8GrgaOB86rqgOA89o6wKHAAe1rDXDimPsmSV1lXDdiJLkvcAnwwBo6SJJrgKdV1Y1J9ga+XFUPSfKBtvyJreuNpYOS1NmSMe57f2Az8HdJHg1cBLwWWD4Uqj8AlrflFcCGofYbW9ndAjjJGgZXyDz84Q9/7JVXXjm2JyBJ20kmKhznEMQS4CDgxKp6DPAz7hpuAKBdGc/oEryqTqqqVVW1avfdd99unZWkuTbOAN4IbKyqC9v6GQwC+aY29EB7vLlt3wSsHGq/TyuTpAVpbAFcVT8ANiR5SCs6BLgKWAusbmWrgbPa8lrgmPZpiCcAtzr+K2khG+cYMMCrgY8n2RW4Dngpg9A/PcmxwA3AUa3u54DDgPXA7a2uJC1YYw3gqroEWDXBpkMmqFvAcePsjyTNJ94JJ0mdGMCS1IkBLEmdGMCS1IkBLEmdGMCS1IkBLEmdGMCS1MmiDOAVK/clyYy+Vqzct3e3JS0w474VeV76/sYNHP2Br8+ozWmveNKYeiNpsVqUV8CSNB8YwJLUiQEsSZ0YwJLUiQEsSZ0YwJLUiQEsSZ0YwJLUiQEsSZ0YwJLUiQEsSZ0YwJLUiQEsSZ0YwJLUiQEsSZ0YwJLUiQEsSZ0YwJLUiQEsSZ0YwJLUiQEsSZ0YwJLUiQEsSZ0YwJLUiQEsSZ0YwJLUiQEsSZ0YwJLUiQEsSZ0YwJLUiQEsSZ0YwJLUyVgDOMn1SS5PckmSda1szyTnJrm2Pe7RypPkPUnWJ7ksyUHj7Jsk9TYXV8BPr6oDq2pVWz8eOK+qDgDOa+sAhwIHtK81wIlz0DdJ6qbHEMThwClt+RTgiKHyj9bAN4GlSfbu0D9JmhPjDuACvpDkoiRrWtnyqrqxLf8AWN6WVwAbhtpubGV3k2RNknVJ1m3evHlc/ZaksVsy5v0/pao2Jbk/cG6S7wxvrKpKUjPZYVWdBJwEsGrVqhm1laT5ZKxXwFW1qT3eDJwJHAzctGVooT3e3KpvAlYONd+nlUnSgjS2AE5yzyT33rIM/AFwBbAWWN2qrQbOastrgWPapyGeANw6NFQhSQvOOIcglgNnJtlynH+oqnOSfAs4PcmxwA3AUa3+54DDgPXA7cBLx9g3SepubAFcVdcBj56g/EfAIROUF3DcuPojSfONd8JJUicGsCR1YgBLUicGsCR1YgBLUicGsCR1YgBLUicGsCR1YgBLUicGsCR1YgBLUicGsCR1YgBLUicGsCR1YgBLUicGsCR1YgBLUicGsCR1YgBLUicGsCR1YgBLUicGsCR1YgBLUicGsCR1YgBLUicGsCR1YgBLUicGsCR1YgBLUicGsCR1YgBLUicGsCR1YgBLUicGsCR1YgBLUicGsCR1YgBLUicGsCR1YgBLUicGsCR1YgBLUidjD+AkOye5OMnZbX3/JBcmWZ/ktCS7tvJ7tPX1bft+4+6bJPU0F1fArwWuHlp/O/DOqnowcAtwbCs/Frillb+z1ZOkBWusAZxkH+DZwIfaeoBnAGe0KqcAR7Tlw9s6bfshrb4kLUjjvgJ+F/DnwK/b+v2An1TVHW19I7CiLa8ANgC07be2+pK0II0tgJM8B7i5qi7azvtdk2RdknWbN2/enruWpDk1zivgJwN/mOR64FQGQw/vBpYmWdLq7ANsasubgJUAbft9gR9tvdOqOqmqVlXVqmXLlo2x+5I0XmML4Kp6U1XtU1X7AS8AvlRVLwTOB45s1VYDZ7XltW2dtv1LVVXj6p8k9dbjc8D/FXh9kvUMxng/3Mo/DNyvlb8eOL5D3yRpziyZvsq2q6ovA19uy9cBB09Q5xfA8+eiP5I0H3gnnCR1YgBLUicGsCR1YgBLUicGsCR1YgBLUicGsCR1YgBLUicGsCR1YgBLUicGsCR1YgBLUicGsCR1YgBLUicGsCR1YgBLUicGsCR1YgBLUicGsCR1YgBLUicGsCR1YgBLUicGsCR1YgBLUicGsCR1YgBLUicGsCR1YgBLUicGsCR1YgBLUicGsCR1YgBLUicjBXCSJ49SJkka3ahXwO8dsUySNKIlU21M8kTgScCyJK8f2nQfYOdxdkySFropAxjYFbhXq3fvofJ/BY4cV6ckaTGYMoCr6gLggiQnV9UNc9QnSVoUprsC3uIeSU4C9htuU1XPGEenJGkxGDWAPwm8H/gQcOf4uiNJi8eoAXxHVZ041p5I0iIz6sfQPpPklUn2TrLnlq+x9kySFrhRr4BXt8c3DpUV8MDt2x1JWjxGCuCq2n+mO06yG/AV4B7tOGdU1V8k2R84FbgfcBHw4qr6ZZJ7AB8FHgv8CDi6qq6f6XElaUcxUgAnOWai8qr66BTN/g14RlXdlmQX4KtJ/hF4PfDOqjo1yfuBY4ET2+MtVfXgJC8A3g4cPYPnIkk7lFHHgB839PVU4K3AH07VoAZua6u7tK8CngGc0cpPAY5oy4e3ddr2Q5JkxP5J0g5n1CGIVw+vJ1nKYBhhSkl2ZjDM8GDgfcD/A35SVXe0KhuBFW15BbChHe+OJLcyGKb44Vb7XAOsAdh3331H6b4kzUuznY7yZ8C048JVdWdVHQjsAxwMPHSWxxve50lVtaqqVi1btmxbdydJ3Yw6BvwZBsMHMJiE52HA6aMepKp+kuR84InA0iRL2lXwPsCmVm0TsBLYmGQJcF8Gb8ZJ0oI06sfQ/mpo+Q7ghqraOFWDJMuAX7Xw3R14JoM31s5nMJHPqQw+3nZWa7K2rX+jbf9SVdVv7ViSFohRx4AvSLKcwZtwANeO0Gxv4JQ2DrwTcHpVnZ3kKuDUJG8DLgY+3Op/GPhYkvXAj4EXzOB5SNIOZ9QhiKOAdwBfBgK8N8kbq+qMydpU1WXAYyYov47BePDW5b8Anj9atyVpxzfqEMSbgcdV1c3wm+GFL3LXx8kkSTM06qcgdtoSvs2PZtBWkjSBUa+Az0nyeeATbf1o4HPj6ZIkLQ7T/U+4BwPLq+qNSf4IeErb9A3g4+PunCQtZNNdAb8LeBNAVX0a+DRAkke1bc8da+8kaQGbbhx3eVVdvnVhK9tvLD2SpEViugBeOsW23bdnRyRpsZkugNclefnWhUlexmCSHUnSLE03Bvw64MwkL+SuwF0F7Ao8b5wdk6SFbsoArqqbgCcleTrwyFb82ar60th7JkkL3KhzQZzPYBIdSdJ24t1sktSJASxJnRjAktSJASxJnRjAktSJASxJnRjAktSJASxJnRjAktSJASxJnRjAktSJASxJnRjAktSJASxJnRjAktSJASxJnRjAktSJASxJnRjAktSJASxJnRjAktSJASxJnRjAktSJASxJnRjAktSJASxJnRjAktSJASxJnRjAktSJASxJnYwtgJOsTHJ+kquSXJnkta18zyTnJrm2Pe7RypPkPUnWJ7ksyUHj6pskzQfjvAK+A/gvVfVw4AnAcUkeDhwPnFdVBwDntXWAQ4ED2tca4MQx9k2SuhtbAFfVjVX17bb8U+BqYAVwOHBKq3YKcERbPhz4aA18E1iaZO9x9U+SepuTMeAk+wGPAS4EllfVjW3TD4DlbXkFsGGo2cZWJkkL0tgDOMm9gE8Br6uqfx3eVlUF1Az3tybJuiTrNm/evB17Kklza6wBnGQXBuH78ar6dCu+acvQQnu8uZVvAlYONd+nld1NVZ1UVauqatWyZcvG13lJGrNxfgoiwIeBq6vqb4Y2rQVWt+XVwFlD5ce0T0M8Abh1aKhCkhacJWPc95OBFwOXJ7mklf034ATg9CTHAjcAR7VtnwMOA9YDtwMvHWPfJKm7sQVwVX0VyCSbD5mgfgHHjas/kjTfeCecJHViAEtSJwawJHViAEtSJwawJHViAEtSJwawJHViAEtSJwawJHViAEtSJwawJHViAEtSJwawJHViAEtSJwawJHViAEtSJwawJHViAEtSJwawJHViAEtSJwawJHViAEtSJwawJHViAEtSJwawJHViAEtSJwawJHViAEtSJwawJHViAEtSJwawJHViAEtSJwawJHViAEtSJwawJHViAEtSJwawJHViAEtSJwawJHViAEtSJwawJHViAEtSJ2ML4CQfSXJzkiuGyvZMcm6Sa9vjHq08Sd6TZH2Sy5IcNK5+SdJ8Mc4r4JOBZ21VdjxwXlUdAJzX1gEOBQ5oX2uAE8fYL0maF8YWwFX1FeDHWxUfDpzSlk8Bjhgq/2gNfBNYmmTvcfVNkuaDuR4DXl5VN7blHwDL2/IKYMNQvY2t7LckWZNkXZJ1mzdvHl9PJWnMur0JV1UF1CzanVRVq6pq1bJly8bQM0maG3MdwDdtGVpojze38k3AyqF6+7QySVqw5jqA1wKr2/Jq4Kyh8mPapyGeANw6NFQhSQvSknHtOMkngKcBeyXZCPwFcAJwepJjgRuAo1r1zwGHAeuB24GXjqtfkjRfjC2Aq+o/TrLpkAnqFnDcuPoiSfORd8JJUicGsCR1YgBLUicGsCR1YgBLUicGsCR1YgBLUicGsCR1YgBLUicGsCR1YgBLUicGsCR1YgBLUicGsCR1YgBLUicGsCR1YgBLUicGsCR1YgBLUicGsCR1YgBLUicGsCR1YgBLUicG8Kh2WkKSGX+tWLlv755LmqeW9O7ADuPXd3D0B74+42anveJJY+iMpIXAK2BJ6sQAlqRODGBJ6sQAnodWrNx3zt7wm8tjSbo734Sbh76/ccOcveE3l8eSdHdeAUtSJwawJHViAEtSJwawJHViAEtSJwawJHViAEtSJwawJHViAGtOzebOO++600LlnXDj1uYR1sBs7rzzrrvtY8XKffn+xg0zavOAfVayacO/jKlHMoDHbRbzCBs4Ggd/+c0/DkFo/pvD/0Yy28mJluy625y0cUhmYZlXV8BJngW8G9gZ+FBVndC5S5rMXA6tzPa/kfzp782qj7OdnGg2V5dz+bw0/8ybAE6yM/A+4JnARuBbSdZW1VV9e6YJ7Qj/ommhDv/sAOd+Lsebd+Sx7XkTwMDBwPqqug4gyanA4YABPCrf8NP2tg0/UzP+5bcNV/ZzdaztHdypqu22s22R5EjgWVX1srb+YuDxVfWqreqtAda01YcA12zHbuwF/HCet/NY/dp5rH7tdoRjTeWHVfWsrQvn0xXwSKrqJOCkcew7ybqqWjWf23msfu08Vr92O8KxZmM+fQpiE7ByaH2fViZJC9J8CuBvAQck2T/JrsALgLWd+yRJYzNvhiCq6o4krwI+z+BjaB+pqivnuBuzHdqYy3Yeq187j9Wv3Y5wrBmbN2/CSdJiM5+GICRpUTGAJakTA3g7SvK6JL8zTZ2vt8f9kvzJDPb91iRvmEH9pUleOWr9XpK8JsnVST4+R8eb0XmcZB8nt8+tT1Vn5DsD2s/CFdvSp9kYOve3JDl+lvt4QJIzZtn2+iR7zabtBPsayzlM8uUkY/tImgG8fb0OmDKAq2rL/aD7ASMH8CwsBX4rgJOM7Y3XDMz0Z+qVwDOr6oVD+5k3bw7P1tD3eT7bcu73mO28K1X1/aqa8peRJreoAzjJi5L8c5JLknwgyc5Jbkvyl0kuTfLNJMsnaLdfku8k+Xi7gjgjyWuABwDnJzl/imPe1hZPAJ7ajv1nk9R9c5LvJvkqg7v+SPKgJOckuSjJPyV56CSHOgF4UNv/t1rdtUxxa/ck5+PkJFckuXyifrZzcU2SjwJXAHcObTsyyclTHO/9wAOBf0xya5KPJfka8LFJ6t8zyWfb9+aKJKuTfHJo+9OSnD1Bu4nO48vbebk0yacm+sulPberk3wwyZVJvpBk98mez1Ztb0tyryTnJfl2O3+Hj9DugUkuTvK49vN3WZIzk+wxav+SHDhC2+Fz/2dJ/naEvp2Q5Lih9bcmecMoV54T/WxNU/+Y1v9L28/Fc5Nc2M7NFyd6XQI7T3AufnMFm2SvJNdPcryJXtNTXkxtF1W1KL+AhwGfAXZp6/8XOAYo4Lmt7P8Ab5mg7X6t3pPb+keANwDXA3tNc9zb2uPTgLOnqPdY4HIGV9T3Ada3Y5wHHNDqPB740iTt9wOuGDrWz4D9Z3g+/gI4d6jO0kmO82vgCcPPry0fCZw8zfm4nsGtn28FLgJ2n6LuHwMfHFq/L/AvwD3b+onAi0Y8j/cbqvM24NWTPLc7gAPb+unAi4CTgSOn+z4z+Jjnfdr6Xu3Ymex7xeCXw8XAo4HLgN9v2/8n8K4Z9G/atlud+5cAfzvCa+YxwAVD61cBT93yczaL19r1TPB6AR4BfHfLNmBPYI8t5w54GfDXI56LLwOrhr4H10/xepnoNf2b9uP42uH/1NsGhzB4cX4rg0k5dgduBn4JbLmKuojB7GwT2VBVX2vLfw+8Zjv376nAmVV1O0C7et0NeBLwydw1kcg9RtzfP1fV96bYPtH5OAd4YJL3Ap8FvjBJ2xuq6psj9mMqa6vq51Nsvxz46yRvZ/DL65+SnAM8N4NxyGcDf75Vm4nOI8Ajk7yNwVDNvRh8/nwi36uqS9ryRQxeqKMK8L+T/B6DX1IrgOXADyaouww4C/gjBneALq2qC9q2U4BPTtBmov49aAZtZ6SqLk5y/yQPaP29BRhlGrLJXmuTeQbwyar6YTvuj5M8Cjgtyd7ArsBEP8vb8r2C8b+mf8tiDuAAp1TVm+5WmLyh2q9ABn9OT3aOtv4A9Vx8oHon4CdVdeAs2v5smu2TnY83A/8B+M/AUcB/mmbfw+dht+3Zx6r6bpKDgMOAtyU5DzgVeBXwY2BdVf10xGOdDBxRVZcmeQmDvxIm8m9Dy3cyCI9RvZBBUD22qn7V/vyd7JzcyuBq/inAaTM4xtb9WzqDtrPxSQZ/2fw7Ru/nZD9bL5nBcd8L/E1VrU3yNAZ/MW1tou/VHdw11Drdz+Ocv6YX8xjwecCRSe4PkGTPJP9+Bu33TfLEtvwnwFeBnwL3HrH9dHW/AhzRxrHuDTwXuB34XpLntz4nyaNnuf+tTXY+dqqqTwFvAQ4aYT83JXlYBm/GPW8Gx59Wu/K6var+HnhH688F7fHlDMJ4axOdRxicmxuT7MIgKMfhvsDNLXyfDkz18/VLBufrGAZX8rckeWrb9mIGz3MUt25D21GcxmCagCMZ/cp6pq+1LwHPT3K/LfUZnMstc8OsnkF/r2dw9U3r81Qmek2P1aK9Aq6qq5K8BfhCC4tfAcdN02zYNcBxST7CYCzsRAYvonOSfL+qnj5N+8uAO5NcymCc9J1b9e/bSU4DLmXw59q32qYXAie2vu/CIHQuneD5/SjJ19obJD8HbpqqM5Ocj9cDZ+auTza8adId3OV4BkM4m4F1DP68314eBbwjya9b//60qu7M4I23lzDBC3OK8/jfgQtbPy9kZr+sRlHAx4HPJLmcwbn4zpQNqn6W5DnAucCnGDzX3wGuA146g2OvBt4/y7ZTqqor2y+yTVV1Y5L9Rmgzo9daO8ZfAhckuZPBuPhbGQy93cIgoPcfsct/BZyewTS2n52m7kSv6edO3WTbeCvyLLQfurOr6pGdu6J5qF25fbuqZvIXlTrq9ZpezEMQ0nbXhkm+weDKS5qSV8CS1IlXwJLUiQEsSZ0YwJLUiQGsHVLumlND2mEZwJLUiQGsHVommXEsU8xklsFMY5e1mbne0W5WIclLMjQrWJKz222vJDkxybq2r/8xVOewNovWRUne024K2TJz20cymAHs4owwE5oWHwNYO7pfAM+rqoOApzOYrGfLTEUHAO+rqkcAP2EwmxrA3wGvaHNq3Ln1Difx5qpaBfwu8PtJfjfJbsAHgEOr6rEM5n34TX0GM9Ud3Pr1jiT3nP3T1EJkAGtHt2XGscuAL3LXjGMwwexYSZYC966qb7TyfxjxOEcl+TaD22IfATwceChw3dAsc58Yqv8HwPFJLmEwpeFuwL4zfXJa2BbtXBBaMKaacWymM5kNz5zFlv0k2Z/B3LCPq6pbMphkfrqZtQL8cVVdM8qT0OLkFbB2dDOZcYyq+gnw0ySPb0UvGNp8PXBgkp2SrAQObuX3YTBV5q0Z/CeGQ1v5NQzmS96vrR89tK/PA6/eMhyS5DGzeG5a4LwC1o5uRjOONccCH2yzql3AYApHgK8xmOj7KuBq4NsAbc7gi9u+N7R6VNXPM/jHp+ck+Rl3zbQG8L+AdwGXtRnAvgc8Z1ueqBYe54LQopPkXlV1W1s+Hti7ql67LftqV7rvA67dempRaTIOQWgxenb7CNoVDP5l0du2YV8vb2+0XclgOOQD26ODWhy8ApakTrwClqRODGBJ6sQAlqRODGBJ6sQAlqRO/j/ARXQyk0ajXQAAAABJRU5ErkJggg==",
            "text/plain": [
              "<Figure size 360x360 with 1 Axes>"
            ]
          },
          "metadata": {
            "needs_background": "light"
          },
          "output_type": "display_data"
        }
      ],
      "source": [
        "sns.displot(df, x=\"language\", discrete=True)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "YmdGqcACIuQM"
      },
      "source": [
        "Keep texts in English only"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 14,
      "metadata": {
        "id": "8G_FTjhj4Rs3"
      },
      "outputs": [],
      "source": [
        "df = df[df['language'] == 'en']"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 15,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "wNZ-L8wC4UNQ",
        "outputId": "608d73d2-5d04-4650-95ad-ba787b02418a"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "<class 'pandas.core.frame.DataFrame'>\n",
            "Int64Index: 573 entries, 0 to 778\n",
            "Data columns (total 8 columns):\n",
            " #   Column           Non-Null Count  Dtype \n",
            "---  ------           --------------  ----- \n",
            " 0   archive_links    573 non-null    object\n",
            " 1   urls             573 non-null    object\n",
            " 2   record_ids       570 non-null    object\n",
            " 3   content_lengths  573 non-null    int64 \n",
            " 4   warc_dates       573 non-null    object\n",
            " 5   html_metadata    569 non-null    object\n",
            " 6   texts            573 non-null    object\n",
            " 7   language         573 non-null    object\n",
            "dtypes: int64(1), object(7)\n",
            "memory usage: 40.3+ KB\n"
          ]
        }
      ],
      "source": [
        "df.info()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 29,
      "metadata": {
        "id": "aAipvmq1-2g4"
      },
      "outputs": [],
      "source": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2M_HrXwXI6nh"
      },
      "source": [
        "##II. Texts pre-processing"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 30,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "uFw0D2YU-4K1",
        "outputId": "2a101fc8-198d-4cfb-98f7-f69dd46d58de"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "(573, 20000)"
            ]
          },
          "execution_count": 30,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "vectorizer = CountVectorizer(stop_words='english', max_features=20000, binary=True)\n",
        "doc_word = vectorizer.fit_transform(df['texts'])\n",
        "doc_word = ss.csr_matrix(doc_word)\n",
        "\n",
        "#nr of documents, nr of words\n",
        "doc_word.shape "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 32,
      "metadata": {
        "id": "pWTNiSa6_Nrr"
      },
      "outputs": [],
      "source": [
        "words = list(np.asarray(vectorizer.get_feature_names_out()))\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 33,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ZyHdQpNV_mN5",
        "outputId": "9900b05c-244e-4580-fc65-8badedc06000"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "(573, 17720)"
            ]
          },
          "execution_count": 33,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "# remove words that are numbers\n",
        "not_digit_inds = [ind for ind,word in enumerate(words) if not word.isdigit()]\n",
        "doc_word = doc_word[:,not_digit_inds]\n",
        "words    = [word for ind,word in enumerate(words) if not word.isdigit()]\n",
        "\n",
        "# number of documents / nr of words\n",
        "doc_word.shape"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "FNrcGYpIJHKW"
      },
      "source": [
        "## III. Model training"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 34,
      "metadata": {
        "id": "Er56HMBg_qQX"
      },
      "outputs": [],
      "source": [
        "# training with 10 topics\n",
        "topic_model = ct.Corex(n_hidden=10, words=words, max_iter=200, verbose=False, seed=1)\n",
        "topic_model.fit(doc_word, words=words);"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 38,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "hzncaKAN_11E",
        "outputId": "bc992fa6-b7f9-485e-a210-a52c29f1c46f"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "0: retail, entertainment, health, travel, doctors, engineers, towns, photography, regulations, waste\n",
            "1: markets, funds, investments, fund, loss, assets, volatility, research, securities, products\n",
            "2: america, investors, korea, portugal, capital, solutions, belgium, africa, investment, china\n",
            "3: machine, makers, stars, rocket, electricity, foundation, jeff, hollywood, bob, east\n",
            "4: sports, cars, login, hotels, deals, fashion, assembly, tourism, music, books\n",
            "5: ventnor, wa, yearly, esc, perth, pixels, captured, presentations, pixadmin, announcements\n",
            "6: states, united, etfs, return, principal, net, date, advisors, type, expenses\n",
            "7: black, policy, netherlands, ishares, terms, poland, spain, listed, singapore, select\n",
            "8: stop, awards, links, facebook, box, share, park, city, arts, green\n",
            "9: kids, campaign, clubs, free, child, women, bible, stories, peace, vision\n"
          ]
        }
      ],
      "source": [
        "topics = topic_model.get_topics()\n",
        "for n,topic in enumerate(topics):\n",
        "    topic_words,_,_ = zip(*topic)\n",
        "    print('{}: '.format(n) + ', '.join(topic_words))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 39,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "j-ZOR5plAFB5",
        "outputId": "6bc4def2-5f6d-4e1a-d868-0118726499bb"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "NOTE: 'docs' not provided to CorEx. Returning top docs as lists of row indices\n"
          ]
        },
        {
          "data": {
            "text/plain": [
              "[(458, 0.0),\n",
              " (323, 0.0),\n",
              " (516, 0.0),\n",
              " (517, 0.0),\n",
              " (64, 0.0),\n",
              " (313, 0.0),\n",
              " (61, 0.0),\n",
              " (524, 0.0),\n",
              " (476, 0.0),\n",
              " (530, 0.0)]"
            ]
          },
          "execution_count": 39,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "# most probable documents by topic (for first topic)\n",
        "topic_model.get_top_docs(topic=0, n_docs=10, sort_by='log_prob')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 40,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "reaHDD7gAfj2",
        "outputId": "a21623ed-24ad-4860-8f94-bc59d3e6744b"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "333.0312724936404"
            ]
          },
          "execution_count": 40,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "# total correlation\n",
        "topic_model.tc\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 41,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 339
        },
        "id": "pqGYoScQAxpO",
        "outputId": "89805755-fb9f-450b-df8d-044b19582c67"
      },
      "outputs": [
        {
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAmMAAAFCCAYAAAC90NpzAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAcz0lEQVR4nO3de7BlZX3m8e8jaADFQEOHq53GS2AoHNTqQYgjoGA0StI6sQxqTIuYdhw1GEgp6TGCGBGjRoljMfZEY0eJYIjaBG9RBBym1ARUJAIpkItyaWhQInjj9ps/1urkeDyXvbr32eucvb+fql17r/Wuvdbv1C7bh3e9631TVUiSJKkfD+u7AEmSpElmGJMkSeqRYUySJKlHhjFJkqQeGcYkSZJ6ZBiTJEnq0fZ9F7C1dt9991q5cmXfZUiSJM3r8ssvv7Oqls/UtmTD2MqVK7nsssv6LkOSJGleSW6arc3blJIkST0yjEmSJPXIMCZJktQjw5gkSVKPDGOSJEk9MoxJkiT1yDAmSZLUI8OYJElSj0YexpLskuS8JNckuTrJYUmWJflCkmvb911HXZckSVIf+ugZOxP4XFUdABwMXA2cDFxYVU8ALmy3JUmSxt5Iw1iSXwYOBz4IUFX3VdXdwGpgQ3vYBuD5o6xLkiSpL6Nem3I/YDPw10kOBi4HTgD2qKrb2mM2AXuMuK4ZrV53zsiutfH0Y0d2LUmStHiM+jbl9sBTgLOq6snAj5h2S7KqCqiZvpxkbZLLkly2efPmBS9WkiRpoY06jN0M3FxVX2u3z6MJZ7cn2Qugfb9jpi9X1fqqWlVVq5YvXz6SgiVJkhbSSMNYVW0Cvpdk/3bXUcBVwPnAmnbfGmDjKOuSJEnqy6jHjAG8Djg7ySOA64HjaELhx5McD9wEvKiHuiRJkkZu5GGsqr4JrJqh6ahR1yJJktQ3Z+CXJEnqkWFMkiSpR4YxSZKkHhnGJEmSemQYkyRJ6pFhTJIkqUeGMUmSpB4ZxiRJknpkGJMkSeqRYUySJKlHhjFJkqQeGcYkSZJ6ZBiTJEnq0fZ9F6DRW73unJFcZ+Ppx47kOpIkLWX2jEmSJPXIMCZJktQjw5gkSVKPDGOSJEk9MoxJkiT1yDAmSZLUI8OYJElSjwxjkiRJPTKMSZIk9cgwJkmS1CPDmCRJUo8MY5IkST0yjEmSJPVo+65fSLInsDewI3AncENV3TfswiRJkibBQGEsySrglcCzgRXTmu9L8s/Ax4CPVtU9wy1RkiRpfM0ZxtoQ9i7gcOBK4ALgG8Bm4CfAMmA/4KnAGcAZSf4ceHdV/XQB65YkSRoL8/WMXQL8H+DVVXX1XAcm2QFYDbyBZizaW4dSoSRJ0hibL4w9rqo2DXKitifsXODcJHtsc2WSJEkTYM6nKQcNYjN87/atK0eSJGmyDDy1RZLdk6yYtu9VSd6X5JjhlyZJkjT+uswz9iHg5C0bSf4UOAt4CbAxye8OcpIkNya5Msk3k1zW7luW5AtJrm3fd+1QlyRJ0pLVJYytAi6csv3fgdOrajfg/cCJHc71jKp6UlWtardPBi6sqie01zh59q9KkiSNjy5hbBlwO0CSg4A9gQ1t26eA/behjtVTzrUBeP42nEuSJGnJ6BLG7gL2bT8/E7i1qq5ttx/e4VwF/GOSy5OsbfftUVW3tZ83AT6NKUmSJkKX5ZC+CJyaZHfgJJresC0OAG4a8Dz/tapuSfIrwBeSXDO1saoqSc30xTa8rQVYsWL6QgCSJElLT5eesTcA3wPeDnwHeMuUtpcClw5ykqq6pX2/A/gkcAhwe5K9ANr3O2b57vqqWlVVq5YvX96hdEmSpMVp4J6xdu6wZ83SfDTN8khzSvJI4GFVdU/7+TeA04DzgTU0SyqtATYOWpckSdJS1mWesS8lOWCW5j2Bzw9wmj2AS5NcAfwT8Omq+hxNCHtWkmtpgt0Zg9YlSZK0lHUZM3Yk8OhZ2nYGjpjvBFV1PXDwDPvvAo7qUIskSdJY6DJmDJonIWfyOODebaxFkiRp4szZM5bkOOC4drOA9UnumXbYjsBB/PyEsJIkSRrAfD1jDwEPtq9M297yuotmWaTjF65MSZKk8TRnz1hVbaCdGT/JRcCrq+qaub4jSZKkwXWZ2uIZC1mIJEnSJOryNCUASQ6mWYdyh+ltVfU3wyhKkiRpUgwcxpLsAnwaOHTLrvZ96hOWhjFJkqQOukxtcTqwG3A4TRB7Ac2C4WcD19MsayRJkqQOuoSxZ9MEsq+22zdX1cVV9fs0i4ifMOziJEmSxl2XMLYXcH1VPQj8lGbW/S0+ATxvmIVJkiRNgi5hbBOwS/v5JuCwKW2PH1pFkiRJE6TL05SX0gzevwD4CHBKkpXAA8Aa4PxhFydJkjTuuoSxtwB7t5/fSTOY/3eBnWiC2OuGW5okSdL46zLp63eA77Sf7wdOal+SJEnaSl3GjEmSJGnIOs3An+SxwIuAFfziDPxVVS4WLkmS1EGXGfifD3ycpjftDuBn0w6pX/iSJEmS5tSlZ+ytwMXAS6tq88KUI0mSNFm6hLHHAicZxCRJkoanywD+a2ims5AkSdKQdAljbwDWtYP4JUmSNARdblOeStMzdnWSa4HvT2uvqjpiWIVJkiRNgi5h7EHgXxeqEEmSpEnUZQb+IxewDkmSpInkDPySJEk9mjOMJXlK1xMm2SHJAVtfkiRJ0uSYr2fsy0nOT/KcJPMFtxVJ1gE3AMcMrUJJkqQxNt+Ysf1pZt7fCPwwyVeAK4DNNMsh7UozGewhwEE0QeykqvrbBatYkiRpjMwZxqrqFuAVSU4GjgOeDZwI7DjlsBuALwMnA5+vKteolCRJGtBAT1NW1R3AO9oXSXYBdgDuqqr7F648SZKk8dZlnrF/V1V3D7sQSZKkSeTUFpIkST0yjEmSJPXIMCZJktSjXsJYku2SfCPJBe32fkm+luS6JOcmeUQfdUmSJI1aXz1jJwBXT9l+B/Ceqno88APg+F6qkiRJGrGteppyWyTZF3ge8DbgxCQBngm8pD1kA3AqcNaoa9PStXrdOSO71sbTjx3ZtSRJ42/gnrEkj0hySpJrkvw4yYPTXg8MeKr3Am8AHmq3dwPurqot378Z2Gfgv0CSJGkJ69Iz9k7gNcBngU/QLIfUSZJjgDuq6vIkR27F99cCawFWrFjR9euSJEmLTpcw9kLglKp62zZc72nAbyd5Ls0M/o8GzgR2SbJ92zu2L3DLTF+uqvXAeoBVq1a57JIkSVryugzgfxTwlW25WFX9SVXtW1UrgWOBL1XVS4GLaMIewBqahcklSZLGXpcw9g/A4QtUxxtpBvNfRzOG7IMLdB1JkqRFpcttyvcBf5PkIeAzwPenH1BV1w96sqq6GLh4yvcO6VCLJEnSWOgSxrbcojwVOGWWY7bbpmokSZImTJcw9grAQfOSJElDNHAYq6oPL2AdkiRJE6nzDPztjPkHAstoxo1dVVX2mEmSJG2FTmtTJnklcBvwLZrB998Cbk3iWpKSJElbYeCesSQvpZlw9ULgo8AmYE/gpcD6JD+uqo8tSJWSJEljqsttyjcAZ1fVy6bt35DkIzRzhRnGJEmSOuhym3J/mh6xmXy0bZckSVIHXcLYPTTrRs5k37ZdkiRJHXQJY58FTk/y9Kk7kxwG/FnbLkmSpA66jhk7FLg4yS00T1XuSdMrdl3bLkmSpA66TPq6KcmTaGbifzrNPGM3ApcAH66qHy9IhdKEWr3unJFda+Ppx47sWpKkn9dp0tc2cP2v9iVJkqRt1GnSV0mSJA3XnD1jSa4HXlBVVyS5gbkXCq+qetxQq5MkSRpz892mvAT44ZTPrkEpSZI0RHOGsao6bsrnly94NZIkSRNm4DFjSd6cZO9Z2vZK8ubhlSVJkjQZugzgP4XZZ+Dfu22XJElSB13CWOZo2xX42TbWIkmSNHHme5rySOCZU3a9Kskx0w7bEXge8O3hliZJkjT+5nua8gjgTe3nAo6b4Zj7gKuAPxxiXZIkSRNhztuUVfWWqnpYVT2M5jbloVu2p7x2qKqnVNVXRlOyJEnS+OiyNqWz9UuSJA1Zp7Upt0jyK8AO0/dX1Xe3uSJJkqQJMnAYS/Iw4M+AVwG7zHLYdsMoSpIkaVJ0ufX4euA1wLtpxo+dThPObgC+A/zB0KuTJEkac13C2HHAacA72u1PVtUpwH8CbgFWDLk2SZKksdcljD0WuKyqHgQeoJlfjKq6H3gv8IrhlydJkjTeuoSxf+M/Bu3fCuw/pW17YNmwipIkSZoUXZ6m/AZwIPD59vWWJD+h6SV7G/D14ZcnSZI03rqEsffS3KqEZlHwpwBnt9s3Aa8dYl2SJEkTocukr1+Y8nlTkkOAxwE7AVe3Y8ckSZLUwVZN+gpQVQVcN8RaJEmSJs6cYSzJ4V1OVlVfnud8OwBfBn6pvfZ5VXVKkv2Ac4DdgMuBl1XVfV2uLWlpWL3unJFcZ+Ppx47kOpK0rebrGbsYqAHOk/a4+Wbg/xnwzKq6N8nDgUuTfBY4EXhPVZ2T5H8DxwNnDXBdSerdqAImGDKlcTRfGHvGMC/W3tq8t918ePsq4JnAS9r9G4BTMYxJkqQJMGcYq6pLhn3BJNvR3Ip8PPB+mqWU7q6qB9pDbgb2GfZ1JUmSFqMuk74CkGT3JMckWZNkWbtvh3Yh8XlV1YNV9SRgX+AQ4IAO116b5LIkl23evLlr6ZIkSYvOwGEsjXfS9FydD3wIWNk2bwT+Z5cLV9XdwEXAYcAuSbb00u1Ls9blTN9ZX1WrqmrV8uXLu1xOkiRpUerSM/YnNBO7ngY8lWbQ/hb/ABwz3wmSLE+yS/t5R+BZwNU0oeyF7WFraMKdJEnS2Osyz9grgdOq6u3tuK+prqOZAHY+ewEb2u8/DPh4VV2Q5CrgnCR/RrPs0gc71CVJkrRkdQlj+wBfnaXtPuCR852gqr4FPHmG/dfTjB+TJEmaKF1uU94CHDRL28HADdtejiRJ0mTpEsb+DnhzkqdN2VdJfg04iWYGfUmSJHXQJYydClxDs5zRte2+vwOubLfPGGplkiRJE2DgMWNV9ZMkR9LMlP9smkH7dwFvBc6eMmmrJEmSBjRQGEvyCOBcmvUjPwJ8ZEGrkiRJmhADhbGqui/J0cCZC1yPJGmRGNUC6C5+rknXZczY/wMOXahCJEmSJlGXecZOAj6V5F7gU8BtQE09oKoeGmJtkiRJY69Lz9iVNLPsnwncRDPR6/1TXvcNvTpJkqQx16Vn7DSm9YRJkiRp2wz6NOV2wCeBW6tq88KWJEmSNDkGvU1ZwGXMsK6kJEmStt5AYawdmP89BlgMXJIkSYPrMoD/A8Dr2wlgJUmSNARdBvDvTPM05fVJPscvTm1RVXXKMIuTJEkad13C2Lopn18xQ3sBhjFJkqQOuiwU3uWWpiRJkgZgwJIkSepRl9uUACQ5BjgCWAZ8H7i4qj497MIkSZImwcBhLMnOwAXA04EHgLuA3YATk/xf4JiqundBqpQkSRpTXW5Tng48BXgZsGNV7QXsCPx+u//04ZcnSZI03rqEsd8B3lRVZ1fVgwBV9WBVnQ38adsuSZKkDrqEsd2Aq2Zpu6ptlyRJUgddwtgNwDGztD23bZckSVIHXZ6m/ADw7iSPAs6mmYF/T+BY4JXAicMvT5Ikabx1mfT1PUmW04Sul7e7A9wHnFFVZw6/PEmSpPHWaZ6xqlqX5J3AofzHPGNfraofLERxkiRJ467zpK9t8PrsAtQiSZI0ceYcwJ/kyUnuSrJ6jmNWt8ccNPzyJEmSxtt8T1O+FriiqjbOdkDbdjnwh8MsTJIkaRLMF8aeAXxkgPOcDRy17eVIkiRNlvnC2N7AdQOc53pgn20vR5IkabLMF8Z+AjxqgPM8CvjptpcjSZI0WeYLY99msNuPRwP/su3lSJIkTZb5wtjfAq9OcshsByQ5FHgVzbixOSV5TJKLklyV5NtJTmj3L0vyhSTXtu+7dvkjJEmSlqr5wth6miclL0lyZpKjkzyhfR2d5EzgIuCy9tj5PACcVFUH0kwc+5okBwInAxdW1ROAC9ttSZKksTfnpK9V9UCS5wB/CbyaZqqLqR4CPgy8vqoenO9iVXUbzZqWVNU9Sa6mGfi/GjiyPWwDcDHwxkH/CEmSpKVq3hn4q+rHwCuTvIlmqovHtE3fAy5uA1ZnSVYCTwa+Buwx5TybgD225pySJElLTZeFwjcBHxvGRZM8Cvh7mh61HyaZep1KUrN8by2wFmDFihXDKEWSJKlX840ZG7okD6cJYmdX1Sfa3bcn2att3wu4Y6bvVtX6qlpVVauWL18+moIlSZIW0EjDWJousA8CV1fVX0xpOh9Y035eA8y6/JIkSdI4Gfg25ZA8DXgZcGWSb7b71gFnAB9PcjxwE/CiEdclSZLUi5GGsaq6FMgsza5tKUmSJs6oe8YkSerN6nXnjOxaG08/dmTX0tI28gH8kiRJ+g9z9owleQiYcZqJGVRV2dMmSZLUwXzh6TQGD2OSJEnqaL7lkE4dUR2SJEkTyTFjkiRJPeo0xivJI4DfBPYHdpjWXFX11mEVJkmS5ucTokvfwGEsyd7ApcBKmnFkW+YLmzqmzDAmSZLUQZfblO8ENgMraILYU4HHAm8Drms/S5IkqYMutymfDvwxcGu7/VBV3Qi8Ocl2wF8Cq4dbniRJ0njr0jO2G3BrVT0E/AjYdUrbl4Ajh1iXJEnSROgSxm4Gdm8/fwf4jSlthwA/HVZRkiRJk6LLbcqLgCOATwEfAN6f5EnA/cCz232SJEnqoEsYexOwDKCqzkqyPfC7wE7An9PM1i9JkqQOBg5jVXUncOeU7fcB71uIoiRJkibFwGPGknwpyQGztP1aki8NryxJkqTJ0GUA/5HAo2dp25lmPJkkSZI66LQcEj8/2/5UjwPu3cZaJEmSgNEt87QYlniaM4wlOQ44rt0sYH2Se6YdtiNwEHDh8MuTJEkab/PdpnwIeLB9Zdr2ltddwFnA8QtXpiRJ0nias2esqjYAGwCSXAS8uqquGUVhkiRJk6DL1BbPWMhCJEmSJlGXpylJ8sQk5yXZnOSB9v3jSZ64UAVKkiSNs4F7xpL8F+AS4CfA+cAmYE/gt4DnJTm8qi5fkColSZLGVJepLd4O/AtwVFX9+xOVSXYGvti2/8Ys35UkSdIMutymPBR4+9QgBtBuvwM4bJiFSZIkTYIuYWy2CV8HbZckSdI0XcLY14B17W3Jf5fkkcAbga8OszBJkqRJMN8M/NcDL6iqK4B1wMXATUkuAG6jGcD/XGAnmrUrJUmS1MF8A/hXAr8EUFX/lORQ4M3As4FlwPeBi4C3VtWVC1inJEnSWOq0UHhVfQt44QLVIkmSNHEGGTPmwHxJkqQFMkjP2FuS3DnAcVVVa7a1IEmSpEkySBh7EvCzAY6zB02SJKmjQcLY86vqn4ZxsSQfAo4B7qiqg9p9y4BzaR4WuBF4UVX9YBjXkyRJWuw6LRQ+BB8GnjNt38nAhVX1BODCdluSJGkijDSMVdWXaabDmGo1sKH9vAF4/ihrkiRJ6tOoe8ZmskdV3dZ+3gTs0WcxkiRJozTnmLGqGnXPWSWZ9UGAJGuBtQArVqwYWV2SJEkLZTH0jN2eZC+A9v2O2Q6sqvVVtaqqVi1fvnxkBUqSJC2UxRDGzge2zE+2BtjYYy2SJEkjNdIwluRjwFeA/ZPcnOR44AzgWUmuBY5utyVJkiZCp7Upt1VVvXiWpqNGWYckSdJisRhuU0qSJE0sw5gkSVKPDGOSJEk9MoxJkiT1yDAmSZLUI8OYJElSjwxjkiRJPTKMSZIk9cgwJkmS1CPDmCRJUo8MY5IkST0yjEmSJPXIMCZJktQjw5gkSVKPDGOSJEk9MoxJkiT1yDAmSZLUI8OYJElSjwxjkiRJPTKMSZIk9cgwJkmS1CPDmCRJUo8MY5IkST0yjEmSJPXIMCZJktQjw5gkSVKPDGOSJEk9MoxJkiT1yDAmSZLUI8OYJElSjwxjkiRJPTKMSZIk9cgwJkmS1CPDmCRJUo8WTRhL8pwk/5rkuiQn912PJEnSKCyKMJZkO+D9wG8CBwIvTnJgv1VJkiQtvEURxoBDgOuq6vqqug84B1jdc02SJEkLbrGEsX2A703ZvrndJ0mSNNZSVX3XQJIXAs+pqle22y8DnlpVr5123Fpgbbu5P/CvIy10cLsDd/ZdhObl77R0+FstHf5WS4e/1Wj9alUtn6lh+1FXMotbgMdM2d633fdzqmo9sH5URW2tJJdV1aq+69Dc/J2WDn+rpcPfaunwt1o8Fsttyn8GnpBkvySPAI4Fzu+5JkmSpAW3KHrGquqBJK8FPg9sB3yoqr7dc1mSJEkLblGEMYCq+gzwmb7rGJJFfytVgL/TUuJvtXT4Wy0d/laLxKIYwC9JkjSpFsuYMUmSpIlkGBsil3RaGpI8JslFSa5K8u0kJ/Rdk2aXZLsk30hyQd+1aHZJdklyXpJrklyd5LC+a9LMkvxR+2/fvyT5WJId+q5p0hnGhsQlnZaUB4CTqupA4FDgNf5Wi9oJwNV9F6F5nQl8rqoOAA7G32xRSrIP8IfAqqo6iOahuWP7rUqGseFxSacloqpuq6qvt5/vofk/DVd8WISS7As8D/irvmvR7JL8MnA48EGAqrqvqu7utyrNYXtgxyTbAzsBt/Zcz8QzjA2PSzotQUlWAk8GvtZvJZrFe4E3AA/1XYjmtB+wGfjr9pbyXyV5ZN9F6RdV1S3Au4DvArcB/1ZV/9hvVTKMaWIleRTw98Drq+qHfdejn5fkGOCOqrq871o0r+2BpwBnVdWTgR8BjptdhJLsSnPXZj9gb+CRSX6v36pkGBuegZZ00uKQ5OE0QezsqvpE3/VoRk8DfjvJjTS3/Z+Z5KP9lqRZ3AzcXFVbepjPowlnWnyOBm6oqs1VdT/wCeDXe65p4hnGhsclnZaIJKEZ23J1Vf1F3/VoZlX1J1W1b1WtpPnf05eqyv+CX4SqahPwvST7t7uOAq7qsSTN7rvAoUl2av8tPAoftujdopmBf6lzSacl5WnAy4Ark3yz3beuXQVC0tZ5HXB2+x+j1wPH9VyPZlBVX0tyHvB1mifLv4Ez8ffOGfglSZJ65G1KSZKkHhnGJEmSemQYkyRJ6pFhTJIkqUeGMUmSpB4ZxiQtaUlqgNeNQ7zey9tzrhzWOSVNNucZk7TUHTZt+5PAFcCpU/b9bIjX+3R7zduGeE5JE8x5xiSNlbYX7FJn65e0VHibUtLYS3JIki8muTfJj5JcmOSQacd8OMnNSX49yT8n+WmSG5O8btpxM96mTPIHSb6e5CdJfpDkkiSu+SdpXoYxSWMtyX8GLgF2BV4O/D7waOCSJAdPO/zRwLnABuD5wMXAXyZ5+TzXeBfNkjJfB14E/B7wZWDFkP4MSWPMMWOSxt2bacaMHVVVdwMk+QJwI3AK8N+mHLszsLaqzmm3P5dkH+AtSTbUDOM6kjwe+CPgPVV14pSmTw/9L5E0luwZkzTuDgcu2BLEAKrqh8D5wBHTjn0Q+Ptp+86h6eHaZ5bzH03zb6mLLUvaKoYxSeNuGTM/+biJ5tblVD+oqvun7bu9fZ8tjO3Wvt+8deVJmnSGMUnj7vvAnjPs3xP4wbR9uyZ5+LR9e7Tvt8xy/jvb99nCmiTNyTAmadxdAjw3yc5bdrSff4tmgP5U2wG/M23fscB3mT2MfRF4CFg7jGIlTR4H8Esad28FjgEuTPIOoIA3AjsBp0079h7gz5PsDlwLvJhmTNjLZxq8D1BV30nyHuDENuSdTzP27BDgmqo6dwH+JkljxDAmaaxV1beSHAm8jWbKigBfBY6oqiumHf5Dmp6wM4En0owXO6GqNsxzjT9Och3wP4A1wI+AbwH/OMQ/RdKYcgZ+SaKZ9BU4uqr27bsWSZPFMWOSJEk9MoxJkiT1yNuUkiRJPbJnTJIkqUeGMUmSpB4ZxiRJknpkGJMkSeqRYUySJKlHhjFJkqQe/X+U5/10X9rHjgAAAABJRU5ErkJggg==",
            "text/plain": [
              "<Figure size 720x360 with 1 Axes>"
            ]
          },
          "metadata": {
            "needs_background": "light"
          },
          "output_type": "display_data"
        }
      ],
      "source": [
        "plt.figure(figsize=(10,5))\n",
        "plt.bar(range(topic_model.tcs.shape[0]), topic_model.tcs, color='#4e79a7', width=0.5)\n",
        "plt.xlabel('Topic', fontsize=16)\n",
        "plt.ylabel('Total Correlation (nats)', fontsize=16);"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Nv0Qd8N6Jqqh"
      },
      "source": [
        "## IV. Anchored topic models "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 45,
      "metadata": {
        "id": "WpNwPmY3A81s"
      },
      "outputs": [],
      "source": [
        "anchor_words = [['larry','fink','ceo','blackrock','letter'],['finance','investment','etf']]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 57,
      "metadata": {
        "id": "ymek6WXMBYly"
      },
      "outputs": [],
      "source": [
        "anchored_topic_model = ct.Corex(n_hidden=10, seed=2)\n",
        "anchored_topic_model.fit(doc_word, words=words, anchors=anchor_words, anchor_strength=6);"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 47,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "rYmqpQFyBd1Z",
        "outputId": "4aaf8929-c6ed-47ef-f256-929a80d11e8d"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "0: blackrock, fink, larry, ceo, money, company, assets, loss, capital, research\n",
            "1: investment, etf, finance, italy, states, france, markets, portugal, brazil, ishares\n"
          ]
        }
      ],
      "source": [
        "for n in range(len(anchor_words)):\n",
        "    topic_words,_,_ = zip(*anchored_topic_model.get_topics(topic=n))\n",
        "    print('{}: '.format(n) + ', '.join(topic_words))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0__h3CyPJ-Wz"
      },
      "source": [
        "#### Per topic correlation score"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 52,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "O724azgYBqCV",
        "outputId": "51c4a06c-5968-4f5d-f059-ad3b50600b27"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "[72.79792572 39.84738616 48.95158613 15.85926739 13.42708678  7.96979507\n",
            " 28.51900498 25.67557543 45.9428479  23.54609754]\n",
            "[72.79792572 39.84738616 48.95158613 15.85926739 13.42708678  7.96979507\n",
            " 28.51900498 25.67557543 45.9428479  23.54609754]\n"
          ]
        }
      ],
      "source": [
        "print(np.mean(anchored_topic_model.log_z, axis=0))\n",
        "print(anchored_topic_model.tcs)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "bIYEeCCOJ26k"
      },
      "source": [
        "## VI. Documents display"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 55,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Ta-6OIdKCTtn",
        "outputId": "fbd000a8-a2c2-4855-aaf9-5c292c378753"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "NOTE: 'docs' not provided to CorEx. Returning top docs as lists of row indices\n"
          ]
        },
        {
          "data": {
            "text/plain": [
              "[(0, 0.0),\n",
              " (264, 0.0),\n",
              " (268, 0.0),\n",
              " (269, 0.0),\n",
              " (279, 0.0),\n",
              " (289, 0.0),\n",
              " (290, 0.0),\n",
              " (297, 0.0),\n",
              " (300, 0.0),\n",
              " (304, 0.0)]"
            ]
          },
          "execution_count": 55,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "anchored_topic_model.get_top_docs(topic=0, n_docs=10, sort_by='log_prob')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 56,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Hz5wGIW-FjRz",
        "outputId": "65f69d86-e24e-4980-b365-6298dc29ae0e"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "NOTE: 'docs' not provided to CorEx. Returning top docs as lists of row indices\n"
          ]
        },
        {
          "data": {
            "text/plain": [
              "[(0, 0.0),\n",
              " (251, 0.0),\n",
              " (256, 0.0),\n",
              " (257, 0.0),\n",
              " (260, 0.0),\n",
              " (268, 0.0),\n",
              " (269, 0.0),\n",
              " (271, 0.0),\n",
              " (274, 0.0),\n",
              " (279, 0.0)]"
            ]
          },
          "execution_count": 56,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "anchored_topic_model.get_top_docs(topic=1, n_docs=10, sort_by='log_prob')"
      ]
    }
  ],
  "metadata": {
    "colab": {
      "name": "archives_selection_blackrock.ipynb",
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}