spike_analysis.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import datetime\n",
    "from datetime import timedelta"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "DATASET_PATH = 'data/'\n",
    "AUTH_FILTERED_PATH = 'data/authority_filtered/'\n",
    "SPIKES_PATH = 'data/spikes/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "AUTH_FILTERED_DATASET = pd.read_csv(AUTH_FILTERED_PATH + 'ferrero_corp_auth.csv')\n",
    "SPIKES_DATASET = pd.read_csv(SPIKES_PATH + 'ferrero_corp_spikes.csv')\n",
    "AUTH_SPIKES_DATASET = pd.read_csv(SPIKES_PATH + 'ferrero_corp_auth_spikes.csv')\n",
    "\n",
    "START_DATE = '2019-12-1'\n",
    "END_DATE = '2019-12-7'\n",
    "DATES_DELTA = 7"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>social</th>\n",
       "      <th>id</th>\n",
       "      <th>topic_id</th>\n",
       "      <th>rule_id</th>\n",
       "      <th>is_comment</th>\n",
       "      <th>text</th>\n",
       "      <th>links</th>\n",
       "      <th>mentions</th>\n",
       "      <th>hashtags</th>\n",
       "      <th>...</th>\n",
       "      <th>place_location_lat</th>\n",
       "      <th>place_location_lon</th>\n",
       "      <th>place_lat</th>\n",
       "      <th>place_lon</th>\n",
       "      <th>user_place_country_code</th>\n",
       "      <th>score_spam</th>\n",
       "      <th>score_performance</th>\n",
       "      <th>user_score_rank</th>\n",
       "      <th>score_rating</th>\n",
       "      <th>date_index</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>34648</td>\n",
       "      <td>twitter</td>\n",
       "      <td>1174618145786650624</td>\n",
       "      <td>7ee62d5c-5811-45ce-9cef-c09d143f041a</td>\n",
       "      <td>12559.0</td>\n",
       "      <td>False</td>\n",
       "      <td>La Ferrero riconosce a 6000 dipendenti un prem...</td>\n",
       "      <td>https://t.co/T3wBQLWAeo</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2019-09-19 09:36:03+00:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>34065</td>\n",
       "      <td>twitter</td>\n",
       "      <td>1176441842096910336</td>\n",
       "      <td>7ee62d5c-5811-45ce-9cef-c09d143f041a</td>\n",
       "      <td>12559.0</td>\n",
       "      <td>False</td>\n",
       "      <td>Il Ministro Fioramonti si permette di giustifi...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>@lofioramonti</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>41.898417</td>\n",
       "      <td>12.545146</td>\n",
       "      <td>41.898417</td>\n",
       "      <td>12.545146</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2019-09-24 10:22:47+00:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>33847</td>\n",
       "      <td>twitter</td>\n",
       "      <td>1176748033066844160</td>\n",
       "      <td>7ee62d5c-5811-45ce-9cef-c09d143f041a</td>\n",
       "      <td>12559.0</td>\n",
       "      <td>False</td>\n",
       "      <td>@theoddmother @lofioramonti In Italia la Ferre...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>@theoddmother, @lofioramonti</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>41.898417</td>\n",
       "      <td>12.545146</td>\n",
       "      <td>41.898417</td>\n",
       "      <td>12.545146</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2019-09-25 06:39:28+00:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>32534</td>\n",
       "      <td>twitter</td>\n",
       "      <td>1183821988437135362</td>\n",
       "      <td>7ee62d5c-5811-45ce-9cef-c09d143f041a</td>\n",
       "      <td>12559.0</td>\n",
       "      <td>False</td>\n",
       "      <td>@gianni_la_marca @angelofilippi Per il mercato...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>@gianni_la_marca, @angelofilippi</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>41.898417</td>\n",
       "      <td>12.545146</td>\n",
       "      <td>41.898417</td>\n",
       "      <td>12.545146</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2019-10-14 19:08:51+00:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>32064</td>\n",
       "      <td>twitter</td>\n",
       "      <td>1186949229438263297</td>\n",
       "      <td>7ee62d5c-5811-45ce-9cef-c09d143f041a</td>\n",
       "      <td>12559.0</td>\n",
       "      <td>False</td>\n",
       "      <td>Siamo in un pericoloso paradosso con il Paese ...</td>\n",
       "      <td>https://t.co/lkkYO9w8VC</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2019-10-23 10:15:23+00:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>278</th>\n",
       "      <td>1013</td>\n",
       "      <td>twitter</td>\n",
       "      <td>1418810115445166080</td>\n",
       "      <td>7ee62d5c-5811-45ce-9cef-c09d143f041a</td>\n",
       "      <td>12559.0</td>\n",
       "      <td>False</td>\n",
       "      <td>Ferrero, la strategia green del big di Alba (è...</td>\n",
       "      <td>https://t.co/r2XvO05KdN, https://t.co/t8k8vmUlQk</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2021-07-24 05:47:46+00:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>279</th>\n",
       "      <td>841</td>\n",
       "      <td>twitter</td>\n",
       "      <td>1420717382830415880</td>\n",
       "      <td>7ee62d5c-5811-45ce-9cef-c09d143f041a</td>\n",
       "      <td>12559.0</td>\n",
       "      <td>False</td>\n",
       "      <td>Ferrero l'azienda meglio percepita sul territo...</td>\n",
       "      <td>https://t.co/4O4c31bPiq, https://t.co/n8RpxZP5Vu</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2021-07-29 12:06:34+00:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>280</th>\n",
       "      <td>822</td>\n",
       "      <td>twitter</td>\n",
       "      <td>1421105542945202182</td>\n",
       "      <td>7ee62d5c-5811-45ce-9cef-c09d143f041a</td>\n",
       "      <td>12559.0</td>\n",
       "      <td>False</td>\n",
       "      <td>L’attimo fuggente, online il numero di agosto ...</td>\n",
       "      <td>https://t.co/Q9FPElZrOU</td>\n",
       "      <td>@ferrero_eu, @generali, @prada, @armani, @ciro...</td>\n",
       "      <td>#luigicremonini, #vincenzocremonini, #alberton...</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2021-07-30 13:48:58+00:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>281</th>\n",
       "      <td>467</td>\n",
       "      <td>twitter</td>\n",
       "      <td>1426075939201421313</td>\n",
       "      <td>7ee62d5c-5811-45ce-9cef-c09d143f041a</td>\n",
       "      <td>12559.0</td>\n",
       "      <td>False</td>\n",
       "      <td>Mediobanca, Gavio arrotonda la quota. I Ferrer...</td>\n",
       "      <td>https://t.co/QPX7deAYBt, https://t.co/XUfJMFKsFE</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2021-08-13 06:59:33+00:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>282</th>\n",
       "      <td>61</td>\n",
       "      <td>twitter</td>\n",
       "      <td>1430765728890859524</td>\n",
       "      <td>7ee62d5c-5811-45ce-9cef-c09d143f041a</td>\n",
       "      <td>12559.0</td>\n",
       "      <td>False</td>\n",
       "      <td>Da anni le aziende alimentari prendono impegni...</td>\n",
       "      <td>https://t.co/79pL8eW8Dm, https://t.co/3MC4xgol6A</td>\n",
       "      <td>@foodwatch_de</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2021-08-26 05:35:06+00:00</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>283 rows × 64 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     Unnamed: 0   social                   id  \\\n",
       "0         34648  twitter  1174618145786650624   \n",
       "1         34065  twitter  1176441842096910336   \n",
       "2         33847  twitter  1176748033066844160   \n",
       "3         32534  twitter  1183821988437135362   \n",
       "4         32064  twitter  1186949229438263297   \n",
       "..          ...      ...                  ...   \n",
       "278        1013  twitter  1418810115445166080   \n",
       "279         841  twitter  1420717382830415880   \n",
       "280         822  twitter  1421105542945202182   \n",
       "281         467  twitter  1426075939201421313   \n",
       "282          61  twitter  1430765728890859524   \n",
       "\n",
       "                                 topic_id  rule_id  is_comment  \\\n",
       "0    7ee62d5c-5811-45ce-9cef-c09d143f041a  12559.0       False   \n",
       "1    7ee62d5c-5811-45ce-9cef-c09d143f041a  12559.0       False   \n",
       "2    7ee62d5c-5811-45ce-9cef-c09d143f041a  12559.0       False   \n",
       "3    7ee62d5c-5811-45ce-9cef-c09d143f041a  12559.0       False   \n",
       "4    7ee62d5c-5811-45ce-9cef-c09d143f041a  12559.0       False   \n",
       "..                                    ...      ...         ...   \n",
       "278  7ee62d5c-5811-45ce-9cef-c09d143f041a  12559.0       False   \n",
       "279  7ee62d5c-5811-45ce-9cef-c09d143f041a  12559.0       False   \n",
       "280  7ee62d5c-5811-45ce-9cef-c09d143f041a  12559.0       False   \n",
       "281  7ee62d5c-5811-45ce-9cef-c09d143f041a  12559.0       False   \n",
       "282  7ee62d5c-5811-45ce-9cef-c09d143f041a  12559.0       False   \n",
       "\n",
       "                                                  text  \\\n",
       "0    La Ferrero riconosce a 6000 dipendenti un prem...   \n",
       "1    Il Ministro Fioramonti si permette di giustifi...   \n",
       "2    @theoddmother @lofioramonti In Italia la Ferre...   \n",
       "3    @gianni_la_marca @angelofilippi Per il mercato...   \n",
       "4    Siamo in un pericoloso paradosso con il Paese ...   \n",
       "..                                                 ...   \n",
       "278  Ferrero, la strategia green del big di Alba (è...   \n",
       "279  Ferrero l'azienda meglio percepita sul territo...   \n",
       "280  L’attimo fuggente, online il numero di agosto ...   \n",
       "281  Mediobanca, Gavio arrotonda la quota. I Ferrer...   \n",
       "282  Da anni le aziende alimentari prendono impegni...   \n",
       "\n",
       "                                                links  \\\n",
       "0                             https://t.co/T3wBQLWAeo   \n",
       "1                                                 NaN   \n",
       "2                                                 NaN   \n",
       "3                                                 NaN   \n",
       "4                             https://t.co/lkkYO9w8VC   \n",
       "..                                                ...   \n",
       "278  https://t.co/r2XvO05KdN, https://t.co/t8k8vmUlQk   \n",
       "279  https://t.co/4O4c31bPiq, https://t.co/n8RpxZP5Vu   \n",
       "280                           https://t.co/Q9FPElZrOU   \n",
       "281  https://t.co/QPX7deAYBt, https://t.co/XUfJMFKsFE   \n",
       "282  https://t.co/79pL8eW8Dm, https://t.co/3MC4xgol6A   \n",
       "\n",
       "                                              mentions  \\\n",
       "0                                                  NaN   \n",
       "1                                        @lofioramonti   \n",
       "2                         @theoddmother, @lofioramonti   \n",
       "3                     @gianni_la_marca, @angelofilippi   \n",
       "4                                                  NaN   \n",
       "..                                                 ...   \n",
       "278                                                NaN   \n",
       "279                                                NaN   \n",
       "280  @ferrero_eu, @generali, @prada, @armani, @ciro...   \n",
       "281                                                NaN   \n",
       "282                                      @foodwatch_de   \n",
       "\n",
       "                                              hashtags  ...  \\\n",
       "0                                                  NaN  ...   \n",
       "1                                                  NaN  ...   \n",
       "2                                                  NaN  ...   \n",
       "3                                                  NaN  ...   \n",
       "4                                                  NaN  ...   \n",
       "..                                                 ...  ...   \n",
       "278                                                NaN  ...   \n",
       "279                                                NaN  ...   \n",
       "280  #luigicremonini, #vincenzocremonini, #alberton...  ...   \n",
       "281                                                NaN  ...   \n",
       "282                                                NaN  ...   \n",
       "\n",
       "    place_location_lat place_location_lon  place_lat  place_lon  \\\n",
       "0                  NaN                NaN        NaN        NaN   \n",
       "1            41.898417          12.545146  41.898417  12.545146   \n",
       "2            41.898417          12.545146  41.898417  12.545146   \n",
       "3            41.898417          12.545146  41.898417  12.545146   \n",
       "4                  NaN                NaN        NaN        NaN   \n",
       "..                 ...                ...        ...        ...   \n",
       "278                NaN                NaN        NaN        NaN   \n",
       "279                NaN                NaN        NaN        NaN   \n",
       "280                NaN                NaN        NaN        NaN   \n",
       "281                NaN                NaN        NaN        NaN   \n",
       "282                NaN                NaN        NaN        NaN   \n",
       "\n",
       "    user_place_country_code  score_spam  score_performance  user_score_rank  \\\n",
       "0                       NaN         NaN                NaN              NaN   \n",
       "1                       NaN         NaN                NaN              NaN   \n",
       "2                       NaN         NaN                NaN              NaN   \n",
       "3                       NaN         NaN                NaN              NaN   \n",
       "4                       NaN         NaN                NaN              NaN   \n",
       "..                      ...         ...                ...              ...   \n",
       "278                     NaN         NaN                NaN              NaN   \n",
       "279                     NaN         NaN                NaN              NaN   \n",
       "280                     NaN         NaN                NaN              NaN   \n",
       "281                     NaN         NaN                NaN              NaN   \n",
       "282                     NaN         NaN                NaN              NaN   \n",
       "\n",
       "     score_rating                date_index  \n",
       "0             NaN 2019-09-19 09:36:03+00:00  \n",
       "1             NaN 2019-09-24 10:22:47+00:00  \n",
       "2             NaN 2019-09-25 06:39:28+00:00  \n",
       "3             NaN 2019-10-14 19:08:51+00:00  \n",
       "4             NaN 2019-10-23 10:15:23+00:00  \n",
       "..            ...                       ...  \n",
       "278           NaN 2021-07-24 05:47:46+00:00  \n",
       "279           NaN 2021-07-29 12:06:34+00:00  \n",
       "280           NaN 2021-07-30 13:48:58+00:00  \n",
       "281           NaN 2021-08-13 06:59:33+00:00  \n",
       "282           NaN 2021-08-26 05:35:06+00:00  \n",
       "\n",
       "[283 rows x 64 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "auth_filtered_df = AUTH_FILTERED_DATASET.copy()\n",
    "auth_filtered_df['date_index'] = pd.to_datetime(auth_filtered_df['date_index'])\n",
    "auth_filtered_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>mIndex</th>\n",
       "      <th>volume</th>\n",
       "      <th>sentiment</th>\n",
       "      <th>sign</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>date_index</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2019-08-20</th>\n",
       "      <td>2.602240</td>\n",
       "      <td>96</td>\n",
       "      <td>0.373657</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-08-21</th>\n",
       "      <td>13.554951</td>\n",
       "      <td>378</td>\n",
       "      <td>0.373657</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-08-22</th>\n",
       "      <td>5.825910</td>\n",
       "      <td>179</td>\n",
       "      <td>0.373657</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-08-23</th>\n",
       "      <td>2.485722</td>\n",
       "      <td>93</td>\n",
       "      <td>0.373657</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-08-24</th>\n",
       "      <td>1.006199</td>\n",
       "      <td>71</td>\n",
       "      <td>0.373657</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2021-08-19</th>\n",
       "      <td>0.774941</td>\n",
       "      <td>36</td>\n",
       "      <td>0.524922</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2021-08-20</th>\n",
       "      <td>0.858849</td>\n",
       "      <td>38</td>\n",
       "      <td>0.493919</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2021-08-23</th>\n",
       "      <td>2.040529</td>\n",
       "      <td>67</td>\n",
       "      <td>0.468272</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2021-08-24</th>\n",
       "      <td>1.287879</td>\n",
       "      <td>49</td>\n",
       "      <td>0.360722</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2021-08-25</th>\n",
       "      <td>0.679956</td>\n",
       "      <td>41</td>\n",
       "      <td>0.437569</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>165 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "               mIndex  volume  sentiment   sign\n",
       "date_index                                     \n",
       "2019-08-20   2.602240      96   0.373657  False\n",
       "2019-08-21  13.554951     378   0.373657  False\n",
       "2019-08-22   5.825910     179   0.373657  False\n",
       "2019-08-23   2.485722      93   0.373657  False\n",
       "2019-08-24   1.006199      71   0.373657  False\n",
       "...               ...     ...        ...    ...\n",
       "2021-08-19   0.774941      36   0.524922   True\n",
       "2021-08-20   0.858849      38   0.493919  False\n",
       "2021-08-23   2.040529      67   0.468272  False\n",
       "2021-08-24   1.287879      49   0.360722  False\n",
       "2021-08-25   0.679956      41   0.437569  False\n",
       "\n",
       "[165 rows x 4 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "spikes_df = SPIKES_DATASET.copy()\n",
    "spikes_df.set_index('date_index', inplace=True)\n",
    "spikes_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>mIndex</th>\n",
       "      <th>volume</th>\n",
       "      <th>sentiment</th>\n",
       "      <th>sign</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>date_index</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2019-10-17</th>\n",
       "      <td>1.100317</td>\n",
       "      <td>1</td>\n",
       "      <td>0.600317</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-10-23</th>\n",
       "      <td>1.411402</td>\n",
       "      <td>1</td>\n",
       "      <td>0.088598</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-10-24</th>\n",
       "      <td>5.524376</td>\n",
       "      <td>3</td>\n",
       "      <td>0.395125</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-10-25</th>\n",
       "      <td>1.182024</td>\n",
       "      <td>2</td>\n",
       "      <td>0.317976</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-10-30</th>\n",
       "      <td>1.289454</td>\n",
       "      <td>1</td>\n",
       "      <td>0.789454</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-10-31</th>\n",
       "      <td>3.668820</td>\n",
       "      <td>2</td>\n",
       "      <td>0.277060</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-11-08</th>\n",
       "      <td>2.393225</td>\n",
       "      <td>3</td>\n",
       "      <td>0.696613</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-12-01</th>\n",
       "      <td>3.318031</td>\n",
       "      <td>2</td>\n",
       "      <td>0.393990</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-12-06</th>\n",
       "      <td>5.530052</td>\n",
       "      <td>6</td>\n",
       "      <td>0.393990</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-12-10</th>\n",
       "      <td>2.212021</td>\n",
       "      <td>3</td>\n",
       "      <td>0.393990</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-12-13</th>\n",
       "      <td>1.106010</td>\n",
       "      <td>2</td>\n",
       "      <td>0.393990</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-12-24</th>\n",
       "      <td>1.036259</td>\n",
       "      <td>2</td>\n",
       "      <td>0.463741</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2020-02-08</th>\n",
       "      <td>1.496028</td>\n",
       "      <td>1</td>\n",
       "      <td>0.003972</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2020-02-10</th>\n",
       "      <td>1.017680</td>\n",
       "      <td>1</td>\n",
       "      <td>0.482320</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2020-02-13</th>\n",
       "      <td>1.463476</td>\n",
       "      <td>1</td>\n",
       "      <td>0.963476</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2020-02-18</th>\n",
       "      <td>1.106010</td>\n",
       "      <td>2</td>\n",
       "      <td>0.393990</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2020-02-19</th>\n",
       "      <td>3.318031</td>\n",
       "      <td>4</td>\n",
       "      <td>0.393990</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2020-02-21</th>\n",
       "      <td>1.106010</td>\n",
       "      <td>1</td>\n",
       "      <td>0.393990</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2020-05-09</th>\n",
       "      <td>1.380941</td>\n",
       "      <td>1</td>\n",
       "      <td>0.880941</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2020-05-14</th>\n",
       "      <td>1.447140</td>\n",
       "      <td>1</td>\n",
       "      <td>0.052860</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2020-10-08</th>\n",
       "      <td>3.318031</td>\n",
       "      <td>2</td>\n",
       "      <td>0.393990</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2021-03-27</th>\n",
       "      <td>1.106010</td>\n",
       "      <td>1</td>\n",
       "      <td>0.393990</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2021-04-16</th>\n",
       "      <td>1.106010</td>\n",
       "      <td>1</td>\n",
       "      <td>0.393990</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2021-04-19</th>\n",
       "      <td>1.106010</td>\n",
       "      <td>2</td>\n",
       "      <td>0.393990</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2021-06-04</th>\n",
       "      <td>1.475816</td>\n",
       "      <td>1</td>\n",
       "      <td>0.024184</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              mIndex  volume  sentiment   sign\n",
       "date_index                                    \n",
       "2019-10-17  1.100317       1   0.600317   True\n",
       "2019-10-23  1.411402       1   0.088598  False\n",
       "2019-10-24  5.524376       3   0.395125  False\n",
       "2019-10-25  1.182024       2   0.317976  False\n",
       "2019-10-30  1.289454       1   0.789454   True\n",
       "2019-10-31  3.668820       2   0.277060  False\n",
       "2019-11-08  2.393225       3   0.696613   True\n",
       "2019-12-01  3.318031       2   0.393990  False\n",
       "2019-12-06  5.530052       6   0.393990  False\n",
       "2019-12-10  2.212021       3   0.393990  False\n",
       "2019-12-13  1.106010       2   0.393990  False\n",
       "2019-12-24  1.036259       2   0.463741  False\n",
       "2020-02-08  1.496028       1   0.003972  False\n",
       "2020-02-10  1.017680       1   0.482320  False\n",
       "2020-02-13  1.463476       1   0.963476   True\n",
       "2020-02-18  1.106010       2   0.393990  False\n",
       "2020-02-19  3.318031       4   0.393990  False\n",
       "2020-02-21  1.106010       1   0.393990  False\n",
       "2020-05-09  1.380941       1   0.880941   True\n",
       "2020-05-14  1.447140       1   0.052860  False\n",
       "2020-10-08  3.318031       2   0.393990  False\n",
       "2021-03-27  1.106010       1   0.393990  False\n",
       "2021-04-16  1.106010       1   0.393990  False\n",
       "2021-04-19  1.106010       2   0.393990  False\n",
       "2021-06-04  1.475816       1   0.024184  False"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "auth_spikes_df = AUTH_SPIKES_DATASET.copy()\n",
    "auth_spikes_df.set_index('date_index', inplace=True)\n",
    "auth_spikes_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_relevance_idxs(df, thresh):\n",
    "    return df[df['mIndex'] > thresh][['mIndex', 'sign']]\n",
    "\n",
    "\n",
    "def filter_by_relevance_idxs(df, col='mIndex', thresh=0.5):\n",
    "    return df[df[col] > thresh]\n",
    "\n",
    "\n",
    "def get_date_range(date, d_range):\n",
    "    date_list = []\n",
    "    date_list.append(date)\n",
    "    for day in range(1, d_range+1):\n",
    "        b_date = (date - datetime.timedelta(days = day))\n",
    "        a_date = (date + datetime.timedelta(days = day))\n",
    "        date_list.append(b_date)\n",
    "        date_list.append(a_date)\n",
    "        date_list.sort()\n",
    "    return date_list\n",
    "\n",
    "\n",
    "def filter_by_previous_dates(df, date, date_range, col=None, include_date=False):\n",
    "    df = df.copy()\n",
    "    formatted_date = pd.to_datetime(date, format='%Y/%m/%d', errors='coerce')\n",
    "    # formatted_date = (formatted_date - pd.to_timedelta(1, unit='days'))            # Needed to compare days by String\n",
    "    mindate = (formatted_date - pd.to_timedelta(date_range, unit='days')).date()    \n",
    "    if include_date is True:                                                         # Includes the signature date into the range\n",
    "        # mindate += timedelta(days=1)\n",
    "        formatted_date += timedelta(days=1)\n",
    "    if (col is None):\n",
    "        df = df.reset_index()\n",
    "        lowerbound = df[df.iloc[:, -1] >= mindate]\n",
    "        lowerbound = lowerbound[lowerbound.iloc[:, -1] < formatted_date]\n",
    "        lowerbound = lowerbound.set_index(lowerbound.iloc[:, -1]).drop(lowerbound.columns[-1], axis=1)\n",
    "        return lowerbound\n",
    "    else:\n",
    "        lowerbound = df[df[col] >= mindate]  \n",
    "        return lowerbound[lowerbound[col] < formatted_date]\n",
    "\n",
    "\n",
    "def filter_by_following_dates(df, date, date_range, col=None, include_date=False):\n",
    "    df = df.copy()\n",
    "    formatted_date = pd.to_datetime(date, format='%Y/%m/%d', errors='coerce')\n",
    "    maxdate = (formatted_date + pd.to_timedelta(date_range, unit='days')).date()\n",
    "    if include_date is True:                                                         # Includes the signature date into the range\n",
    "        # maxdate -= timedelta(days=1)  \n",
    "        formatted_date -= timedelta(days=1)  \n",
    "    if (col is None):\n",
    "        df = df.reset_index()\n",
    "        lowerbound = df[df.iloc[:, -1] > formatted_date]\n",
    "        lowerbound = lowerbound[lowerbound.iloc[:, -1] <= maxdate]\n",
    "        lowerbound = lowerbound.set_index(lowerbound.iloc[:, -1]).drop(lowerbound.columns[-1], axis=1)\n",
    "        return lowerbound\n",
    "    else:\n",
    "        lowerbound = df[df[col] > formatted_date]  \n",
    "        return lowerbound[lowerbound[col] <= maxdate]\n",
    "\n",
    "\n",
    "def get_previous_dates(date, date_range):\n",
    "    date_list = []\n",
    "    date_list.append(date)\n",
    "    for day in range(1, date_range+1):\n",
    "        b_date = (date - datetime.timedelta(days = day))\n",
    "        date_list.append(b_date)\n",
    "        date_list.sort()\n",
    "    return date_list\n",
    "\n",
    "\n",
    "def filter_by_date_range(dataset, mindate, maxdate, col=None):\n",
    "    formatted_mindate = pd.to_datetime(mindate, format='%Y/%m/%d', errors='coerce').date()\n",
    "    formatted_maxdate = pd.to_datetime(maxdate, format='%Y/%m/%d', errors='coerce').date()\n",
    "    if (col is None):\n",
    "        lowerbound = dataset[dataset.index > formatted_mindate]    \n",
    "        return lowerbound[lowerbound.index <= formatted_maxdate]\n",
    "    else:\n",
    "        lowerbound = dataset[dataset[col] > formatted_mindate]    \n",
    "        return lowerbound[lowerbound[col] <= formatted_maxdate]\n",
    "\n",
    "\n",
    "def convert_datetime_col_to_iso_format(df, col=None):\n",
    "    df = df.copy()\n",
    "    if (not col):\n",
    "        df = df.reset_index()\n",
    "        df['date_index'] = df['date_index'].apply(lambda x: pd.to_datetime(x, format='%Y/%m/%d', errors='coerce').date())\n",
    "        df = df.set_index(keys=['date_index'])\n",
    "        return df\n",
    "    else:\n",
    "        df['date_index'] = df['date_index'].apply(lambda x: pd.to_datetime(x, format='%Y/%m/%d', errors='coerce').date())\n",
    "        return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0    <class 'datetime.date'>\n",
      "1    <class 'datetime.date'>\n",
      "2    <class 'datetime.date'>\n",
      "3    <class 'datetime.date'>\n",
      "4    <class 'datetime.date'>\n",
      "5    <class 'datetime.date'>\n",
      "6    <class 'datetime.date'>\n",
      "7    <class 'datetime.date'>\n",
      "8    <class 'datetime.date'>\n",
      "9    <class 'datetime.date'>\n",
      "Name: date_index, dtype: object\n"
     ]
    }
   ],
   "source": [
    "lista = {'date_index': ['2020-10-09','2020-10-10', '2020-10-11', '2020-10-12', '2020-10-13', '2020-10-14', '2020-10-15', '2020-10-16', '2020-10-17' ,'2020-10-18']}\n",
    "lista = pd.DataFrame(lista)\n",
    "lista['date_index'] = lista['date_index'].apply(lambda x: pd.to_datetime(x, format='%Y/%m/%d', errors='coerce').date())\n",
    "print(lista['date_index'].apply(lambda x: type(x)))\n",
    "lista = lista.set_index('date_index')\n",
    "# lista"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>date_index</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2020-10-10</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2020-10-11</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2020-10-12</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2020-10-13</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2020-10-14</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2020-10-15</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2020-10-16</th>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: [2020-10-10, 2020-10-11, 2020-10-12, 2020-10-13, 2020-10-14, 2020-10-15, 2020-10-16]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "filter_by_previous_dates(lista, '2020-10-17', 7)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "auth_filtered_df = convert_datetime_col_to_iso_format(auth_filtered_df, 'date_index')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>social</th>\n",
       "      <th>id</th>\n",
       "      <th>topic_id</th>\n",
       "      <th>rule_id</th>\n",
       "      <th>is_comment</th>\n",
       "      <th>text</th>\n",
       "      <th>links</th>\n",
       "      <th>mentions</th>\n",
       "      <th>hashtags</th>\n",
       "      <th>...</th>\n",
       "      <th>place_location_lat</th>\n",
       "      <th>place_location_lon</th>\n",
       "      <th>place_lat</th>\n",
       "      <th>place_lon</th>\n",
       "      <th>user_place_country_code</th>\n",
       "      <th>score_spam</th>\n",
       "      <th>score_performance</th>\n",
       "      <th>user_score_rank</th>\n",
       "      <th>score_rating</th>\n",
       "      <th>date_index</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>34648</td>\n",
       "      <td>twitter</td>\n",
       "      <td>1174618145786650624</td>\n",
       "      <td>7ee62d5c-5811-45ce-9cef-c09d143f041a</td>\n",
       "      <td>12559.0</td>\n",
       "      <td>False</td>\n",
       "      <td>La Ferrero riconosce a 6000 dipendenti un prem...</td>\n",
       "      <td>https://t.co/T3wBQLWAeo</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2019-09-19</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>34065</td>\n",
       "      <td>twitter</td>\n",
       "      <td>1176441842096910336</td>\n",
       "      <td>7ee62d5c-5811-45ce-9cef-c09d143f041a</td>\n",
       "      <td>12559.0</td>\n",
       "      <td>False</td>\n",
       "      <td>Il Ministro Fioramonti si permette di giustifi...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>@lofioramonti</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>41.898417</td>\n",
       "      <td>12.545146</td>\n",
       "      <td>41.898417</td>\n",
       "      <td>12.545146</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2019-09-24</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>33847</td>\n",
       "      <td>twitter</td>\n",
       "      <td>1176748033066844160</td>\n",
       "      <td>7ee62d5c-5811-45ce-9cef-c09d143f041a</td>\n",
       "      <td>12559.0</td>\n",
       "      <td>False</td>\n",
       "      <td>@theoddmother @lofioramonti In Italia la Ferre...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>@theoddmother, @lofioramonti</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>41.898417</td>\n",
       "      <td>12.545146</td>\n",
       "      <td>41.898417</td>\n",
       "      <td>12.545146</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2019-09-25</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>32534</td>\n",
       "      <td>twitter</td>\n",
       "      <td>1183821988437135362</td>\n",
       "      <td>7ee62d5c-5811-45ce-9cef-c09d143f041a</td>\n",
       "      <td>12559.0</td>\n",
       "      <td>False</td>\n",
       "      <td>@gianni_la_marca @angelofilippi Per il mercato...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>@gianni_la_marca, @angelofilippi</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>41.898417</td>\n",
       "      <td>12.545146</td>\n",
       "      <td>41.898417</td>\n",
       "      <td>12.545146</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2019-10-14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>32064</td>\n",
       "      <td>twitter</td>\n",
       "      <td>1186949229438263297</td>\n",
       "      <td>7ee62d5c-5811-45ce-9cef-c09d143f041a</td>\n",
       "      <td>12559.0</td>\n",
       "      <td>False</td>\n",
       "      <td>Siamo in un pericoloso paradosso con il Paese ...</td>\n",
       "      <td>https://t.co/lkkYO9w8VC</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2019-10-23</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>278</th>\n",
       "      <td>1013</td>\n",
       "      <td>twitter</td>\n",
       "      <td>1418810115445166080</td>\n",
       "      <td>7ee62d5c-5811-45ce-9cef-c09d143f041a</td>\n",
       "      <td>12559.0</td>\n",
       "      <td>False</td>\n",
       "      <td>Ferrero, la strategia green del big di Alba (è...</td>\n",
       "      <td>https://t.co/r2XvO05KdN, https://t.co/t8k8vmUlQk</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2021-07-24</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>279</th>\n",
       "      <td>841</td>\n",
       "      <td>twitter</td>\n",
       "      <td>1420717382830415880</td>\n",
       "      <td>7ee62d5c-5811-45ce-9cef-c09d143f041a</td>\n",
       "      <td>12559.0</td>\n",
       "      <td>False</td>\n",
       "      <td>Ferrero l'azienda meglio percepita sul territo...</td>\n",
       "      <td>https://t.co/4O4c31bPiq, https://t.co/n8RpxZP5Vu</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2021-07-29</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>280</th>\n",
       "      <td>822</td>\n",
       "      <td>twitter</td>\n",
       "      <td>1421105542945202182</td>\n",
       "      <td>7ee62d5c-5811-45ce-9cef-c09d143f041a</td>\n",
       "      <td>12559.0</td>\n",
       "      <td>False</td>\n",
       "      <td>L’attimo fuggente, online il numero di agosto ...</td>\n",
       "      <td>https://t.co/Q9FPElZrOU</td>\n",
       "      <td>@ferrero_eu, @generali, @prada, @armani, @ciro...</td>\n",
       "      <td>#luigicremonini, #vincenzocremonini, #alberton...</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2021-07-30</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>281</th>\n",
       "      <td>467</td>\n",
       "      <td>twitter</td>\n",
       "      <td>1426075939201421313</td>\n",
       "      <td>7ee62d5c-5811-45ce-9cef-c09d143f041a</td>\n",
       "      <td>12559.0</td>\n",
       "      <td>False</td>\n",
       "      <td>Mediobanca, Gavio arrotonda la quota. I Ferrer...</td>\n",
       "      <td>https://t.co/QPX7deAYBt, https://t.co/XUfJMFKsFE</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2021-08-13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>282</th>\n",
       "      <td>61</td>\n",
       "      <td>twitter</td>\n",
       "      <td>1430765728890859524</td>\n",
       "      <td>7ee62d5c-5811-45ce-9cef-c09d143f041a</td>\n",
       "      <td>12559.0</td>\n",
       "      <td>False</td>\n",
       "      <td>Da anni le aziende alimentari prendono impegni...</td>\n",
       "      <td>https://t.co/79pL8eW8Dm, https://t.co/3MC4xgol6A</td>\n",
       "      <td>@foodwatch_de</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2021-08-26</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>283 rows × 64 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     Unnamed: 0   social                   id  \\\n",
       "0         34648  twitter  1174618145786650624   \n",
       "1         34065  twitter  1176441842096910336   \n",
       "2         33847  twitter  1176748033066844160   \n",
       "3         32534  twitter  1183821988437135362   \n",
       "4         32064  twitter  1186949229438263297   \n",
       "..          ...      ...                  ...   \n",
       "278        1013  twitter  1418810115445166080   \n",
       "279         841  twitter  1420717382830415880   \n",
       "280         822  twitter  1421105542945202182   \n",
       "281         467  twitter  1426075939201421313   \n",
       "282          61  twitter  1430765728890859524   \n",
       "\n",
       "                                 topic_id  rule_id  is_comment  \\\n",
       "0    7ee62d5c-5811-45ce-9cef-c09d143f041a  12559.0       False   \n",
       "1    7ee62d5c-5811-45ce-9cef-c09d143f041a  12559.0       False   \n",
       "2    7ee62d5c-5811-45ce-9cef-c09d143f041a  12559.0       False   \n",
       "3    7ee62d5c-5811-45ce-9cef-c09d143f041a  12559.0       False   \n",
       "4    7ee62d5c-5811-45ce-9cef-c09d143f041a  12559.0       False   \n",
       "..                                    ...      ...         ...   \n",
       "278  7ee62d5c-5811-45ce-9cef-c09d143f041a  12559.0       False   \n",
       "279  7ee62d5c-5811-45ce-9cef-c09d143f041a  12559.0       False   \n",
       "280  7ee62d5c-5811-45ce-9cef-c09d143f041a  12559.0       False   \n",
       "281  7ee62d5c-5811-45ce-9cef-c09d143f041a  12559.0       False   \n",
       "282  7ee62d5c-5811-45ce-9cef-c09d143f041a  12559.0       False   \n",
       "\n",
       "                                                  text  \\\n",
       "0    La Ferrero riconosce a 6000 dipendenti un prem...   \n",
       "1    Il Ministro Fioramonti si permette di giustifi...   \n",
       "2    @theoddmother @lofioramonti In Italia la Ferre...   \n",
       "3    @gianni_la_marca @angelofilippi Per il mercato...   \n",
       "4    Siamo in un pericoloso paradosso con il Paese ...   \n",
       "..                                                 ...   \n",
       "278  Ferrero, la strategia green del big di Alba (è...   \n",
       "279  Ferrero l'azienda meglio percepita sul territo...   \n",
       "280  L’attimo fuggente, online il numero di agosto ...   \n",
       "281  Mediobanca, Gavio arrotonda la quota. I Ferrer...   \n",
       "282  Da anni le aziende alimentari prendono impegni...   \n",
       "\n",
       "                                                links  \\\n",
       "0                             https://t.co/T3wBQLWAeo   \n",
       "1                                                 NaN   \n",
       "2                                                 NaN   \n",
       "3                                                 NaN   \n",
       "4                             https://t.co/lkkYO9w8VC   \n",
       "..                                                ...   \n",
       "278  https://t.co/r2XvO05KdN, https://t.co/t8k8vmUlQk   \n",
       "279  https://t.co/4O4c31bPiq, https://t.co/n8RpxZP5Vu   \n",
       "280                           https://t.co/Q9FPElZrOU   \n",
       "281  https://t.co/QPX7deAYBt, https://t.co/XUfJMFKsFE   \n",
       "282  https://t.co/79pL8eW8Dm, https://t.co/3MC4xgol6A   \n",
       "\n",
       "                                              mentions  \\\n",
       "0                                                  NaN   \n",
       "1                                        @lofioramonti   \n",
       "2                         @theoddmother, @lofioramonti   \n",
       "3                     @gianni_la_marca, @angelofilippi   \n",
       "4                                                  NaN   \n",
       "..                                                 ...   \n",
       "278                                                NaN   \n",
       "279                                                NaN   \n",
       "280  @ferrero_eu, @generali, @prada, @armani, @ciro...   \n",
       "281                                                NaN   \n",
       "282                                      @foodwatch_de   \n",
       "\n",
       "                                              hashtags  ...  \\\n",
       "0                                                  NaN  ...   \n",
       "1                                                  NaN  ...   \n",
       "2                                                  NaN  ...   \n",
       "3                                                  NaN  ...   \n",
       "4                                                  NaN  ...   \n",
       "..                                                 ...  ...   \n",
       "278                                                NaN  ...   \n",
       "279                                                NaN  ...   \n",
       "280  #luigicremonini, #vincenzocremonini, #alberton...  ...   \n",
       "281                                                NaN  ...   \n",
       "282                                                NaN  ...   \n",
       "\n",
       "    place_location_lat place_location_lon  place_lat  place_lon  \\\n",
       "0                  NaN                NaN        NaN        NaN   \n",
       "1            41.898417          12.545146  41.898417  12.545146   \n",
       "2            41.898417          12.545146  41.898417  12.545146   \n",
       "3            41.898417          12.545146  41.898417  12.545146   \n",
       "4                  NaN                NaN        NaN        NaN   \n",
       "..                 ...                ...        ...        ...   \n",
       "278                NaN                NaN        NaN        NaN   \n",
       "279                NaN                NaN        NaN        NaN   \n",
       "280                NaN                NaN        NaN        NaN   \n",
       "281                NaN                NaN        NaN        NaN   \n",
       "282                NaN                NaN        NaN        NaN   \n",
       "\n",
       "    user_place_country_code  score_spam  score_performance  user_score_rank  \\\n",
       "0                       NaN         NaN                NaN              NaN   \n",
       "1                       NaN         NaN                NaN              NaN   \n",
       "2                       NaN         NaN                NaN              NaN   \n",
       "3                       NaN         NaN                NaN              NaN   \n",
       "4                       NaN         NaN                NaN              NaN   \n",
       "..                      ...         ...                ...              ...   \n",
       "278                     NaN         NaN                NaN              NaN   \n",
       "279                     NaN         NaN                NaN              NaN   \n",
       "280                     NaN         NaN                NaN              NaN   \n",
       "281                     NaN         NaN                NaN              NaN   \n",
       "282                     NaN         NaN                NaN              NaN   \n",
       "\n",
       "     score_rating  date_index  \n",
       "0             NaN  2019-09-19  \n",
       "1             NaN  2019-09-24  \n",
       "2             NaN  2019-09-25  \n",
       "3             NaN  2019-10-14  \n",
       "4             NaN  2019-10-23  \n",
       "..            ...         ...  \n",
       "278           NaN  2021-07-24  \n",
       "279           NaN  2021-07-29  \n",
       "280           NaN  2021-07-30  \n",
       "281           NaN  2021-08-13  \n",
       "282           NaN  2021-08-26  \n",
       "\n",
       "[283 rows x 64 columns]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "auth_filtered_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "spikes_df = convert_datetime_col_to_iso_format(spikes_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>mIndex</th>\n",
       "      <th>volume</th>\n",
       "      <th>sentiment</th>\n",
       "      <th>sign</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>date_index</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2019-08-20</th>\n",
       "      <td>2.602240</td>\n",
       "      <td>96</td>\n",
       "      <td>0.373657</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-08-21</th>\n",
       "      <td>13.554951</td>\n",
       "      <td>378</td>\n",
       "      <td>0.373657</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-08-22</th>\n",
       "      <td>5.825910</td>\n",
       "      <td>179</td>\n",
       "      <td>0.373657</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-08-23</th>\n",
       "      <td>2.485722</td>\n",
       "      <td>93</td>\n",
       "      <td>0.373657</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-08-24</th>\n",
       "      <td>1.006199</td>\n",
       "      <td>71</td>\n",
       "      <td>0.373657</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2021-08-19</th>\n",
       "      <td>0.774941</td>\n",
       "      <td>36</td>\n",
       "      <td>0.524922</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2021-08-20</th>\n",
       "      <td>0.858849</td>\n",
       "      <td>38</td>\n",
       "      <td>0.493919</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2021-08-23</th>\n",
       "      <td>2.040529</td>\n",
       "      <td>67</td>\n",
       "      <td>0.468272</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2021-08-24</th>\n",
       "      <td>1.287879</td>\n",
       "      <td>49</td>\n",
       "      <td>0.360722</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2021-08-25</th>\n",
       "      <td>0.679956</td>\n",
       "      <td>41</td>\n",
       "      <td>0.437569</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>165 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "               mIndex  volume  sentiment   sign\n",
       "date_index                                     \n",
       "2019-08-20   2.602240      96   0.373657  False\n",
       "2019-08-21  13.554951     378   0.373657  False\n",
       "2019-08-22   5.825910     179   0.373657  False\n",
       "2019-08-23   2.485722      93   0.373657  False\n",
       "2019-08-24   1.006199      71   0.373657  False\n",
       "...               ...     ...        ...    ...\n",
       "2021-08-19   0.774941      36   0.524922   True\n",
       "2021-08-20   0.858849      38   0.493919  False\n",
       "2021-08-23   2.040529      67   0.468272  False\n",
       "2021-08-24   1.287879      49   0.360722  False\n",
       "2021-08-25   0.679956      41   0.437569  False\n",
       "\n",
       "[165 rows x 4 columns]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "spikes_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "spikes_auth_df = spikes_df.merge(auth_filtered_df, on='date_index')\n",
    "# spikes_auth_df = pd.concat(spikes_df, auth_filtered_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>date_index</th>\n",
       "      <th>mIndex</th>\n",
       "      <th>volume</th>\n",
       "      <th>sentiment</th>\n",
       "      <th>sign</th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>social</th>\n",
       "      <th>id</th>\n",
       "      <th>topic_id</th>\n",
       "      <th>rule_id</th>\n",
       "      <th>...</th>\n",
       "      <th>place_country_code</th>\n",
       "      <th>place_location_lat</th>\n",
       "      <th>place_location_lon</th>\n",
       "      <th>place_lat</th>\n",
       "      <th>place_lon</th>\n",
       "      <th>user_place_country_code</th>\n",
       "      <th>score_spam</th>\n",
       "      <th>score_performance</th>\n",
       "      <th>user_score_rank</th>\n",
       "      <th>score_rating</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2019-08-21</td>\n",
       "      <td>13.554951</td>\n",
       "      <td>378</td>\n",
       "      <td>0.373657</td>\n",
       "      <td>False</td>\n",
       "      <td>36721</td>\n",
       "      <td>twitter</td>\n",
       "      <td>1163965966323109889</td>\n",
       "      <td>7ee62d5c-5811-45ce-9cef-c09d143f041a</td>\n",
       "      <td>12559.0</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2019-08-22</td>\n",
       "      <td>5.825910</td>\n",
       "      <td>179</td>\n",
       "      <td>0.373657</td>\n",
       "      <td>False</td>\n",
       "      <td>36180</td>\n",
       "      <td>twitter</td>\n",
       "      <td>1164639655574196224</td>\n",
       "      <td>7ee62d5c-5811-45ce-9cef-c09d143f041a</td>\n",
       "      <td>12559.0</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2019-08-23</td>\n",
       "      <td>2.485722</td>\n",
       "      <td>93</td>\n",
       "      <td>0.373657</td>\n",
       "      <td>False</td>\n",
       "      <td>36097</td>\n",
       "      <td>twitter</td>\n",
       "      <td>1164947272284672001</td>\n",
       "      <td>7ee62d5c-5811-45ce-9cef-c09d143f041a</td>\n",
       "      <td>12559.0</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2019-09-18</td>\n",
       "      <td>4.131696</td>\n",
       "      <td>202</td>\n",
       "      <td>0.366064</td>\n",
       "      <td>False</td>\n",
       "      <td>34882</td>\n",
       "      <td>twitter</td>\n",
       "      <td>1174264995162599426</td>\n",
       "      <td>7ee62d5c-5811-45ce-9cef-c09d143f041a</td>\n",
       "      <td>12559.0</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2019-09-18</td>\n",
       "      <td>4.131696</td>\n",
       "      <td>202</td>\n",
       "      <td>0.366064</td>\n",
       "      <td>False</td>\n",
       "      <td>34844</td>\n",
       "      <td>twitter</td>\n",
       "      <td>1174293258714808321</td>\n",
       "      <td>7ee62d5c-5811-45ce-9cef-c09d143f041a</td>\n",
       "      <td>12559.0</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>135</th>\n",
       "      <td>2021-06-01</td>\n",
       "      <td>0.798738</td>\n",
       "      <td>80</td>\n",
       "      <td>0.553408</td>\n",
       "      <td>True</td>\n",
       "      <td>2989</td>\n",
       "      <td>twitter</td>\n",
       "      <td>1399674473960206336</td>\n",
       "      <td>7ee62d5c-5811-45ce-9cef-c09d143f041a</td>\n",
       "      <td>12559.0</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>136</th>\n",
       "      <td>2021-06-01</td>\n",
       "      <td>0.798738</td>\n",
       "      <td>80</td>\n",
       "      <td>0.553408</td>\n",
       "      <td>True</td>\n",
       "      <td>2967</td>\n",
       "      <td>twitter</td>\n",
       "      <td>1399707442536095744</td>\n",
       "      <td>7ee62d5c-5811-45ce-9cef-c09d143f041a</td>\n",
       "      <td>12559.0</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>137</th>\n",
       "      <td>2021-06-01</td>\n",
       "      <td>0.798738</td>\n",
       "      <td>80</td>\n",
       "      <td>0.553408</td>\n",
       "      <td>True</td>\n",
       "      <td>2959</td>\n",
       "      <td>twitter</td>\n",
       "      <td>1399733841699520517</td>\n",
       "      <td>7ee62d5c-5811-45ce-9cef-c09d143f041a</td>\n",
       "      <td>12559.0</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>138</th>\n",
       "      <td>2021-07-20</td>\n",
       "      <td>0.631011</td>\n",
       "      <td>32</td>\n",
       "      <td>0.295342</td>\n",
       "      <td>False</td>\n",
       "      <td>1115</td>\n",
       "      <td>twitter</td>\n",
       "      <td>1417393591022936065</td>\n",
       "      <td>7ee62d5c-5811-45ce-9cef-c09d143f041a</td>\n",
       "      <td>12559.0</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>139</th>\n",
       "      <td>2021-07-21</td>\n",
       "      <td>0.525460</td>\n",
       "      <td>31</td>\n",
       "      <td>0.396534</td>\n",
       "      <td>False</td>\n",
       "      <td>1070</td>\n",
       "      <td>twitter</td>\n",
       "      <td>1417876798998732810</td>\n",
       "      <td>7ee62d5c-5811-45ce-9cef-c09d143f041a</td>\n",
       "      <td>12559.0</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>140 rows × 68 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     date_index     mIndex  volume  sentiment   sign  Unnamed: 0   social  \\\n",
       "0    2019-08-21  13.554951     378   0.373657  False       36721  twitter   \n",
       "1    2019-08-22   5.825910     179   0.373657  False       36180  twitter   \n",
       "2    2019-08-23   2.485722      93   0.373657  False       36097  twitter   \n",
       "3    2019-09-18   4.131696     202   0.366064  False       34882  twitter   \n",
       "4    2019-09-18   4.131696     202   0.366064  False       34844  twitter   \n",
       "..          ...        ...     ...        ...    ...         ...      ...   \n",
       "135  2021-06-01   0.798738      80   0.553408   True        2989  twitter   \n",
       "136  2021-06-01   0.798738      80   0.553408   True        2967  twitter   \n",
       "137  2021-06-01   0.798738      80   0.553408   True        2959  twitter   \n",
       "138  2021-07-20   0.631011      32   0.295342  False        1115  twitter   \n",
       "139  2021-07-21   0.525460      31   0.396534  False        1070  twitter   \n",
       "\n",
       "                      id                              topic_id  rule_id  ...  \\\n",
       "0    1163965966323109889  7ee62d5c-5811-45ce-9cef-c09d143f041a  12559.0  ...   \n",
       "1    1164639655574196224  7ee62d5c-5811-45ce-9cef-c09d143f041a  12559.0  ...   \n",
       "2    1164947272284672001  7ee62d5c-5811-45ce-9cef-c09d143f041a  12559.0  ...   \n",
       "3    1174264995162599426  7ee62d5c-5811-45ce-9cef-c09d143f041a  12559.0  ...   \n",
       "4    1174293258714808321  7ee62d5c-5811-45ce-9cef-c09d143f041a  12559.0  ...   \n",
       "..                   ...                                   ...      ...  ...   \n",
       "135  1399674473960206336  7ee62d5c-5811-45ce-9cef-c09d143f041a  12559.0  ...   \n",
       "136  1399707442536095744  7ee62d5c-5811-45ce-9cef-c09d143f041a  12559.0  ...   \n",
       "137  1399733841699520517  7ee62d5c-5811-45ce-9cef-c09d143f041a  12559.0  ...   \n",
       "138  1417393591022936065  7ee62d5c-5811-45ce-9cef-c09d143f041a  12559.0  ...   \n",
       "139  1417876798998732810  7ee62d5c-5811-45ce-9cef-c09d143f041a  12559.0  ...   \n",
       "\n",
       "     place_country_code place_location_lat place_location_lon place_lat  \\\n",
       "0                   NaN                NaN                NaN       NaN   \n",
       "1                   NaN                NaN                NaN       NaN   \n",
       "2                   NaN                NaN                NaN       NaN   \n",
       "3                   NaN                NaN                NaN       NaN   \n",
       "4                   NaN                NaN                NaN       NaN   \n",
       "..                  ...                ...                ...       ...   \n",
       "135                 NaN                NaN                NaN       NaN   \n",
       "136                 NaN                NaN                NaN       NaN   \n",
       "137                 NaN                NaN                NaN       NaN   \n",
       "138                 NaN                NaN                NaN       NaN   \n",
       "139                 NaN                NaN                NaN       NaN   \n",
       "\n",
       "    place_lon user_place_country_code score_spam score_performance  \\\n",
       "0         NaN                     NaN        NaN               NaN   \n",
       "1         NaN                     NaN        NaN               NaN   \n",
       "2         NaN                     NaN        NaN               NaN   \n",
       "3         NaN                     NaN        NaN               NaN   \n",
       "4         NaN                     NaN        NaN               NaN   \n",
       "..        ...                     ...        ...               ...   \n",
       "135       NaN                     NaN        NaN               NaN   \n",
       "136       NaN                     NaN        NaN               NaN   \n",
       "137       NaN                     NaN        NaN               NaN   \n",
       "138       NaN                     NaN        NaN               NaN   \n",
       "139       NaN                     NaN        NaN               NaN   \n",
       "\n",
       "     user_score_rank score_rating  \n",
       "0                NaN          NaN  \n",
       "1                NaN          NaN  \n",
       "2                NaN          NaN  \n",
       "3                NaN          NaN  \n",
       "4                NaN          NaN  \n",
       "..               ...          ...  \n",
       "135              NaN          NaN  \n",
       "136              NaN          NaN  \n",
       "137              NaN          NaN  \n",
       "138              NaN          NaN  \n",
       "139              NaN          NaN  \n",
       "\n",
       "[140 rows x 68 columns]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "spikes_auth_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>date_index</th>\n",
       "      <th>id</th>\n",
       "      <th>user_username</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2019-08-21</td>\n",
       "      <td>1163965966323109889</td>\n",
       "      <td>MilanoFinanza</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2019-08-22</td>\n",
       "      <td>1164639655574196224</td>\n",
       "      <td>repubblica</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2019-08-23</td>\n",
       "      <td>1164947272284672001</td>\n",
       "      <td>fattoquotidiano</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2019-09-18</td>\n",
       "      <td>1174264995162599426</td>\n",
       "      <td>Corriere</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2019-09-18</td>\n",
       "      <td>1174293258714808321</td>\n",
       "      <td>MilanoFinanza</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>135</th>\n",
       "      <td>2021-06-01</td>\n",
       "      <td>1399674473960206336</td>\n",
       "      <td>repubblica</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>136</th>\n",
       "      <td>2021-06-01</td>\n",
       "      <td>1399707442536095744</td>\n",
       "      <td>Affaritaliani</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>137</th>\n",
       "      <td>2021-06-01</td>\n",
       "      <td>1399733841699520517</td>\n",
       "      <td>ItaliaaTavola</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>138</th>\n",
       "      <td>2021-07-20</td>\n",
       "      <td>1417393591022936065</td>\n",
       "      <td>ilgiornale</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>139</th>\n",
       "      <td>2021-07-21</td>\n",
       "      <td>1417876798998732810</td>\n",
       "      <td>LegaSalvini</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>140 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     date_index                   id    user_username\n",
       "0    2019-08-21  1163965966323109889    MilanoFinanza\n",
       "1    2019-08-22  1164639655574196224       repubblica\n",
       "2    2019-08-23  1164947272284672001  fattoquotidiano\n",
       "3    2019-09-18  1174264995162599426         Corriere\n",
       "4    2019-09-18  1174293258714808321    MilanoFinanza\n",
       "..          ...                  ...              ...\n",
       "135  2021-06-01  1399674473960206336       repubblica\n",
       "136  2021-06-01  1399707442536095744    Affaritaliani\n",
       "137  2021-06-01  1399733841699520517    ItaliaaTavola\n",
       "138  2021-07-20  1417393591022936065       ilgiornale\n",
       "139  2021-07-21  1417876798998732810      LegaSalvini\n",
       "\n",
       "[140 rows x 3 columns]"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# id and username of authority tweets posted on a spike day\n",
    "spike_auth_tweets = spikes_auth_df[['date_index', 'id', 'user_username']]\n",
    "spike_auth_tweets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "spike_auth_tweets.to_csv('data/authority_filtered/auth_tweets.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>date_index</th>\n",
       "      <th>id</th>\n",
       "      <th>user_username</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>2019-11-30</td>\n",
       "      <td>1200773162155282444</td>\n",
       "      <td>stanzaselvaggia</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>2019-11-30</td>\n",
       "      <td>1200685934309728257</td>\n",
       "      <td>ilSalvagenteit</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>2019-12-05</td>\n",
       "      <td>1202583411128320000</td>\n",
       "      <td>ADUC1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33</th>\n",
       "      <td>2019-12-06</td>\n",
       "      <td>1202907937200627719</td>\n",
       "      <td>matteosalvinimi</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34</th>\n",
       "      <td>2019-12-06</td>\n",
       "      <td>1203005958793973760</td>\n",
       "      <td>GuidoCrosetto</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35</th>\n",
       "      <td>2019-12-06</td>\n",
       "      <td>1203010130884907009</td>\n",
       "      <td>NFratoianni</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36</th>\n",
       "      <td>2019-12-06</td>\n",
       "      <td>1202870147297562629</td>\n",
       "      <td>stanzaselvaggia</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37</th>\n",
       "      <td>2019-12-06</td>\n",
       "      <td>1202901039147618304</td>\n",
       "      <td>LucaBizzarri</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38</th>\n",
       "      <td>2019-12-06</td>\n",
       "      <td>1202842418485305344</td>\n",
       "      <td>giornalettismo</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    date_index                   id    user_username\n",
       "30  2019-11-30  1200773162155282444  stanzaselvaggia\n",
       "31  2019-11-30  1200685934309728257   ilSalvagenteit\n",
       "32  2019-12-05  1202583411128320000            ADUC1\n",
       "33  2019-12-06  1202907937200627719  matteosalvinimi\n",
       "34  2019-12-06  1203005958793973760    GuidoCrosetto\n",
       "35  2019-12-06  1203010130884907009      NFratoianni\n",
       "36  2019-12-06  1202870147297562629  stanzaselvaggia\n",
       "37  2019-12-06  1202901039147618304     LucaBizzarri\n",
       "38  2019-12-06  1202842418485305344   giornalettismo"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "filter_by_previous_dates(spike_auth_tweets, END_DATE, DATES_DELTA, 'date_index')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "# We need to explicitate each spike - tweet relashionship over the course of a chosen time delta. \n",
    "def get_all_spikes_content(df_spikes, \n",
    "                           df_spike_tweets, \n",
    "                           dates_delta=7,\n",
    "                           include_date=False):\n",
    "    spike_tweets_lst = []\n",
    "    for i, row in df_spikes.iterrows():\n",
    "        date = row['date_index']\n",
    "        spike_tweets_df = filter_by_previous_dates(df_spike_tweets, \n",
    "                                                   date, \n",
    "                                                   dates_delta, \n",
    "                                                   'date_index',\n",
    "                                                   include_date)\n",
    "        spike_tweets_df['spike_id'] = row['index']\n",
    "        spike_tweets_lst.append(spike_tweets_df)\n",
    "    return pd.concat(spike_tweets_lst)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>mIndex</th>\n",
       "      <th>volume</th>\n",
       "      <th>sentiment</th>\n",
       "      <th>sign</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>date_index</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2019-08-20</th>\n",
       "      <td>2.602240</td>\n",
       "      <td>96</td>\n",
       "      <td>0.373657</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-08-21</th>\n",
       "      <td>13.554951</td>\n",
       "      <td>378</td>\n",
       "      <td>0.373657</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-08-22</th>\n",
       "      <td>5.825910</td>\n",
       "      <td>179</td>\n",
       "      <td>0.373657</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-08-23</th>\n",
       "      <td>2.485722</td>\n",
       "      <td>93</td>\n",
       "      <td>0.373657</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-08-24</th>\n",
       "      <td>1.006199</td>\n",
       "      <td>71</td>\n",
       "      <td>0.373657</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2021-08-19</th>\n",
       "      <td>0.774941</td>\n",
       "      <td>36</td>\n",
       "      <td>0.524922</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2021-08-20</th>\n",
       "      <td>0.858849</td>\n",
       "      <td>38</td>\n",
       "      <td>0.493919</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2021-08-23</th>\n",
       "      <td>2.040529</td>\n",
       "      <td>67</td>\n",
       "      <td>0.468272</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2021-08-24</th>\n",
       "      <td>1.287879</td>\n",
       "      <td>49</td>\n",
       "      <td>0.360722</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2021-08-25</th>\n",
       "      <td>0.679956</td>\n",
       "      <td>41</td>\n",
       "      <td>0.437569</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>165 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "               mIndex  volume  sentiment   sign\n",
       "date_index                                     \n",
       "2019-08-20   2.602240      96   0.373657  False\n",
       "2019-08-21  13.554951     378   0.373657  False\n",
       "2019-08-22   5.825910     179   0.373657  False\n",
       "2019-08-23   2.485722      93   0.373657  False\n",
       "2019-08-24   1.006199      71   0.373657  False\n",
       "...               ...     ...        ...    ...\n",
       "2021-08-19   0.774941      36   0.524922   True\n",
       "2021-08-20   0.858849      38   0.493919  False\n",
       "2021-08-23   2.040529      67   0.468272  False\n",
       "2021-08-24   1.287879      49   0.360722  False\n",
       "2021-08-25   0.679956      41   0.437569  False\n",
       "\n",
       "[165 rows x 4 columns]"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "spikes_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>index</th>\n",
       "      <th>date_index</th>\n",
       "      <th>mIndex</th>\n",
       "      <th>volume</th>\n",
       "      <th>sentiment</th>\n",
       "      <th>sign</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>2019-08-20</td>\n",
       "      <td>2.602240</td>\n",
       "      <td>96</td>\n",
       "      <td>0.373657</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>2019-08-21</td>\n",
       "      <td>13.554951</td>\n",
       "      <td>378</td>\n",
       "      <td>0.373657</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>2019-08-22</td>\n",
       "      <td>5.825910</td>\n",
       "      <td>179</td>\n",
       "      <td>0.373657</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>2019-08-23</td>\n",
       "      <td>2.485722</td>\n",
       "      <td>93</td>\n",
       "      <td>0.373657</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>2019-08-24</td>\n",
       "      <td>1.006199</td>\n",
       "      <td>71</td>\n",
       "      <td>0.373657</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>160</th>\n",
       "      <td>160</td>\n",
       "      <td>2021-08-19</td>\n",
       "      <td>0.774941</td>\n",
       "      <td>36</td>\n",
       "      <td>0.524922</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>161</th>\n",
       "      <td>161</td>\n",
       "      <td>2021-08-20</td>\n",
       "      <td>0.858849</td>\n",
       "      <td>38</td>\n",
       "      <td>0.493919</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>162</th>\n",
       "      <td>162</td>\n",
       "      <td>2021-08-23</td>\n",
       "      <td>2.040529</td>\n",
       "      <td>67</td>\n",
       "      <td>0.468272</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>163</th>\n",
       "      <td>163</td>\n",
       "      <td>2021-08-24</td>\n",
       "      <td>1.287879</td>\n",
       "      <td>49</td>\n",
       "      <td>0.360722</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>164</th>\n",
       "      <td>164</td>\n",
       "      <td>2021-08-25</td>\n",
       "      <td>0.679956</td>\n",
       "      <td>41</td>\n",
       "      <td>0.437569</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>165 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     index  date_index     mIndex  volume  sentiment   sign\n",
       "0        0  2019-08-20   2.602240      96   0.373657  False\n",
       "1        1  2019-08-21  13.554951     378   0.373657  False\n",
       "2        2  2019-08-22   5.825910     179   0.373657  False\n",
       "3        3  2019-08-23   2.485722      93   0.373657  False\n",
       "4        4  2019-08-24   1.006199      71   0.373657  False\n",
       "..     ...         ...        ...     ...        ...    ...\n",
       "160    160  2021-08-19   0.774941      36   0.524922   True\n",
       "161    161  2021-08-20   0.858849      38   0.493919  False\n",
       "162    162  2021-08-23   2.040529      67   0.468272  False\n",
       "163    163  2021-08-24   1.287879      49   0.360722  False\n",
       "164    164  2021-08-25   0.679956      41   0.437569  False\n",
       "\n",
       "[165 rows x 6 columns]"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "spikes_df_tmp = spikes_df.reset_index().reset_index()\n",
    "spikes_df_tmp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "date_index       63\n",
      "user_username    33\n",
      "dtype: int64\n"
     ]
    }
   ],
   "source": [
    "# (index, gg_spike, id_tweet, auth_username\n",
    "# (index, gg_spike, id_tweet, auth_username) join (gg_spike, mIndex, volume, ...)\n",
    "# double resetting index so that each spike can be associated to an id (spike_id)\n",
    "normalized_df = get_all_spikes_content(spikes_df_tmp, spike_auth_tweets)\n",
    "\n",
    "# Dataframe index needs to be spike_id + tweet_id to accurately represent every event connected to each spike\n",
    "normalized_df['spike_id'] = normalized_df['spike_id'].astype('int64')\n",
    "normalized_df['id'] = normalized_df['id'].astype('int64')\n",
    "normalized_df = normalized_df.set_index(['spike_id', 'id'], verify_integrity=True)\n",
    "print(normalized_df.nunique())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>date_index</th>\n",
       "      <th>user_username</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>spike_id</th>\n",
       "      <th>id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <th>1163965966323109889</th>\n",
       "      <td>2019-08-21</td>\n",
       "      <td>MilanoFinanza</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">3</th>\n",
       "      <th>1163965966323109889</th>\n",
       "      <td>2019-08-21</td>\n",
       "      <td>MilanoFinanza</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1164639655574196224</th>\n",
       "      <td>2019-08-22</td>\n",
       "      <td>repubblica</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">4</th>\n",
       "      <th>1163965966323109889</th>\n",
       "      <td>2019-08-21</td>\n",
       "      <td>MilanoFinanza</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1164639655574196224</th>\n",
       "      <td>2019-08-22</td>\n",
       "      <td>repubblica</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>148</th>\n",
       "      <th>1399733841699520517</th>\n",
       "      <td>2021-06-01</td>\n",
       "      <td>ItaliaaTavola</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>155</th>\n",
       "      <th>1417393591022936065</th>\n",
       "      <td>2021-07-20</td>\n",
       "      <td>ilgiornale</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">156</th>\n",
       "      <th>1417393591022936065</th>\n",
       "      <td>2021-07-20</td>\n",
       "      <td>ilgiornale</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1417876798998732810</th>\n",
       "      <td>2021-07-21</td>\n",
       "      <td>LegaSalvini</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>157</th>\n",
       "      <th>1417876798998732810</th>\n",
       "      <td>2021-07-21</td>\n",
       "      <td>LegaSalvini</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>305 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                              date_index  user_username\n",
       "spike_id id                                            \n",
       "2        1163965966323109889  2019-08-21  MilanoFinanza\n",
       "3        1163965966323109889  2019-08-21  MilanoFinanza\n",
       "         1164639655574196224  2019-08-22     repubblica\n",
       "4        1163965966323109889  2019-08-21  MilanoFinanza\n",
       "         1164639655574196224  2019-08-22     repubblica\n",
       "...                                  ...            ...\n",
       "148      1399733841699520517  2021-06-01  ItaliaaTavola\n",
       "155      1417393591022936065  2021-07-20     ilgiornale\n",
       "156      1417393591022936065  2021-07-20     ilgiornale\n",
       "         1417876798998732810  2021-07-21    LegaSalvini\n",
       "157      1417876798998732810  2021-07-21    LegaSalvini\n",
       "\n",
       "[305 rows x 2 columns]"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "normalized_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "normalized_df.to_csv('data/authority_filtered/normalized_df.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "date_index       74\n",
      "user_username    35\n",
      "dtype: int64\n"
     ]
    }
   ],
   "source": [
    "normalized_date_included_df = get_all_spikes_content(spikes_df_tmp, spike_auth_tweets, include_date=True)\n",
    "\n",
    "# Dataframe index needs to be spike_id + tweet_id to accurately represent every event connected to each spike\n",
    "normalized_date_included_df = normalized_date_included_df.set_index(['spike_id', 'id'], verify_integrity=True)\n",
    "print(normalized_date_included_df.nunique())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>date_index</th>\n",
       "      <th>user_username</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>spike_id</th>\n",
       "      <th>id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <th>1163965966323109889</th>\n",
       "      <td>2019-08-21</td>\n",
       "      <td>MilanoFinanza</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">2</th>\n",
       "      <th>1163965966323109889</th>\n",
       "      <td>2019-08-21</td>\n",
       "      <td>MilanoFinanza</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1164639655574196224</th>\n",
       "      <td>2019-08-22</td>\n",
       "      <td>repubblica</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">3</th>\n",
       "      <th>1163965966323109889</th>\n",
       "      <td>2019-08-21</td>\n",
       "      <td>MilanoFinanza</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1164639655574196224</th>\n",
       "      <td>2019-08-22</td>\n",
       "      <td>repubblica</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">155</th>\n",
       "      <th>1417393591022936065</th>\n",
       "      <td>2021-07-20</td>\n",
       "      <td>ilgiornale</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1417876798998732810</th>\n",
       "      <td>2021-07-21</td>\n",
       "      <td>LegaSalvini</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">156</th>\n",
       "      <th>1417393591022936065</th>\n",
       "      <td>2021-07-20</td>\n",
       "      <td>ilgiornale</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1417876798998732810</th>\n",
       "      <td>2021-07-21</td>\n",
       "      <td>LegaSalvini</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>157</th>\n",
       "      <th>1417876798998732810</th>\n",
       "      <td>2021-07-21</td>\n",
       "      <td>LegaSalvini</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>445 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                              date_index  user_username\n",
       "spike_id id                                            \n",
       "1        1163965966323109889  2019-08-21  MilanoFinanza\n",
       "2        1163965966323109889  2019-08-21  MilanoFinanza\n",
       "         1164639655574196224  2019-08-22     repubblica\n",
       "3        1163965966323109889  2019-08-21  MilanoFinanza\n",
       "         1164639655574196224  2019-08-22     repubblica\n",
       "...                                  ...            ...\n",
       "155      1417393591022936065  2021-07-20     ilgiornale\n",
       "         1417876798998732810  2021-07-21    LegaSalvini\n",
       "156      1417393591022936065  2021-07-20     ilgiornale\n",
       "         1417876798998732810  2021-07-21    LegaSalvini\n",
       "157      1417876798998732810  2021-07-21    LegaSalvini\n",
       "\n",
       "[445 rows x 2 columns]"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "normalized_date_included_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "normalized_date_included_df.to_csv('data/authority_filtered/normalized_date_included_df.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'numpy.int64'>\n"
     ]
    }
   ],
   "source": [
    "print(type(normalized_df.reset_index()['id'][0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "merged_spikes_df = spikes_df_tmp.merge(normalized_df.reset_index().convert_dtypes(), left_on='index', right_on='spike_id', how='outer')\n",
    "merged_spikes_date_included_df =  spikes_df_tmp.merge(normalized_date_included_df.reset_index().convert_dtypes(), left_on='index', right_on='spike_id', how='outer')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>index</th>\n",
       "      <th>date_index_x</th>\n",
       "      <th>mIndex</th>\n",
       "      <th>volume</th>\n",
       "      <th>sentiment</th>\n",
       "      <th>sign</th>\n",
       "      <th>spike_id</th>\n",
       "      <th>id</th>\n",
       "      <th>date_index_y</th>\n",
       "      <th>user_username</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>2019-08-20</td>\n",
       "      <td>2.602240</td>\n",
       "      <td>96</td>\n",
       "      <td>0.373657</td>\n",
       "      <td>False</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>NaN</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>2019-08-21</td>\n",
       "      <td>13.554951</td>\n",
       "      <td>378</td>\n",
       "      <td>0.373657</td>\n",
       "      <td>False</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>NaN</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>2019-08-22</td>\n",
       "      <td>5.825910</td>\n",
       "      <td>179</td>\n",
       "      <td>0.373657</td>\n",
       "      <td>False</td>\n",
       "      <td>2</td>\n",
       "      <td>1163965966323109889</td>\n",
       "      <td>2019-08-21</td>\n",
       "      <td>MilanoFinanza</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>2019-08-23</td>\n",
       "      <td>2.485722</td>\n",
       "      <td>93</td>\n",
       "      <td>0.373657</td>\n",
       "      <td>False</td>\n",
       "      <td>3</td>\n",
       "      <td>1163965966323109889</td>\n",
       "      <td>2019-08-21</td>\n",
       "      <td>MilanoFinanza</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>3</td>\n",
       "      <td>2019-08-23</td>\n",
       "      <td>2.485722</td>\n",
       "      <td>93</td>\n",
       "      <td>0.373657</td>\n",
       "      <td>False</td>\n",
       "      <td>3</td>\n",
       "      <td>1164639655574196224</td>\n",
       "      <td>2019-08-22</td>\n",
       "      <td>repubblica</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>381</th>\n",
       "      <td>160</td>\n",
       "      <td>2021-08-19</td>\n",
       "      <td>0.774941</td>\n",
       "      <td>36</td>\n",
       "      <td>0.524922</td>\n",
       "      <td>True</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>NaN</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>382</th>\n",
       "      <td>161</td>\n",
       "      <td>2021-08-20</td>\n",
       "      <td>0.858849</td>\n",
       "      <td>38</td>\n",
       "      <td>0.493919</td>\n",
       "      <td>False</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>NaN</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>383</th>\n",
       "      <td>162</td>\n",
       "      <td>2021-08-23</td>\n",
       "      <td>2.040529</td>\n",
       "      <td>67</td>\n",
       "      <td>0.468272</td>\n",
       "      <td>False</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>NaN</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>384</th>\n",
       "      <td>163</td>\n",
       "      <td>2021-08-24</td>\n",
       "      <td>1.287879</td>\n",
       "      <td>49</td>\n",
       "      <td>0.360722</td>\n",
       "      <td>False</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>NaN</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>385</th>\n",
       "      <td>164</td>\n",
       "      <td>2021-08-25</td>\n",
       "      <td>0.679956</td>\n",
       "      <td>41</td>\n",
       "      <td>0.437569</td>\n",
       "      <td>False</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>NaN</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>386 rows × 10 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     index date_index_x     mIndex  volume  sentiment   sign  spike_id  \\\n",
       "0        0   2019-08-20   2.602240      96   0.373657  False      <NA>   \n",
       "1        1   2019-08-21  13.554951     378   0.373657  False      <NA>   \n",
       "2        2   2019-08-22   5.825910     179   0.373657  False         2   \n",
       "3        3   2019-08-23   2.485722      93   0.373657  False         3   \n",
       "4        3   2019-08-23   2.485722      93   0.373657  False         3   \n",
       "..     ...          ...        ...     ...        ...    ...       ...   \n",
       "381    160   2021-08-19   0.774941      36   0.524922   True      <NA>   \n",
       "382    161   2021-08-20   0.858849      38   0.493919  False      <NA>   \n",
       "383    162   2021-08-23   2.040529      67   0.468272  False      <NA>   \n",
       "384    163   2021-08-24   1.287879      49   0.360722  False      <NA>   \n",
       "385    164   2021-08-25   0.679956      41   0.437569  False      <NA>   \n",
       "\n",
       "                      id date_index_y  user_username  \n",
       "0                   <NA>          NaN           <NA>  \n",
       "1                   <NA>          NaN           <NA>  \n",
       "2    1163965966323109889   2019-08-21  MilanoFinanza  \n",
       "3    1163965966323109889   2019-08-21  MilanoFinanza  \n",
       "4    1164639655574196224   2019-08-22     repubblica  \n",
       "..                   ...          ...            ...  \n",
       "381                 <NA>          NaN           <NA>  \n",
       "382                 <NA>          NaN           <NA>  \n",
       "383                 <NA>          NaN           <NA>  \n",
       "384                 <NA>          NaN           <NA>  \n",
       "385                 <NA>          NaN           <NA>  \n",
       "\n",
       "[386 rows x 10 columns]"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "merged_spikes_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>index</th>\n",
       "      <th>date_index_x</th>\n",
       "      <th>mIndex</th>\n",
       "      <th>volume</th>\n",
       "      <th>sentiment</th>\n",
       "      <th>sign</th>\n",
       "      <th>spike_id</th>\n",
       "      <th>id</th>\n",
       "      <th>date_index_y</th>\n",
       "      <th>user_username</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>2019-08-20</td>\n",
       "      <td>2.602240</td>\n",
       "      <td>96</td>\n",
       "      <td>0.373657</td>\n",
       "      <td>False</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>NaN</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>2019-08-21</td>\n",
       "      <td>13.554951</td>\n",
       "      <td>378</td>\n",
       "      <td>0.373657</td>\n",
       "      <td>False</td>\n",
       "      <td>1</td>\n",
       "      <td>1163965966323109889</td>\n",
       "      <td>2019-08-21</td>\n",
       "      <td>MilanoFinanza</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>2019-08-22</td>\n",
       "      <td>5.825910</td>\n",
       "      <td>179</td>\n",
       "      <td>0.373657</td>\n",
       "      <td>False</td>\n",
       "      <td>2</td>\n",
       "      <td>1163965966323109889</td>\n",
       "      <td>2019-08-21</td>\n",
       "      <td>MilanoFinanza</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2</td>\n",
       "      <td>2019-08-22</td>\n",
       "      <td>5.825910</td>\n",
       "      <td>179</td>\n",
       "      <td>0.373657</td>\n",
       "      <td>False</td>\n",
       "      <td>2</td>\n",
       "      <td>1164639655574196224</td>\n",
       "      <td>2019-08-22</td>\n",
       "      <td>repubblica</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>3</td>\n",
       "      <td>2019-08-23</td>\n",
       "      <td>2.485722</td>\n",
       "      <td>93</td>\n",
       "      <td>0.373657</td>\n",
       "      <td>False</td>\n",
       "      <td>3</td>\n",
       "      <td>1163965966323109889</td>\n",
       "      <td>2019-08-21</td>\n",
       "      <td>MilanoFinanza</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>491</th>\n",
       "      <td>160</td>\n",
       "      <td>2021-08-19</td>\n",
       "      <td>0.774941</td>\n",
       "      <td>36</td>\n",
       "      <td>0.524922</td>\n",
       "      <td>True</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>NaN</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>492</th>\n",
       "      <td>161</td>\n",
       "      <td>2021-08-20</td>\n",
       "      <td>0.858849</td>\n",
       "      <td>38</td>\n",
       "      <td>0.493919</td>\n",
       "      <td>False</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>NaN</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>493</th>\n",
       "      <td>162</td>\n",
       "      <td>2021-08-23</td>\n",
       "      <td>2.040529</td>\n",
       "      <td>67</td>\n",
       "      <td>0.468272</td>\n",
       "      <td>False</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>NaN</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>494</th>\n",
       "      <td>163</td>\n",
       "      <td>2021-08-24</td>\n",
       "      <td>1.287879</td>\n",
       "      <td>49</td>\n",
       "      <td>0.360722</td>\n",
       "      <td>False</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>NaN</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>495</th>\n",
       "      <td>164</td>\n",
       "      <td>2021-08-25</td>\n",
       "      <td>0.679956</td>\n",
       "      <td>41</td>\n",
       "      <td>0.437569</td>\n",
       "      <td>False</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>NaN</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>496 rows × 10 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     index date_index_x     mIndex  volume  sentiment   sign  spike_id  \\\n",
       "0        0   2019-08-20   2.602240      96   0.373657  False      <NA>   \n",
       "1        1   2019-08-21  13.554951     378   0.373657  False         1   \n",
       "2        2   2019-08-22   5.825910     179   0.373657  False         2   \n",
       "3        2   2019-08-22   5.825910     179   0.373657  False         2   \n",
       "4        3   2019-08-23   2.485722      93   0.373657  False         3   \n",
       "..     ...          ...        ...     ...        ...    ...       ...   \n",
       "491    160   2021-08-19   0.774941      36   0.524922   True      <NA>   \n",
       "492    161   2021-08-20   0.858849      38   0.493919  False      <NA>   \n",
       "493    162   2021-08-23   2.040529      67   0.468272  False      <NA>   \n",
       "494    163   2021-08-24   1.287879      49   0.360722  False      <NA>   \n",
       "495    164   2021-08-25   0.679956      41   0.437569  False      <NA>   \n",
       "\n",
       "                      id date_index_y  user_username  \n",
       "0                   <NA>          NaN           <NA>  \n",
       "1    1163965966323109889   2019-08-21  MilanoFinanza  \n",
       "2    1163965966323109889   2019-08-21  MilanoFinanza  \n",
       "3    1164639655574196224   2019-08-22     repubblica  \n",
       "4    1163965966323109889   2019-08-21  MilanoFinanza  \n",
       "..                   ...          ...            ...  \n",
       "491                 <NA>          NaN           <NA>  \n",
       "492                 <NA>          NaN           <NA>  \n",
       "493                 <NA>          NaN           <NA>  \n",
       "494                 <NA>          NaN           <NA>  \n",
       "495                 <NA>          NaN           <NA>  \n",
       "\n",
       "[496 rows x 10 columns]"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "merged_spikes_date_included_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "merged_spikes_df['spike_window_date'] = merged_spikes_df['date_index_x']\n",
    "merged_spikes_df['tweet_date'] = merged_spikes_df['date_index_y']\n",
    "merged_spikes_df['tweet_id'] = merged_spikes_df['id']\n",
    "merged_spikes_df['spike_id'] = merged_spikes_df['index']\n",
    "merged_spikes_df = merged_spikes_df[['spike_window_date', 'spike_id', 'tweet_date', 'tweet_id', 'user_username', 'mIndex', 'volume', 'sentiment']]\n",
    "merged_spikes_df.convert_dtypes().set_index(['spike_window_date', 'tweet_id'], inplace=True)\n",
    "\n",
    "merged_spikes_date_included_df['spike_window_date'] = merged_spikes_date_included_df['date_index_x']\n",
    "merged_spikes_date_included_df['tweet_date'] = merged_spikes_date_included_df['date_index_y']\n",
    "merged_spikes_date_included_df['tweet_id'] = merged_spikes_date_included_df['id']\n",
    "merged_spikes_date_included_df['spike_id'] = merged_spikes_date_included_df['index']\n",
    "merged_spikes_date_included_df = merged_spikes_date_included_df[['spike_window_date', 'spike_id', 'tweet_date', 'tweet_id', 'user_username', 'mIndex', 'volume', 'sentiment']]\n",
    "merged_spikes_date_included_df.convert_dtypes().set_index(['spike_window_date', 'tweet_id'], inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>spike_window_date</th>\n",
       "      <th>spike_id</th>\n",
       "      <th>tweet_date</th>\n",
       "      <th>tweet_id</th>\n",
       "      <th>user_username</th>\n",
       "      <th>mIndex</th>\n",
       "      <th>volume</th>\n",
       "      <th>sentiment</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2019-08-20</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>2.602240</td>\n",
       "      <td>96</td>\n",
       "      <td>0.373657</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2019-08-21</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>13.554951</td>\n",
       "      <td>378</td>\n",
       "      <td>0.373657</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2019-08-22</td>\n",
       "      <td>2</td>\n",
       "      <td>2019-08-21</td>\n",
       "      <td>1163965966323109889</td>\n",
       "      <td>MilanoFinanza</td>\n",
       "      <td>5.825910</td>\n",
       "      <td>179</td>\n",
       "      <td>0.373657</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2019-08-23</td>\n",
       "      <td>3</td>\n",
       "      <td>2019-08-21</td>\n",
       "      <td>1163965966323109889</td>\n",
       "      <td>MilanoFinanza</td>\n",
       "      <td>2.485722</td>\n",
       "      <td>93</td>\n",
       "      <td>0.373657</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2019-08-23</td>\n",
       "      <td>3</td>\n",
       "      <td>2019-08-22</td>\n",
       "      <td>1164639655574196224</td>\n",
       "      <td>repubblica</td>\n",
       "      <td>2.485722</td>\n",
       "      <td>93</td>\n",
       "      <td>0.373657</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>381</th>\n",
       "      <td>2021-08-19</td>\n",
       "      <td>160</td>\n",
       "      <td>NaN</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>0.774941</td>\n",
       "      <td>36</td>\n",
       "      <td>0.524922</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>382</th>\n",
       "      <td>2021-08-20</td>\n",
       "      <td>161</td>\n",
       "      <td>NaN</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>0.858849</td>\n",
       "      <td>38</td>\n",
       "      <td>0.493919</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>383</th>\n",
       "      <td>2021-08-23</td>\n",
       "      <td>162</td>\n",
       "      <td>NaN</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>2.040529</td>\n",
       "      <td>67</td>\n",
       "      <td>0.468272</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>384</th>\n",
       "      <td>2021-08-24</td>\n",
       "      <td>163</td>\n",
       "      <td>NaN</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>1.287879</td>\n",
       "      <td>49</td>\n",
       "      <td>0.360722</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>385</th>\n",
       "      <td>2021-08-25</td>\n",
       "      <td>164</td>\n",
       "      <td>NaN</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>0.679956</td>\n",
       "      <td>41</td>\n",
       "      <td>0.437569</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>386 rows × 8 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    spike_window_date  spike_id  tweet_date             tweet_id  \\\n",
       "0          2019-08-20         0         NaN                 <NA>   \n",
       "1          2019-08-21         1         NaN                 <NA>   \n",
       "2          2019-08-22         2  2019-08-21  1163965966323109889   \n",
       "3          2019-08-23         3  2019-08-21  1163965966323109889   \n",
       "4          2019-08-23         3  2019-08-22  1164639655574196224   \n",
       "..                ...       ...         ...                  ...   \n",
       "381        2021-08-19       160         NaN                 <NA>   \n",
       "382        2021-08-20       161         NaN                 <NA>   \n",
       "383        2021-08-23       162         NaN                 <NA>   \n",
       "384        2021-08-24       163         NaN                 <NA>   \n",
       "385        2021-08-25       164         NaN                 <NA>   \n",
       "\n",
       "     user_username     mIndex  volume  sentiment  \n",
       "0             <NA>   2.602240      96   0.373657  \n",
       "1             <NA>  13.554951     378   0.373657  \n",
       "2    MilanoFinanza   5.825910     179   0.373657  \n",
       "3    MilanoFinanza   2.485722      93   0.373657  \n",
       "4       repubblica   2.485722      93   0.373657  \n",
       "..             ...        ...     ...        ...  \n",
       "381           <NA>   0.774941      36   0.524922  \n",
       "382           <NA>   0.858849      38   0.493919  \n",
       "383           <NA>   2.040529      67   0.468272  \n",
       "384           <NA>   1.287879      49   0.360722  \n",
       "385           <NA>   0.679956      41   0.437569  \n",
       "\n",
       "[386 rows x 8 columns]"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "merged_spikes_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>spike_window_date</th>\n",
       "      <th>spike_id</th>\n",
       "      <th>tweet_date</th>\n",
       "      <th>tweet_id</th>\n",
       "      <th>user_username</th>\n",
       "      <th>mIndex</th>\n",
       "      <th>volume</th>\n",
       "      <th>sentiment</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2019-08-20</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>2.602240</td>\n",
       "      <td>96</td>\n",
       "      <td>0.373657</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2019-08-21</td>\n",
       "      <td>1</td>\n",
       "      <td>2019-08-21</td>\n",
       "      <td>1163965966323109889</td>\n",
       "      <td>MilanoFinanza</td>\n",
       "      <td>13.554951</td>\n",
       "      <td>378</td>\n",
       "      <td>0.373657</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2019-08-22</td>\n",
       "      <td>2</td>\n",
       "      <td>2019-08-21</td>\n",
       "      <td>1163965966323109889</td>\n",
       "      <td>MilanoFinanza</td>\n",
       "      <td>5.825910</td>\n",
       "      <td>179</td>\n",
       "      <td>0.373657</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2019-08-22</td>\n",
       "      <td>2</td>\n",
       "      <td>2019-08-22</td>\n",
       "      <td>1164639655574196224</td>\n",
       "      <td>repubblica</td>\n",
       "      <td>5.825910</td>\n",
       "      <td>179</td>\n",
       "      <td>0.373657</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2019-08-23</td>\n",
       "      <td>3</td>\n",
       "      <td>2019-08-21</td>\n",
       "      <td>1163965966323109889</td>\n",
       "      <td>MilanoFinanza</td>\n",
       "      <td>2.485722</td>\n",
       "      <td>93</td>\n",
       "      <td>0.373657</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>491</th>\n",
       "      <td>2021-08-19</td>\n",
       "      <td>160</td>\n",
       "      <td>NaN</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>0.774941</td>\n",
       "      <td>36</td>\n",
       "      <td>0.524922</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>492</th>\n",
       "      <td>2021-08-20</td>\n",
       "      <td>161</td>\n",
       "      <td>NaN</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>0.858849</td>\n",
       "      <td>38</td>\n",
       "      <td>0.493919</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>493</th>\n",
       "      <td>2021-08-23</td>\n",
       "      <td>162</td>\n",
       "      <td>NaN</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>2.040529</td>\n",
       "      <td>67</td>\n",
       "      <td>0.468272</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>494</th>\n",
       "      <td>2021-08-24</td>\n",
       "      <td>163</td>\n",
       "      <td>NaN</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>1.287879</td>\n",
       "      <td>49</td>\n",
       "      <td>0.360722</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>495</th>\n",
       "      <td>2021-08-25</td>\n",
       "      <td>164</td>\n",
       "      <td>NaN</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>0.679956</td>\n",
       "      <td>41</td>\n",
       "      <td>0.437569</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>496 rows × 8 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    spike_window_date  spike_id  tweet_date             tweet_id  \\\n",
       "0          2019-08-20         0         NaN                 <NA>   \n",
       "1          2019-08-21         1  2019-08-21  1163965966323109889   \n",
       "2          2019-08-22         2  2019-08-21  1163965966323109889   \n",
       "3          2019-08-22         2  2019-08-22  1164639655574196224   \n",
       "4          2019-08-23         3  2019-08-21  1163965966323109889   \n",
       "..                ...       ...         ...                  ...   \n",
       "491        2021-08-19       160         NaN                 <NA>   \n",
       "492        2021-08-20       161         NaN                 <NA>   \n",
       "493        2021-08-23       162         NaN                 <NA>   \n",
       "494        2021-08-24       163         NaN                 <NA>   \n",
       "495        2021-08-25       164         NaN                 <NA>   \n",
       "\n",
       "     user_username     mIndex  volume  sentiment  \n",
       "0             <NA>   2.602240      96   0.373657  \n",
       "1    MilanoFinanza  13.554951     378   0.373657  \n",
       "2    MilanoFinanza   5.825910     179   0.373657  \n",
       "3       repubblica   5.825910     179   0.373657  \n",
       "4    MilanoFinanza   2.485722      93   0.373657  \n",
       "..             ...        ...     ...        ...  \n",
       "491           <NA>   0.774941      36   0.524922  \n",
       "492           <NA>   0.858849      38   0.493919  \n",
       "493           <NA>   2.040529      67   0.468272  \n",
       "494           <NA>   1.287879      49   0.360722  \n",
       "495           <NA>   0.679956      41   0.437569  \n",
       "\n",
       "[496 rows x 8 columns]"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "merged_spikes_date_included_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_formatted_date(date):\n",
    "    return pd.to_datetime(date, format='%Y/%m/%d', errors='coerce').date()\n",
    "\n",
    "def collapse_consecutive_spike_days(df):\n",
    "    output = df.copy()\n",
    "    collapsed_spikes = 0\n",
    "    formatted_date = get_formatted_date(df['spike_window_date'].iloc[0])\n",
    "    prev_date = formatted_date\n",
    "    spike_starting_date = formatted_date\n",
    "    spike_ending_date = formatted_date\n",
    "    output['spike_starting_date'] = spike_starting_date\n",
    "    output['spike_ending_date'] = spike_ending_date\n",
    "    last_spike_start_row_idx = -1\n",
    "    prev_spike = df['spike_id'].iloc[0]\n",
    "    \n",
    "    for i, row in df.iterrows():\n",
    "        formatted_date = get_formatted_date(row['spike_window_date'])\n",
    "        curr_prev_date = formatted_date - pd.to_timedelta(1, unit='days')\n",
    "        if(curr_prev_date == prev_date):\n",
    "            collapsed_spikes += 1\n",
    "        elif(formatted_date != prev_date):\n",
    "            spike_starting_date = formatted_date\n",
    "        \n",
    "        prev_date = formatted_date\n",
    "        curr_spike = row['spike_id'] - collapsed_spikes\n",
    "        # prev_spike = prev_spike - collapsed_spikes\n",
    "        output['spike_id'].iloc[i] = curr_spike\n",
    "        output['spike_starting_date'].iloc[i] = spike_starting_date\n",
    "        output['spike_ending_date'].iloc[i] = formatted_date\n",
    "        try:\n",
    "            formatted_ending_date = get_formatted_date(df['spike_window_date'].iloc[i+1])       # execute the next block only if there is a succeeding tuple\n",
    "            if (formatted_ending_date > (formatted_date + pd.to_timedelta(1, unit='days'))):    # if the next date has a timedelta superior to 1 from the examined one, it means that it belongs to another spike\n",
    "                if (curr_spike==prev_spike):                                                    # update the ending date only if the spike isn't changed\n",
    "                    for j in range(last_spike_start_row_idx, i):\n",
    "                        output['spike_ending_date'].iloc[j] = formatted_date                    \n",
    "                last_spike_start_row_idx = i+1     \n",
    "        except:\n",
    "            print('There are not next lines in the DataFrame')\n",
    "        else:\n",
    "            last_row_idx = i\n",
    "        prev_spike = curr_spike\n",
    "\n",
    "        if (curr_spike==prev_spike):\n",
    "            for k in range(last_spike_start_row_idx, last_row_idx + 1):\n",
    "                output['spike_ending_date'].iloc[k] = formatted_date\n",
    "    return output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "merged_spikes_df.to_csv('data/authority_filtered/merged_spikes_df.csv')\n",
    "merged_spikes_date_included_df.to_csv('data/authority_filtered/merged_spikes_date_included_df.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\alxau\\anaconda3\\lib\\site-packages\\pandas\\core\\indexing.py:1637: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  self._setitem_single_block(indexer, value, name)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "There are not next lines in the DataFrame\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>index</th>\n",
       "      <th>spike_window_date</th>\n",
       "      <th>spike_id</th>\n",
       "      <th>tweet_date</th>\n",
       "      <th>tweet_id</th>\n",
       "      <th>user_username</th>\n",
       "      <th>mIndex</th>\n",
       "      <th>volume</th>\n",
       "      <th>sentiment</th>\n",
       "      <th>spike_starting_date</th>\n",
       "      <th>spike_ending_date</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>2019-08-22</td>\n",
       "      <td>0</td>\n",
       "      <td>2019-08-21</td>\n",
       "      <td>1163965966323109889</td>\n",
       "      <td>MilanoFinanza</td>\n",
       "      <td>5.825910</td>\n",
       "      <td>179</td>\n",
       "      <td>0.373657</td>\n",
       "      <td>2019-08-20</td>\n",
       "      <td>2019-08-25</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>2019-08-23</td>\n",
       "      <td>0</td>\n",
       "      <td>2019-08-21</td>\n",
       "      <td>1163965966323109889</td>\n",
       "      <td>MilanoFinanza</td>\n",
       "      <td>2.485722</td>\n",
       "      <td>93</td>\n",
       "      <td>0.373657</td>\n",
       "      <td>2019-08-20</td>\n",
       "      <td>2019-08-25</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>2019-08-23</td>\n",
       "      <td>0</td>\n",
       "      <td>2019-08-22</td>\n",
       "      <td>1164639655574196224</td>\n",
       "      <td>repubblica</td>\n",
       "      <td>2.485722</td>\n",
       "      <td>93</td>\n",
       "      <td>0.373657</td>\n",
       "      <td>2019-08-20</td>\n",
       "      <td>2019-08-25</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>5</td>\n",
       "      <td>2019-08-24</td>\n",
       "      <td>0</td>\n",
       "      <td>2019-08-21</td>\n",
       "      <td>1163965966323109889</td>\n",
       "      <td>MilanoFinanza</td>\n",
       "      <td>1.006199</td>\n",
       "      <td>71</td>\n",
       "      <td>0.373657</td>\n",
       "      <td>2019-08-20</td>\n",
       "      <td>2019-08-25</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>6</td>\n",
       "      <td>2019-08-24</td>\n",
       "      <td>0</td>\n",
       "      <td>2019-08-22</td>\n",
       "      <td>1164639655574196224</td>\n",
       "      <td>repubblica</td>\n",
       "      <td>1.006199</td>\n",
       "      <td>71</td>\n",
       "      <td>0.373657</td>\n",
       "      <td>2019-08-20</td>\n",
       "      <td>2019-08-25</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>368</th>\n",
       "      <td>368</td>\n",
       "      <td>2021-06-07</td>\n",
       "      <td>77</td>\n",
       "      <td>2021-06-01</td>\n",
       "      <td>1399733841699520517</td>\n",
       "      <td>ItaliaaTavola</td>\n",
       "      <td>0.829249</td>\n",
       "      <td>69</td>\n",
       "      <td>0.453243</td>\n",
       "      <td>2021-06-07</td>\n",
       "      <td>2021-06-07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>375</th>\n",
       "      <td>375</td>\n",
       "      <td>2021-07-21</td>\n",
       "      <td>82</td>\n",
       "      <td>2021-07-20</td>\n",
       "      <td>1417393591022936065</td>\n",
       "      <td>ilgiornale</td>\n",
       "      <td>0.525460</td>\n",
       "      <td>31</td>\n",
       "      <td>0.396534</td>\n",
       "      <td>2021-07-20</td>\n",
       "      <td>2021-07-21</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>376</th>\n",
       "      <td>376</td>\n",
       "      <td>2021-07-26</td>\n",
       "      <td>83</td>\n",
       "      <td>2021-07-20</td>\n",
       "      <td>1417393591022936065</td>\n",
       "      <td>ilgiornale</td>\n",
       "      <td>2.897790</td>\n",
       "      <td>77</td>\n",
       "      <td>0.516768</td>\n",
       "      <td>2021-07-26</td>\n",
       "      <td>2021-07-26</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>377</th>\n",
       "      <td>377</td>\n",
       "      <td>2021-07-26</td>\n",
       "      <td>83</td>\n",
       "      <td>2021-07-21</td>\n",
       "      <td>1417876798998732810</td>\n",
       "      <td>LegaSalvini</td>\n",
       "      <td>2.897790</td>\n",
       "      <td>77</td>\n",
       "      <td>0.516768</td>\n",
       "      <td>2021-07-26</td>\n",
       "      <td>2021-07-26</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>378</th>\n",
       "      <td>378</td>\n",
       "      <td>2021-07-28</td>\n",
       "      <td>84</td>\n",
       "      <td>2021-07-21</td>\n",
       "      <td>1417876798998732810</td>\n",
       "      <td>LegaSalvini</td>\n",
       "      <td>0.539660</td>\n",
       "      <td>31</td>\n",
       "      <td>0.446378</td>\n",
       "      <td>2021-07-28</td>\n",
       "      <td>2021-07-28</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>305 rows × 11 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     index spike_window_date  spike_id  tweet_date             tweet_id  \\\n",
       "2        2        2019-08-22         0  2019-08-21  1163965966323109889   \n",
       "3        3        2019-08-23         0  2019-08-21  1163965966323109889   \n",
       "4        4        2019-08-23         0  2019-08-22  1164639655574196224   \n",
       "5        5        2019-08-24         0  2019-08-21  1163965966323109889   \n",
       "6        6        2019-08-24         0  2019-08-22  1164639655574196224   \n",
       "..     ...               ...       ...         ...                  ...   \n",
       "368    368        2021-06-07        77  2021-06-01  1399733841699520517   \n",
       "375    375        2021-07-21        82  2021-07-20  1417393591022936065   \n",
       "376    376        2021-07-26        83  2021-07-20  1417393591022936065   \n",
       "377    377        2021-07-26        83  2021-07-21  1417876798998732810   \n",
       "378    378        2021-07-28        84  2021-07-21  1417876798998732810   \n",
       "\n",
       "     user_username    mIndex  volume  sentiment spike_starting_date  \\\n",
       "2    MilanoFinanza  5.825910     179   0.373657          2019-08-20   \n",
       "3    MilanoFinanza  2.485722      93   0.373657          2019-08-20   \n",
       "4       repubblica  2.485722      93   0.373657          2019-08-20   \n",
       "5    MilanoFinanza  1.006199      71   0.373657          2019-08-20   \n",
       "6       repubblica  1.006199      71   0.373657          2019-08-20   \n",
       "..             ...       ...     ...        ...                 ...   \n",
       "368  ItaliaaTavola  0.829249      69   0.453243          2021-06-07   \n",
       "375     ilgiornale  0.525460      31   0.396534          2021-07-20   \n",
       "376     ilgiornale  2.897790      77   0.516768          2021-07-26   \n",
       "377    LegaSalvini  2.897790      77   0.516768          2021-07-26   \n",
       "378    LegaSalvini  0.539660      31   0.446378          2021-07-28   \n",
       "\n",
       "    spike_ending_date  \n",
       "2          2019-08-25  \n",
       "3          2019-08-25  \n",
       "4          2019-08-25  \n",
       "5          2019-08-25  \n",
       "6          2019-08-25  \n",
       "..                ...  \n",
       "368        2021-06-07  \n",
       "375        2021-07-21  \n",
       "376        2021-07-26  \n",
       "377        2021-07-26  \n",
       "378        2021-07-28  \n",
       "\n",
       "[305 rows x 11 columns]"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "collapsed_merged_spikes_df = merged_spikes_df.reset_index()\n",
    "collapsed_merged_spikes_df = collapse_consecutive_spike_days(collapsed_merged_spikes_df)\n",
    "collapsed_merged_spikes_df.dropna(inplace=True)\n",
    "collapsed_merged_spikes_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\alxau\\anaconda3\\lib\\site-packages\\pandas\\core\\indexing.py:1637: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  self._setitem_single_block(indexer, value, name)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "There are not next lines in the DataFrame\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>index</th>\n",
       "      <th>spike_window_date</th>\n",
       "      <th>spike_id</th>\n",
       "      <th>tweet_date</th>\n",
       "      <th>tweet_id</th>\n",
       "      <th>user_username</th>\n",
       "      <th>mIndex</th>\n",
       "      <th>volume</th>\n",
       "      <th>sentiment</th>\n",
       "      <th>spike_starting_date</th>\n",
       "      <th>spike_ending_date</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>2019-08-21</td>\n",
       "      <td>0</td>\n",
       "      <td>2019-08-21</td>\n",
       "      <td>1163965966323109889</td>\n",
       "      <td>MilanoFinanza</td>\n",
       "      <td>13.554951</td>\n",
       "      <td>378</td>\n",
       "      <td>0.373657</td>\n",
       "      <td>2019-08-20</td>\n",
       "      <td>2019-08-25</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>2019-08-22</td>\n",
       "      <td>0</td>\n",
       "      <td>2019-08-21</td>\n",
       "      <td>1163965966323109889</td>\n",
       "      <td>MilanoFinanza</td>\n",
       "      <td>5.825910</td>\n",
       "      <td>179</td>\n",
       "      <td>0.373657</td>\n",
       "      <td>2019-08-20</td>\n",
       "      <td>2019-08-25</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>2019-08-22</td>\n",
       "      <td>0</td>\n",
       "      <td>2019-08-22</td>\n",
       "      <td>1164639655574196224</td>\n",
       "      <td>repubblica</td>\n",
       "      <td>5.825910</td>\n",
       "      <td>179</td>\n",
       "      <td>0.373657</td>\n",
       "      <td>2019-08-20</td>\n",
       "      <td>2019-08-25</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>2019-08-23</td>\n",
       "      <td>0</td>\n",
       "      <td>2019-08-21</td>\n",
       "      <td>1163965966323109889</td>\n",
       "      <td>MilanoFinanza</td>\n",
       "      <td>2.485722</td>\n",
       "      <td>93</td>\n",
       "      <td>0.373657</td>\n",
       "      <td>2019-08-20</td>\n",
       "      <td>2019-08-25</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>5</td>\n",
       "      <td>2019-08-23</td>\n",
       "      <td>0</td>\n",
       "      <td>2019-08-22</td>\n",
       "      <td>1164639655574196224</td>\n",
       "      <td>repubblica</td>\n",
       "      <td>2.485722</td>\n",
       "      <td>93</td>\n",
       "      <td>0.373657</td>\n",
       "      <td>2019-08-20</td>\n",
       "      <td>2019-08-25</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>484</th>\n",
       "      <td>484</td>\n",
       "      <td>2021-07-21</td>\n",
       "      <td>82</td>\n",
       "      <td>2021-07-20</td>\n",
       "      <td>1417393591022936065</td>\n",
       "      <td>ilgiornale</td>\n",
       "      <td>0.525460</td>\n",
       "      <td>31</td>\n",
       "      <td>0.396534</td>\n",
       "      <td>2021-07-20</td>\n",
       "      <td>2021-07-21</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>485</th>\n",
       "      <td>485</td>\n",
       "      <td>2021-07-21</td>\n",
       "      <td>82</td>\n",
       "      <td>2021-07-21</td>\n",
       "      <td>1417876798998732810</td>\n",
       "      <td>LegaSalvini</td>\n",
       "      <td>0.525460</td>\n",
       "      <td>31</td>\n",
       "      <td>0.396534</td>\n",
       "      <td>2021-07-20</td>\n",
       "      <td>2021-07-21</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>486</th>\n",
       "      <td>486</td>\n",
       "      <td>2021-07-26</td>\n",
       "      <td>83</td>\n",
       "      <td>2021-07-20</td>\n",
       "      <td>1417393591022936065</td>\n",
       "      <td>ilgiornale</td>\n",
       "      <td>2.897790</td>\n",
       "      <td>77</td>\n",
       "      <td>0.516768</td>\n",
       "      <td>2021-07-26</td>\n",
       "      <td>2021-07-26</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>487</th>\n",
       "      <td>487</td>\n",
       "      <td>2021-07-26</td>\n",
       "      <td>83</td>\n",
       "      <td>2021-07-21</td>\n",
       "      <td>1417876798998732810</td>\n",
       "      <td>LegaSalvini</td>\n",
       "      <td>2.897790</td>\n",
       "      <td>77</td>\n",
       "      <td>0.516768</td>\n",
       "      <td>2021-07-26</td>\n",
       "      <td>2021-07-26</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>488</th>\n",
       "      <td>488</td>\n",
       "      <td>2021-07-28</td>\n",
       "      <td>84</td>\n",
       "      <td>2021-07-21</td>\n",
       "      <td>1417876798998732810</td>\n",
       "      <td>LegaSalvini</td>\n",
       "      <td>0.539660</td>\n",
       "      <td>31</td>\n",
       "      <td>0.446378</td>\n",
       "      <td>2021-07-28</td>\n",
       "      <td>2021-07-28</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>445 rows × 11 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     index spike_window_date  spike_id  tweet_date             tweet_id  \\\n",
       "1        1        2019-08-21         0  2019-08-21  1163965966323109889   \n",
       "2        2        2019-08-22         0  2019-08-21  1163965966323109889   \n",
       "3        3        2019-08-22         0  2019-08-22  1164639655574196224   \n",
       "4        4        2019-08-23         0  2019-08-21  1163965966323109889   \n",
       "5        5        2019-08-23         0  2019-08-22  1164639655574196224   \n",
       "..     ...               ...       ...         ...                  ...   \n",
       "484    484        2021-07-21        82  2021-07-20  1417393591022936065   \n",
       "485    485        2021-07-21        82  2021-07-21  1417876798998732810   \n",
       "486    486        2021-07-26        83  2021-07-20  1417393591022936065   \n",
       "487    487        2021-07-26        83  2021-07-21  1417876798998732810   \n",
       "488    488        2021-07-28        84  2021-07-21  1417876798998732810   \n",
       "\n",
       "     user_username     mIndex  volume  sentiment spike_starting_date  \\\n",
       "1    MilanoFinanza  13.554951     378   0.373657          2019-08-20   \n",
       "2    MilanoFinanza   5.825910     179   0.373657          2019-08-20   \n",
       "3       repubblica   5.825910     179   0.373657          2019-08-20   \n",
       "4    MilanoFinanza   2.485722      93   0.373657          2019-08-20   \n",
       "5       repubblica   2.485722      93   0.373657          2019-08-20   \n",
       "..             ...        ...     ...        ...                 ...   \n",
       "484     ilgiornale   0.525460      31   0.396534          2021-07-20   \n",
       "485    LegaSalvini   0.525460      31   0.396534          2021-07-20   \n",
       "486     ilgiornale   2.897790      77   0.516768          2021-07-26   \n",
       "487    LegaSalvini   2.897790      77   0.516768          2021-07-26   \n",
       "488    LegaSalvini   0.539660      31   0.446378          2021-07-28   \n",
       "\n",
       "    spike_ending_date  \n",
       "1          2019-08-25  \n",
       "2          2019-08-25  \n",
       "3          2019-08-25  \n",
       "4          2019-08-25  \n",
       "5          2019-08-25  \n",
       "..                ...  \n",
       "484        2021-07-21  \n",
       "485        2021-07-21  \n",
       "486        2021-07-26  \n",
       "487        2021-07-26  \n",
       "488        2021-07-28  \n",
       "\n",
       "[445 rows x 11 columns]"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "collapsed_merged_spikes_date_included_df = merged_spikes_date_included_df.reset_index()\n",
    "collapsed_merged_spikes_date_included_df = collapse_consecutive_spike_days(collapsed_merged_spikes_date_included_df)\n",
    "collapsed_merged_spikes_date_included_df.dropna(inplace=True)\n",
    "collapsed_merged_spikes_date_included_df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>index</th>\n",
       "      <th>tweet_date</th>\n",
       "      <th>user_username</th>\n",
       "      <th>mIndex</th>\n",
       "      <th>volume</th>\n",
       "      <th>sentiment</th>\n",
       "      <th>spike_starting_date</th>\n",
       "      <th>spike_ending_date</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>spike_id</th>\n",
       "      <th>tweet_id</th>\n",
       "      <th>spike_window_date</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">0</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">1163965966323109889</th>\n",
       "      <th>2019-08-22</th>\n",
       "      <td>2</td>\n",
       "      <td>2019-08-21</td>\n",
       "      <td>MilanoFinanza</td>\n",
       "      <td>5.825910</td>\n",
       "      <td>179</td>\n",
       "      <td>0.373657</td>\n",
       "      <td>2019-08-20</td>\n",
       "      <td>2019-08-25</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-08-23</th>\n",
       "      <td>3</td>\n",
       "      <td>2019-08-21</td>\n",
       "      <td>MilanoFinanza</td>\n",
       "      <td>2.485722</td>\n",
       "      <td>93</td>\n",
       "      <td>0.373657</td>\n",
       "      <td>2019-08-20</td>\n",
       "      <td>2019-08-25</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1164639655574196224</th>\n",
       "      <th>2019-08-23</th>\n",
       "      <td>4</td>\n",
       "      <td>2019-08-22</td>\n",
       "      <td>repubblica</td>\n",
       "      <td>2.485722</td>\n",
       "      <td>93</td>\n",
       "      <td>0.373657</td>\n",
       "      <td>2019-08-20</td>\n",
       "      <td>2019-08-25</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1163965966323109889</th>\n",
       "      <th>2019-08-24</th>\n",
       "      <td>5</td>\n",
       "      <td>2019-08-21</td>\n",
       "      <td>MilanoFinanza</td>\n",
       "      <td>1.006199</td>\n",
       "      <td>71</td>\n",
       "      <td>0.373657</td>\n",
       "      <td>2019-08-20</td>\n",
       "      <td>2019-08-25</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1164639655574196224</th>\n",
       "      <th>2019-08-24</th>\n",
       "      <td>6</td>\n",
       "      <td>2019-08-22</td>\n",
       "      <td>repubblica</td>\n",
       "      <td>1.006199</td>\n",
       "      <td>71</td>\n",
       "      <td>0.373657</td>\n",
       "      <td>2019-08-20</td>\n",
       "      <td>2019-08-25</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>77</th>\n",
       "      <th>1399733841699520517</th>\n",
       "      <th>2021-06-07</th>\n",
       "      <td>368</td>\n",
       "      <td>2021-06-01</td>\n",
       "      <td>ItaliaaTavola</td>\n",
       "      <td>0.829249</td>\n",
       "      <td>69</td>\n",
       "      <td>0.453243</td>\n",
       "      <td>2021-06-07</td>\n",
       "      <td>2021-06-07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>82</th>\n",
       "      <th>1417393591022936065</th>\n",
       "      <th>2021-07-21</th>\n",
       "      <td>375</td>\n",
       "      <td>2021-07-20</td>\n",
       "      <td>ilgiornale</td>\n",
       "      <td>0.525460</td>\n",
       "      <td>31</td>\n",
       "      <td>0.396534</td>\n",
       "      <td>2021-07-20</td>\n",
       "      <td>2021-07-21</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">83</th>\n",
       "      <th>1417393591022936065</th>\n",
       "      <th>2021-07-26</th>\n",
       "      <td>376</td>\n",
       "      <td>2021-07-20</td>\n",
       "      <td>ilgiornale</td>\n",
       "      <td>2.897790</td>\n",
       "      <td>77</td>\n",
       "      <td>0.516768</td>\n",
       "      <td>2021-07-26</td>\n",
       "      <td>2021-07-26</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1417876798998732810</th>\n",
       "      <th>2021-07-26</th>\n",
       "      <td>377</td>\n",
       "      <td>2021-07-21</td>\n",
       "      <td>LegaSalvini</td>\n",
       "      <td>2.897790</td>\n",
       "      <td>77</td>\n",
       "      <td>0.516768</td>\n",
       "      <td>2021-07-26</td>\n",
       "      <td>2021-07-26</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>84</th>\n",
       "      <th>1417876798998732810</th>\n",
       "      <th>2021-07-28</th>\n",
       "      <td>378</td>\n",
       "      <td>2021-07-21</td>\n",
       "      <td>LegaSalvini</td>\n",
       "      <td>0.539660</td>\n",
       "      <td>31</td>\n",
       "      <td>0.446378</td>\n",
       "      <td>2021-07-28</td>\n",
       "      <td>2021-07-28</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>305 rows × 8 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                index  tweet_date  \\\n",
       "spike_id tweet_id            spike_window_date                      \n",
       "0        1163965966323109889 2019-08-22             2  2019-08-21   \n",
       "                             2019-08-23             3  2019-08-21   \n",
       "         1164639655574196224 2019-08-23             4  2019-08-22   \n",
       "         1163965966323109889 2019-08-24             5  2019-08-21   \n",
       "         1164639655574196224 2019-08-24             6  2019-08-22   \n",
       "...                                               ...         ...   \n",
       "77       1399733841699520517 2021-06-07           368  2021-06-01   \n",
       "82       1417393591022936065 2021-07-21           375  2021-07-20   \n",
       "83       1417393591022936065 2021-07-26           376  2021-07-20   \n",
       "         1417876798998732810 2021-07-26           377  2021-07-21   \n",
       "84       1417876798998732810 2021-07-28           378  2021-07-21   \n",
       "\n",
       "                                                user_username    mIndex  \\\n",
       "spike_id tweet_id            spike_window_date                            \n",
       "0        1163965966323109889 2019-08-22         MilanoFinanza  5.825910   \n",
       "                             2019-08-23         MilanoFinanza  2.485722   \n",
       "         1164639655574196224 2019-08-23            repubblica  2.485722   \n",
       "         1163965966323109889 2019-08-24         MilanoFinanza  1.006199   \n",
       "         1164639655574196224 2019-08-24            repubblica  1.006199   \n",
       "...                                                       ...       ...   \n",
       "77       1399733841699520517 2021-06-07         ItaliaaTavola  0.829249   \n",
       "82       1417393591022936065 2021-07-21            ilgiornale  0.525460   \n",
       "83       1417393591022936065 2021-07-26            ilgiornale  2.897790   \n",
       "         1417876798998732810 2021-07-26           LegaSalvini  2.897790   \n",
       "84       1417876798998732810 2021-07-28           LegaSalvini  0.539660   \n",
       "\n",
       "                                                volume  sentiment  \\\n",
       "spike_id tweet_id            spike_window_date                      \n",
       "0        1163965966323109889 2019-08-22            179   0.373657   \n",
       "                             2019-08-23             93   0.373657   \n",
       "         1164639655574196224 2019-08-23             93   0.373657   \n",
       "         1163965966323109889 2019-08-24             71   0.373657   \n",
       "         1164639655574196224 2019-08-24             71   0.373657   \n",
       "...                                                ...        ...   \n",
       "77       1399733841699520517 2021-06-07             69   0.453243   \n",
       "82       1417393591022936065 2021-07-21             31   0.396534   \n",
       "83       1417393591022936065 2021-07-26             77   0.516768   \n",
       "         1417876798998732810 2021-07-26             77   0.516768   \n",
       "84       1417876798998732810 2021-07-28             31   0.446378   \n",
       "\n",
       "                                               spike_starting_date  \\\n",
       "spike_id tweet_id            spike_window_date                       \n",
       "0        1163965966323109889 2019-08-22                 2019-08-20   \n",
       "                             2019-08-23                 2019-08-20   \n",
       "         1164639655574196224 2019-08-23                 2019-08-20   \n",
       "         1163965966323109889 2019-08-24                 2019-08-20   \n",
       "         1164639655574196224 2019-08-24                 2019-08-20   \n",
       "...                                                            ...   \n",
       "77       1399733841699520517 2021-06-07                 2021-06-07   \n",
       "82       1417393591022936065 2021-07-21                 2021-07-20   \n",
       "83       1417393591022936065 2021-07-26                 2021-07-26   \n",
       "         1417876798998732810 2021-07-26                 2021-07-26   \n",
       "84       1417876798998732810 2021-07-28                 2021-07-28   \n",
       "\n",
       "                                               spike_ending_date  \n",
       "spike_id tweet_id            spike_window_date                    \n",
       "0        1163965966323109889 2019-08-22               2019-08-25  \n",
       "                             2019-08-23               2019-08-25  \n",
       "         1164639655574196224 2019-08-23               2019-08-25  \n",
       "         1163965966323109889 2019-08-24               2019-08-25  \n",
       "         1164639655574196224 2019-08-24               2019-08-25  \n",
       "...                                                          ...  \n",
       "77       1399733841699520517 2021-06-07               2021-06-07  \n",
       "82       1417393591022936065 2021-07-21               2021-07-21  \n",
       "83       1417393591022936065 2021-07-26               2021-07-26  \n",
       "         1417876798998732810 2021-07-26               2021-07-26  \n",
       "84       1417876798998732810 2021-07-28               2021-07-28  \n",
       "\n",
       "[305 rows x 8 columns]"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Should tweets be counted multiple times if they are present in the date_delta of multiple days on the same spike? (ATM I'm leaving duplicates on this dataset)\n",
    "collapsed_merged_spikes_df.set_index(['spike_id', 'tweet_id', 'spike_window_date'], inplace=True, verify_integrity=True)\n",
    "collapsed_merged_spikes_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>index</th>\n",
       "      <th>tweet_date</th>\n",
       "      <th>user_username</th>\n",
       "      <th>mIndex</th>\n",
       "      <th>volume</th>\n",
       "      <th>sentiment</th>\n",
       "      <th>spike_starting_date</th>\n",
       "      <th>spike_ending_date</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>spike_id</th>\n",
       "      <th>tweet_id</th>\n",
       "      <th>spike_window_date</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">0</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">1163965966323109889</th>\n",
       "      <th>2019-08-21</th>\n",
       "      <td>1</td>\n",
       "      <td>2019-08-21</td>\n",
       "      <td>MilanoFinanza</td>\n",
       "      <td>13.554951</td>\n",
       "      <td>378</td>\n",
       "      <td>0.373657</td>\n",
       "      <td>2019-08-20</td>\n",
       "      <td>2019-08-25</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-08-22</th>\n",
       "      <td>2</td>\n",
       "      <td>2019-08-21</td>\n",
       "      <td>MilanoFinanza</td>\n",
       "      <td>5.825910</td>\n",
       "      <td>179</td>\n",
       "      <td>0.373657</td>\n",
       "      <td>2019-08-20</td>\n",
       "      <td>2019-08-25</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1164639655574196224</th>\n",
       "      <th>2019-08-22</th>\n",
       "      <td>3</td>\n",
       "      <td>2019-08-22</td>\n",
       "      <td>repubblica</td>\n",
       "      <td>5.825910</td>\n",
       "      <td>179</td>\n",
       "      <td>0.373657</td>\n",
       "      <td>2019-08-20</td>\n",
       "      <td>2019-08-25</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1163965966323109889</th>\n",
       "      <th>2019-08-23</th>\n",
       "      <td>4</td>\n",
       "      <td>2019-08-21</td>\n",
       "      <td>MilanoFinanza</td>\n",
       "      <td>2.485722</td>\n",
       "      <td>93</td>\n",
       "      <td>0.373657</td>\n",
       "      <td>2019-08-20</td>\n",
       "      <td>2019-08-25</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1164639655574196224</th>\n",
       "      <th>2019-08-23</th>\n",
       "      <td>5</td>\n",
       "      <td>2019-08-22</td>\n",
       "      <td>repubblica</td>\n",
       "      <td>2.485722</td>\n",
       "      <td>93</td>\n",
       "      <td>0.373657</td>\n",
       "      <td>2019-08-20</td>\n",
       "      <td>2019-08-25</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">82</th>\n",
       "      <th>1417393591022936065</th>\n",
       "      <th>2021-07-21</th>\n",
       "      <td>484</td>\n",
       "      <td>2021-07-20</td>\n",
       "      <td>ilgiornale</td>\n",
       "      <td>0.525460</td>\n",
       "      <td>31</td>\n",
       "      <td>0.396534</td>\n",
       "      <td>2021-07-20</td>\n",
       "      <td>2021-07-21</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1417876798998732810</th>\n",
       "      <th>2021-07-21</th>\n",
       "      <td>485</td>\n",
       "      <td>2021-07-21</td>\n",
       "      <td>LegaSalvini</td>\n",
       "      <td>0.525460</td>\n",
       "      <td>31</td>\n",
       "      <td>0.396534</td>\n",
       "      <td>2021-07-20</td>\n",
       "      <td>2021-07-21</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">83</th>\n",
       "      <th>1417393591022936065</th>\n",
       "      <th>2021-07-26</th>\n",
       "      <td>486</td>\n",
       "      <td>2021-07-20</td>\n",
       "      <td>ilgiornale</td>\n",
       "      <td>2.897790</td>\n",
       "      <td>77</td>\n",
       "      <td>0.516768</td>\n",
       "      <td>2021-07-26</td>\n",
       "      <td>2021-07-26</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1417876798998732810</th>\n",
       "      <th>2021-07-26</th>\n",
       "      <td>487</td>\n",
       "      <td>2021-07-21</td>\n",
       "      <td>LegaSalvini</td>\n",
       "      <td>2.897790</td>\n",
       "      <td>77</td>\n",
       "      <td>0.516768</td>\n",
       "      <td>2021-07-26</td>\n",
       "      <td>2021-07-26</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>84</th>\n",
       "      <th>1417876798998732810</th>\n",
       "      <th>2021-07-28</th>\n",
       "      <td>488</td>\n",
       "      <td>2021-07-21</td>\n",
       "      <td>LegaSalvini</td>\n",
       "      <td>0.539660</td>\n",
       "      <td>31</td>\n",
       "      <td>0.446378</td>\n",
       "      <td>2021-07-28</td>\n",
       "      <td>2021-07-28</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>445 rows × 8 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                index  tweet_date  \\\n",
       "spike_id tweet_id            spike_window_date                      \n",
       "0        1163965966323109889 2019-08-21             1  2019-08-21   \n",
       "                             2019-08-22             2  2019-08-21   \n",
       "         1164639655574196224 2019-08-22             3  2019-08-22   \n",
       "         1163965966323109889 2019-08-23             4  2019-08-21   \n",
       "         1164639655574196224 2019-08-23             5  2019-08-22   \n",
       "...                                               ...         ...   \n",
       "82       1417393591022936065 2021-07-21           484  2021-07-20   \n",
       "         1417876798998732810 2021-07-21           485  2021-07-21   \n",
       "83       1417393591022936065 2021-07-26           486  2021-07-20   \n",
       "         1417876798998732810 2021-07-26           487  2021-07-21   \n",
       "84       1417876798998732810 2021-07-28           488  2021-07-21   \n",
       "\n",
       "                                                user_username     mIndex  \\\n",
       "spike_id tweet_id            spike_window_date                             \n",
       "0        1163965966323109889 2019-08-21         MilanoFinanza  13.554951   \n",
       "                             2019-08-22         MilanoFinanza   5.825910   \n",
       "         1164639655574196224 2019-08-22            repubblica   5.825910   \n",
       "         1163965966323109889 2019-08-23         MilanoFinanza   2.485722   \n",
       "         1164639655574196224 2019-08-23            repubblica   2.485722   \n",
       "...                                                       ...        ...   \n",
       "82       1417393591022936065 2021-07-21            ilgiornale   0.525460   \n",
       "         1417876798998732810 2021-07-21           LegaSalvini   0.525460   \n",
       "83       1417393591022936065 2021-07-26            ilgiornale   2.897790   \n",
       "         1417876798998732810 2021-07-26           LegaSalvini   2.897790   \n",
       "84       1417876798998732810 2021-07-28           LegaSalvini   0.539660   \n",
       "\n",
       "                                                volume  sentiment  \\\n",
       "spike_id tweet_id            spike_window_date                      \n",
       "0        1163965966323109889 2019-08-21            378   0.373657   \n",
       "                             2019-08-22            179   0.373657   \n",
       "         1164639655574196224 2019-08-22            179   0.373657   \n",
       "         1163965966323109889 2019-08-23             93   0.373657   \n",
       "         1164639655574196224 2019-08-23             93   0.373657   \n",
       "...                                                ...        ...   \n",
       "82       1417393591022936065 2021-07-21             31   0.396534   \n",
       "         1417876798998732810 2021-07-21             31   0.396534   \n",
       "83       1417393591022936065 2021-07-26             77   0.516768   \n",
       "         1417876798998732810 2021-07-26             77   0.516768   \n",
       "84       1417876798998732810 2021-07-28             31   0.446378   \n",
       "\n",
       "                                               spike_starting_date  \\\n",
       "spike_id tweet_id            spike_window_date                       \n",
       "0        1163965966323109889 2019-08-21                 2019-08-20   \n",
       "                             2019-08-22                 2019-08-20   \n",
       "         1164639655574196224 2019-08-22                 2019-08-20   \n",
       "         1163965966323109889 2019-08-23                 2019-08-20   \n",
       "         1164639655574196224 2019-08-23                 2019-08-20   \n",
       "...                                                            ...   \n",
       "82       1417393591022936065 2021-07-21                 2021-07-20   \n",
       "         1417876798998732810 2021-07-21                 2021-07-20   \n",
       "83       1417393591022936065 2021-07-26                 2021-07-26   \n",
       "         1417876798998732810 2021-07-26                 2021-07-26   \n",
       "84       1417876798998732810 2021-07-28                 2021-07-28   \n",
       "\n",
       "                                               spike_ending_date  \n",
       "spike_id tweet_id            spike_window_date                    \n",
       "0        1163965966323109889 2019-08-21               2019-08-25  \n",
       "                             2019-08-22               2019-08-25  \n",
       "         1164639655574196224 2019-08-22               2019-08-25  \n",
       "         1163965966323109889 2019-08-23               2019-08-25  \n",
       "         1164639655574196224 2019-08-23               2019-08-25  \n",
       "...                                                          ...  \n",
       "82       1417393591022936065 2021-07-21               2021-07-21  \n",
       "         1417876798998732810 2021-07-21               2021-07-21  \n",
       "83       1417393591022936065 2021-07-26               2021-07-26  \n",
       "         1417876798998732810 2021-07-26               2021-07-26  \n",
       "84       1417876798998732810 2021-07-28               2021-07-28  \n",
       "\n",
       "[445 rows x 8 columns]"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "collapsed_merged_spikes_date_included_df.set_index(['spike_id', 'tweet_id', 'spike_window_date'], inplace=True, verify_integrity=True)\n",
    "collapsed_merged_spikes_date_included_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "collapsed_merged_spikes_df.to_csv('data/authority_filtered/collapsed_merged_spikes_df.csv')\n",
    "collapsed_merged_spikes_date_included_df.to_csv('data/authority_filtered/collapsed_merged_spikes_date_included_df.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>social</th>\n",
       "      <th>id</th>\n",
       "      <th>topic_id</th>\n",
       "      <th>rule_id</th>\n",
       "      <th>is_comment</th>\n",
       "      <th>text</th>\n",
       "      <th>links</th>\n",
       "      <th>mentions</th>\n",
       "      <th>hashtags</th>\n",
       "      <th>...</th>\n",
       "      <th>place_location_lat</th>\n",
       "      <th>place_location_lon</th>\n",
       "      <th>place_lat</th>\n",
       "      <th>place_lon</th>\n",
       "      <th>user_place_country_code</th>\n",
       "      <th>score_spam</th>\n",
       "      <th>score_performance</th>\n",
       "      <th>user_score_rank</th>\n",
       "      <th>score_rating</th>\n",
       "      <th>date_index</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>56</th>\n",
       "      <td>37400</td>\n",
       "      <td>twitter</td>\n",
       "      <td>1157355441531305984</td>\n",
       "      <td>7ee62d5c-5811-45ce-9cef-c09d143f041a</td>\n",
       "      <td>12559.0</td>\n",
       "      <td>False</td>\n",
       "      <td>Veleno in buste inviate a Lavazza, Ferrero, Il...</td>\n",
       "      <td>https://t.co/jic2LqYWV6</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2019-08-02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>57</th>\n",
       "      <td>37247</td>\n",
       "      <td>twitter</td>\n",
       "      <td>1159017304333672448</td>\n",
       "      <td>7ee62d5c-5811-45ce-9cef-c09d143f041a</td>\n",
       "      <td>12559.0</td>\n",
       "      <td>False</td>\n",
       "      <td>#Kinder Merendero: una lettrice chiede l’orig...</td>\n",
       "      <td>https://t.co/S9eROgogFr, https://t.co/ngu8qQCH3f</td>\n",
       "      <td>NaN</td>\n",
       "      <td>#kinder, #latte, #ferrero, #kindermerendero, #...</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2019-08-07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>59</th>\n",
       "      <td>36920</td>\n",
       "      <td>twitter</td>\n",
       "      <td>1162025296825966592</td>\n",
       "      <td>7ee62d5c-5811-45ce-9cef-c09d143f041a</td>\n",
       "      <td>12559.0</td>\n",
       "      <td>False</td>\n",
       "      <td>Il biografo Ferrero: “Napoleone corre ancora d...</td>\n",
       "      <td>https://t.co/Rn42abkm1Q, https://t.co/aydRu3t8JE</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2019-08-15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>58</th>\n",
       "      <td>36947</td>\n",
       "      <td>twitter</td>\n",
       "      <td>1161912740325273601</td>\n",
       "      <td>7ee62d5c-5811-45ce-9cef-c09d143f041a</td>\n",
       "      <td>12559.0</td>\n",
       "      <td>False</td>\n",
       "      <td>Quelli che guadagnano 4 milioni ogni ora: le 2...</td>\n",
       "      <td>https://t.co/8YtXHE9vVh</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2019-08-15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>60</th>\n",
       "      <td>36835</td>\n",
       "      <td>twitter</td>\n",
       "      <td>1163369496805031937</td>\n",
       "      <td>7ee62d5c-5811-45ce-9cef-c09d143f041a</td>\n",
       "      <td>12559.0</td>\n",
       "      <td>False</td>\n",
       "      <td>Le nocciole turche sono l’oro di Erdogan e Fer...</td>\n",
       "      <td>https://t.co/wB2eZKgXFI</td>\n",
       "      <td>NaN</td>\n",
       "      <td>#fattoquotidiano, #edicola, #19agosto</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2019-08-19</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>278</th>\n",
       "      <td>1013</td>\n",
       "      <td>twitter</td>\n",
       "      <td>1418810115445166080</td>\n",
       "      <td>7ee62d5c-5811-45ce-9cef-c09d143f041a</td>\n",
       "      <td>12559.0</td>\n",
       "      <td>False</td>\n",
       "      <td>Ferrero, la strategia green del big di Alba (è...</td>\n",
       "      <td>https://t.co/r2XvO05KdN, https://t.co/t8k8vmUlQk</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2021-07-24</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>279</th>\n",
       "      <td>841</td>\n",
       "      <td>twitter</td>\n",
       "      <td>1420717382830415880</td>\n",
       "      <td>7ee62d5c-5811-45ce-9cef-c09d143f041a</td>\n",
       "      <td>12559.0</td>\n",
       "      <td>False</td>\n",
       "      <td>Ferrero l'azienda meglio percepita sul territo...</td>\n",
       "      <td>https://t.co/4O4c31bPiq, https://t.co/n8RpxZP5Vu</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2021-07-29</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>280</th>\n",
       "      <td>822</td>\n",
       "      <td>twitter</td>\n",
       "      <td>1421105542945202182</td>\n",
       "      <td>7ee62d5c-5811-45ce-9cef-c09d143f041a</td>\n",
       "      <td>12559.0</td>\n",
       "      <td>False</td>\n",
       "      <td>L’attimo fuggente, online il numero di agosto ...</td>\n",
       "      <td>https://t.co/Q9FPElZrOU</td>\n",
       "      <td>@ferrero_eu, @generali, @prada, @armani, @ciro...</td>\n",
       "      <td>#luigicremonini, #vincenzocremonini, #alberton...</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2021-07-30</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>281</th>\n",
       "      <td>467</td>\n",
       "      <td>twitter</td>\n",
       "      <td>1426075939201421313</td>\n",
       "      <td>7ee62d5c-5811-45ce-9cef-c09d143f041a</td>\n",
       "      <td>12559.0</td>\n",
       "      <td>False</td>\n",
       "      <td>Mediobanca, Gavio arrotonda la quota. I Ferrer...</td>\n",
       "      <td>https://t.co/QPX7deAYBt, https://t.co/XUfJMFKsFE</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2021-08-13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>282</th>\n",
       "      <td>61</td>\n",
       "      <td>twitter</td>\n",
       "      <td>1430765728890859524</td>\n",
       "      <td>7ee62d5c-5811-45ce-9cef-c09d143f041a</td>\n",
       "      <td>12559.0</td>\n",
       "      <td>False</td>\n",
       "      <td>Da anni le aziende alimentari prendono impegni...</td>\n",
       "      <td>https://t.co/79pL8eW8Dm, https://t.co/3MC4xgol6A</td>\n",
       "      <td>@foodwatch_de</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2021-08-26</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>283 rows × 64 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     Unnamed: 0   social                   id  \\\n",
       "56        37400  twitter  1157355441531305984   \n",
       "57        37247  twitter  1159017304333672448   \n",
       "59        36920  twitter  1162025296825966592   \n",
       "58        36947  twitter  1161912740325273601   \n",
       "60        36835  twitter  1163369496805031937   \n",
       "..          ...      ...                  ...   \n",
       "278        1013  twitter  1418810115445166080   \n",
       "279         841  twitter  1420717382830415880   \n",
       "280         822  twitter  1421105542945202182   \n",
       "281         467  twitter  1426075939201421313   \n",
       "282          61  twitter  1430765728890859524   \n",
       "\n",
       "                                 topic_id  rule_id  is_comment  \\\n",
       "56   7ee62d5c-5811-45ce-9cef-c09d143f041a  12559.0       False   \n",
       "57   7ee62d5c-5811-45ce-9cef-c09d143f041a  12559.0       False   \n",
       "59   7ee62d5c-5811-45ce-9cef-c09d143f041a  12559.0       False   \n",
       "58   7ee62d5c-5811-45ce-9cef-c09d143f041a  12559.0       False   \n",
       "60   7ee62d5c-5811-45ce-9cef-c09d143f041a  12559.0       False   \n",
       "..                                    ...      ...         ...   \n",
       "278  7ee62d5c-5811-45ce-9cef-c09d143f041a  12559.0       False   \n",
       "279  7ee62d5c-5811-45ce-9cef-c09d143f041a  12559.0       False   \n",
       "280  7ee62d5c-5811-45ce-9cef-c09d143f041a  12559.0       False   \n",
       "281  7ee62d5c-5811-45ce-9cef-c09d143f041a  12559.0       False   \n",
       "282  7ee62d5c-5811-45ce-9cef-c09d143f041a  12559.0       False   \n",
       "\n",
       "                                                  text  \\\n",
       "56   Veleno in buste inviate a Lavazza, Ferrero, Il...   \n",
       "57    #Kinder Merendero: una lettrice chiede l’orig...   \n",
       "59   Il biografo Ferrero: “Napoleone corre ancora d...   \n",
       "58   Quelli che guadagnano 4 milioni ogni ora: le 2...   \n",
       "60   Le nocciole turche sono l’oro di Erdogan e Fer...   \n",
       "..                                                 ...   \n",
       "278  Ferrero, la strategia green del big di Alba (è...   \n",
       "279  Ferrero l'azienda meglio percepita sul territo...   \n",
       "280  L’attimo fuggente, online il numero di agosto ...   \n",
       "281  Mediobanca, Gavio arrotonda la quota. I Ferrer...   \n",
       "282  Da anni le aziende alimentari prendono impegni...   \n",
       "\n",
       "                                                links  \\\n",
       "56                            https://t.co/jic2LqYWV6   \n",
       "57   https://t.co/S9eROgogFr, https://t.co/ngu8qQCH3f   \n",
       "59   https://t.co/Rn42abkm1Q, https://t.co/aydRu3t8JE   \n",
       "58                            https://t.co/8YtXHE9vVh   \n",
       "60                            https://t.co/wB2eZKgXFI   \n",
       "..                                                ...   \n",
       "278  https://t.co/r2XvO05KdN, https://t.co/t8k8vmUlQk   \n",
       "279  https://t.co/4O4c31bPiq, https://t.co/n8RpxZP5Vu   \n",
       "280                           https://t.co/Q9FPElZrOU   \n",
       "281  https://t.co/QPX7deAYBt, https://t.co/XUfJMFKsFE   \n",
       "282  https://t.co/79pL8eW8Dm, https://t.co/3MC4xgol6A   \n",
       "\n",
       "                                              mentions  \\\n",
       "56                                                 NaN   \n",
       "57                                                 NaN   \n",
       "59                                                 NaN   \n",
       "58                                                 NaN   \n",
       "60                                                 NaN   \n",
       "..                                                 ...   \n",
       "278                                                NaN   \n",
       "279                                                NaN   \n",
       "280  @ferrero_eu, @generali, @prada, @armani, @ciro...   \n",
       "281                                                NaN   \n",
       "282                                      @foodwatch_de   \n",
       "\n",
       "                                              hashtags  ...  \\\n",
       "56                                                 NaN  ...   \n",
       "57   #kinder, #latte, #ferrero, #kindermerendero, #...  ...   \n",
       "59                                                 NaN  ...   \n",
       "58                                                 NaN  ...   \n",
       "60               #fattoquotidiano, #edicola, #19agosto  ...   \n",
       "..                                                 ...  ...   \n",
       "278                                                NaN  ...   \n",
       "279                                                NaN  ...   \n",
       "280  #luigicremonini, #vincenzocremonini, #alberton...  ...   \n",
       "281                                                NaN  ...   \n",
       "282                                                NaN  ...   \n",
       "\n",
       "    place_location_lat place_location_lon place_lat  place_lon  \\\n",
       "56                 NaN                NaN       NaN        NaN   \n",
       "57                 NaN                NaN       NaN        NaN   \n",
       "59                 NaN                NaN       NaN        NaN   \n",
       "58                 NaN                NaN       NaN        NaN   \n",
       "60                 NaN                NaN       NaN        NaN   \n",
       "..                 ...                ...       ...        ...   \n",
       "278                NaN                NaN       NaN        NaN   \n",
       "279                NaN                NaN       NaN        NaN   \n",
       "280                NaN                NaN       NaN        NaN   \n",
       "281                NaN                NaN       NaN        NaN   \n",
       "282                NaN                NaN       NaN        NaN   \n",
       "\n",
       "    user_place_country_code  score_spam  score_performance  user_score_rank  \\\n",
       "56                      NaN         NaN                NaN              NaN   \n",
       "57                      NaN         NaN                NaN              NaN   \n",
       "59                      NaN         NaN                NaN              NaN   \n",
       "58                      NaN         NaN                NaN              NaN   \n",
       "60                      NaN         NaN                NaN              NaN   \n",
       "..                      ...         ...                ...              ...   \n",
       "278                     NaN         NaN                NaN              NaN   \n",
       "279                     NaN         NaN                NaN              NaN   \n",
       "280                     NaN         NaN                NaN              NaN   \n",
       "281                     NaN         NaN                NaN              NaN   \n",
       "282                     NaN         NaN                NaN              NaN   \n",
       "\n",
       "     score_rating  date_index  \n",
       "56            NaN  2019-08-02  \n",
       "57            NaN  2019-08-07  \n",
       "59            NaN  2019-08-15  \n",
       "58            NaN  2019-08-15  \n",
       "60            NaN  2019-08-19  \n",
       "..            ...         ...  \n",
       "278           NaN  2021-07-24  \n",
       "279           NaN  2021-07-29  \n",
       "280           NaN  2021-07-30  \n",
       "281           NaN  2021-08-13  \n",
       "282           NaN  2021-08-26  \n",
       "\n",
       "[283 rows x 64 columns]"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "auth_filtered_df.sort_values('date_index')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "spike_auth_tweets = collapsed_merged_spikes_df.reset_index().convert_dtypes().merge(auth_filtered_df.convert_dtypes(), left_on='tweet_id', right_on='id')\n",
    "spike_auth_tweets_date_included = collapsed_merged_spikes_date_included_df.reset_index().convert_dtypes().merge(auth_filtered_df.convert_dtypes(), left_on='tweet_id', right_on='id')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>spike_id</th>\n",
       "      <th>tweet_id</th>\n",
       "      <th>spike_window_date</th>\n",
       "      <th>index</th>\n",
       "      <th>tweet_date</th>\n",
       "      <th>user_username_x</th>\n",
       "      <th>mIndex</th>\n",
       "      <th>volume</th>\n",
       "      <th>sentiment</th>\n",
       "      <th>spike_starting_date</th>\n",
       "      <th>...</th>\n",
       "      <th>place_location_lat</th>\n",
       "      <th>place_location_lon</th>\n",
       "      <th>place_lat</th>\n",
       "      <th>place_lon</th>\n",
       "      <th>user_place_country_code</th>\n",
       "      <th>score_spam</th>\n",
       "      <th>score_performance</th>\n",
       "      <th>user_score_rank</th>\n",
       "      <th>score_rating</th>\n",
       "      <th>date_index</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>1163965966323109889</td>\n",
       "      <td>2019-08-22</td>\n",
       "      <td>2</td>\n",
       "      <td>2019-08-21</td>\n",
       "      <td>MilanoFinanza</td>\n",
       "      <td>5.82591</td>\n",
       "      <td>179</td>\n",
       "      <td>0.373657</td>\n",
       "      <td>2019-08-20</td>\n",
       "      <td>...</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>2019-08-21</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>1163965966323109889</td>\n",
       "      <td>2019-08-23</td>\n",
       "      <td>3</td>\n",
       "      <td>2019-08-21</td>\n",
       "      <td>MilanoFinanza</td>\n",
       "      <td>2.485722</td>\n",
       "      <td>93</td>\n",
       "      <td>0.373657</td>\n",
       "      <td>2019-08-20</td>\n",
       "      <td>...</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>2019-08-21</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>1163965966323109889</td>\n",
       "      <td>2019-08-24</td>\n",
       "      <td>5</td>\n",
       "      <td>2019-08-21</td>\n",
       "      <td>MilanoFinanza</td>\n",
       "      <td>1.006199</td>\n",
       "      <td>71</td>\n",
       "      <td>0.373657</td>\n",
       "      <td>2019-08-20</td>\n",
       "      <td>...</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>2019-08-21</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>1163965966323109889</td>\n",
       "      <td>2019-08-25</td>\n",
       "      <td>8</td>\n",
       "      <td>2019-08-21</td>\n",
       "      <td>MilanoFinanza</td>\n",
       "      <td>0.657033</td>\n",
       "      <td>76</td>\n",
       "      <td>0.373657</td>\n",
       "      <td>2019-08-20</td>\n",
       "      <td>...</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>2019-08-21</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>1164639655574196224</td>\n",
       "      <td>2019-08-23</td>\n",
       "      <td>4</td>\n",
       "      <td>2019-08-22</td>\n",
       "      <td>repubblica</td>\n",
       "      <td>2.485722</td>\n",
       "      <td>93</td>\n",
       "      <td>0.373657</td>\n",
       "      <td>2019-08-20</td>\n",
       "      <td>...</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>2019-08-22</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>300</th>\n",
       "      <td>77</td>\n",
       "      <td>1399733841699520517</td>\n",
       "      <td>2021-06-07</td>\n",
       "      <td>368</td>\n",
       "      <td>2021-06-01</td>\n",
       "      <td>ItaliaaTavola</td>\n",
       "      <td>0.829249</td>\n",
       "      <td>69</td>\n",
       "      <td>0.453243</td>\n",
       "      <td>2021-06-07</td>\n",
       "      <td>...</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>2021-06-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>301</th>\n",
       "      <td>82</td>\n",
       "      <td>1417393591022936065</td>\n",
       "      <td>2021-07-21</td>\n",
       "      <td>375</td>\n",
       "      <td>2021-07-20</td>\n",
       "      <td>ilgiornale</td>\n",
       "      <td>0.52546</td>\n",
       "      <td>31</td>\n",
       "      <td>0.396534</td>\n",
       "      <td>2021-07-20</td>\n",
       "      <td>...</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>2021-07-20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>302</th>\n",
       "      <td>83</td>\n",
       "      <td>1417393591022936065</td>\n",
       "      <td>2021-07-26</td>\n",
       "      <td>376</td>\n",
       "      <td>2021-07-20</td>\n",
       "      <td>ilgiornale</td>\n",
       "      <td>2.89779</td>\n",
       "      <td>77</td>\n",
       "      <td>0.516768</td>\n",
       "      <td>2021-07-26</td>\n",
       "      <td>...</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>2021-07-20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>303</th>\n",
       "      <td>83</td>\n",
       "      <td>1417876798998732810</td>\n",
       "      <td>2021-07-26</td>\n",
       "      <td>377</td>\n",
       "      <td>2021-07-21</td>\n",
       "      <td>LegaSalvini</td>\n",
       "      <td>2.89779</td>\n",
       "      <td>77</td>\n",
       "      <td>0.516768</td>\n",
       "      <td>2021-07-26</td>\n",
       "      <td>...</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>2021-07-21</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>304</th>\n",
       "      <td>84</td>\n",
       "      <td>1417876798998732810</td>\n",
       "      <td>2021-07-28</td>\n",
       "      <td>378</td>\n",
       "      <td>2021-07-21</td>\n",
       "      <td>LegaSalvini</td>\n",
       "      <td>0.53966</td>\n",
       "      <td>31</td>\n",
       "      <td>0.446378</td>\n",
       "      <td>2021-07-28</td>\n",
       "      <td>...</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>2021-07-21</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>305 rows × 75 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     spike_id             tweet_id spike_window_date  index  tweet_date  \\\n",
       "0           0  1163965966323109889        2019-08-22      2  2019-08-21   \n",
       "1           0  1163965966323109889        2019-08-23      3  2019-08-21   \n",
       "2           0  1163965966323109889        2019-08-24      5  2019-08-21   \n",
       "3           0  1163965966323109889        2019-08-25      8  2019-08-21   \n",
       "4           0  1164639655574196224        2019-08-23      4  2019-08-22   \n",
       "..        ...                  ...               ...    ...         ...   \n",
       "300        77  1399733841699520517        2021-06-07    368  2021-06-01   \n",
       "301        82  1417393591022936065        2021-07-21    375  2021-07-20   \n",
       "302        83  1417393591022936065        2021-07-26    376  2021-07-20   \n",
       "303        83  1417876798998732810        2021-07-26    377  2021-07-21   \n",
       "304        84  1417876798998732810        2021-07-28    378  2021-07-21   \n",
       "\n",
       "    user_username_x    mIndex  volume  sentiment spike_starting_date  ...  \\\n",
       "0     MilanoFinanza   5.82591     179   0.373657          2019-08-20  ...   \n",
       "1     MilanoFinanza  2.485722      93   0.373657          2019-08-20  ...   \n",
       "2     MilanoFinanza  1.006199      71   0.373657          2019-08-20  ...   \n",
       "3     MilanoFinanza  0.657033      76   0.373657          2019-08-20  ...   \n",
       "4        repubblica  2.485722      93   0.373657          2019-08-20  ...   \n",
       "..              ...       ...     ...        ...                 ...  ...   \n",
       "300   ItaliaaTavola  0.829249      69   0.453243          2021-06-07  ...   \n",
       "301      ilgiornale   0.52546      31   0.396534          2021-07-20  ...   \n",
       "302      ilgiornale   2.89779      77   0.516768          2021-07-26  ...   \n",
       "303     LegaSalvini   2.89779      77   0.516768          2021-07-26  ...   \n",
       "304     LegaSalvini   0.53966      31   0.446378          2021-07-28  ...   \n",
       "\n",
       "    place_location_lat  place_location_lon place_lat  place_lon  \\\n",
       "0                 <NA>                <NA>      <NA>       <NA>   \n",
       "1                 <NA>                <NA>      <NA>       <NA>   \n",
       "2                 <NA>                <NA>      <NA>       <NA>   \n",
       "3                 <NA>                <NA>      <NA>       <NA>   \n",
       "4                 <NA>                <NA>      <NA>       <NA>   \n",
       "..                 ...                 ...       ...        ...   \n",
       "300               <NA>                <NA>      <NA>       <NA>   \n",
       "301               <NA>                <NA>      <NA>       <NA>   \n",
       "302               <NA>                <NA>      <NA>       <NA>   \n",
       "303               <NA>                <NA>      <NA>       <NA>   \n",
       "304               <NA>                <NA>      <NA>       <NA>   \n",
       "\n",
       "    user_place_country_code  score_spam  score_performance user_score_rank  \\\n",
       "0                      <NA>        <NA>               <NA>            <NA>   \n",
       "1                      <NA>        <NA>               <NA>            <NA>   \n",
       "2                      <NA>        <NA>               <NA>            <NA>   \n",
       "3                      <NA>        <NA>               <NA>            <NA>   \n",
       "4                      <NA>        <NA>               <NA>            <NA>   \n",
       "..                      ...         ...                ...             ...   \n",
       "300                    <NA>        <NA>               <NA>            <NA>   \n",
       "301                    <NA>        <NA>               <NA>            <NA>   \n",
       "302                    <NA>        <NA>               <NA>            <NA>   \n",
       "303                    <NA>        <NA>               <NA>            <NA>   \n",
       "304                    <NA>        <NA>               <NA>            <NA>   \n",
       "\n",
       "    score_rating  date_index  \n",
       "0           <NA>  2019-08-21  \n",
       "1           <NA>  2019-08-21  \n",
       "2           <NA>  2019-08-21  \n",
       "3           <NA>  2019-08-21  \n",
       "4           <NA>  2019-08-22  \n",
       "..           ...         ...  \n",
       "300         <NA>  2021-06-01  \n",
       "301         <NA>  2021-07-20  \n",
       "302         <NA>  2021-07-20  \n",
       "303         <NA>  2021-07-21  \n",
       "304         <NA>  2021-07-21  \n",
       "\n",
       "[305 rows x 75 columns]"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "spike_auth_tweets\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "spike_auth_tweets.drop_duplicates()\n",
    "spike_auth_tweets.set_index(['spike_id', 'tweet_id'], inplace=True)\n",
    "\n",
    "spike_auth_tweets_date_included.drop_duplicates()\n",
    "spike_auth_tweets_date_included.set_index(['spike_id', 'tweet_id'], inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>spike_starting_date</th>\n",
       "      <th>spike_window_date</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>spike_id</th>\n",
       "      <th>tweet_id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">0</th>\n",
       "      <th>1163965966323109889</th>\n",
       "      <td>2019-08-20</td>\n",
       "      <td>2019-08-22</td>\n",
       "      <td>Ferrero, per la holding nuove munizioni da 134...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1163965966323109889</th>\n",
       "      <td>2019-08-20</td>\n",
       "      <td>2019-08-23</td>\n",
       "      <td>Ferrero, per la holding nuove munizioni da 134...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1163965966323109889</th>\n",
       "      <td>2019-08-20</td>\n",
       "      <td>2019-08-24</td>\n",
       "      <td>Ferrero, per la holding nuove munizioni da 134...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1163965966323109889</th>\n",
       "      <td>2019-08-20</td>\n",
       "      <td>2019-08-25</td>\n",
       "      <td>Ferrero, per la holding nuove munizioni da 134...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1164639655574196224</th>\n",
       "      <td>2019-08-20</td>\n",
       "      <td>2019-08-23</td>\n",
       "      <td>Il 5S Barillari attacca la Nutella, Twitter no...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>77</th>\n",
       "      <th>1399733841699520517</th>\n",
       "      <td>2021-06-07</td>\n",
       "      <td>2021-06-07</td>\n",
       "      <td>Il Gruppo piemontese con la holding Cth ha fat...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>82</th>\n",
       "      <th>1417393591022936065</th>\n",
       "      <td>2021-07-20</td>\n",
       "      <td>2021-07-21</td>\n",
       "      <td>Allievo di Lombroso, è stato tra i padri della...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">83</th>\n",
       "      <th>1417393591022936065</th>\n",
       "      <td>2021-07-26</td>\n",
       "      <td>2021-07-26</td>\n",
       "      <td>Allievo di Lombroso, è stato tra i padri della...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1417876798998732810</th>\n",
       "      <td>2021-07-26</td>\n",
       "      <td>2021-07-26</td>\n",
       "      <td>Roberta #Ferrero: Con il dl Sostegni bis andia...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>84</th>\n",
       "      <th>1417876798998732810</th>\n",
       "      <td>2021-07-28</td>\n",
       "      <td>2021-07-28</td>\n",
       "      <td>Roberta #Ferrero: Con il dl Sostegni bis andia...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>305 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                             spike_starting_date spike_window_date  \\\n",
       "spike_id tweet_id                                                    \n",
       "0        1163965966323109889          2019-08-20        2019-08-22   \n",
       "         1163965966323109889          2019-08-20        2019-08-23   \n",
       "         1163965966323109889          2019-08-20        2019-08-24   \n",
       "         1163965966323109889          2019-08-20        2019-08-25   \n",
       "         1164639655574196224          2019-08-20        2019-08-23   \n",
       "...                                          ...               ...   \n",
       "77       1399733841699520517          2021-06-07        2021-06-07   \n",
       "82       1417393591022936065          2021-07-20        2021-07-21   \n",
       "83       1417393591022936065          2021-07-26        2021-07-26   \n",
       "         1417876798998732810          2021-07-26        2021-07-26   \n",
       "84       1417876798998732810          2021-07-28        2021-07-28   \n",
       "\n",
       "                                                                           text  \n",
       "spike_id tweet_id                                                                \n",
       "0        1163965966323109889  Ferrero, per la holding nuove munizioni da 134...  \n",
       "         1163965966323109889  Ferrero, per la holding nuove munizioni da 134...  \n",
       "         1163965966323109889  Ferrero, per la holding nuove munizioni da 134...  \n",
       "         1163965966323109889  Ferrero, per la holding nuove munizioni da 134...  \n",
       "         1164639655574196224  Il 5S Barillari attacca la Nutella, Twitter no...  \n",
       "...                                                                         ...  \n",
       "77       1399733841699520517  Il Gruppo piemontese con la holding Cth ha fat...  \n",
       "82       1417393591022936065  Allievo di Lombroso, è stato tra i padri della...  \n",
       "83       1417393591022936065  Allievo di Lombroso, è stato tra i padri della...  \n",
       "         1417876798998732810  Roberta #Ferrero: Con il dl Sostegni bis andia...  \n",
       "84       1417876798998732810  Roberta #Ferrero: Con il dl Sostegni bis andia...  \n",
       "\n",
       "[305 rows x 3 columns]"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "spike_auth_tweets[['spike_starting_date', 'spike_window_date', 'text']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>spike_starting_date</th>\n",
       "      <th>spike_window_date</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>spike_id</th>\n",
       "      <th>tweet_id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">0</th>\n",
       "      <th>1163965966323109889</th>\n",
       "      <td>2019-08-20</td>\n",
       "      <td>2019-08-21</td>\n",
       "      <td>Ferrero, per la holding nuove munizioni da 134...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1163965966323109889</th>\n",
       "      <td>2019-08-20</td>\n",
       "      <td>2019-08-22</td>\n",
       "      <td>Ferrero, per la holding nuove munizioni da 134...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1163965966323109889</th>\n",
       "      <td>2019-08-20</td>\n",
       "      <td>2019-08-23</td>\n",
       "      <td>Ferrero, per la holding nuove munizioni da 134...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1163965966323109889</th>\n",
       "      <td>2019-08-20</td>\n",
       "      <td>2019-08-24</td>\n",
       "      <td>Ferrero, per la holding nuove munizioni da 134...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1163965966323109889</th>\n",
       "      <td>2019-08-20</td>\n",
       "      <td>2019-08-25</td>\n",
       "      <td>Ferrero, per la holding nuove munizioni da 134...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>82</th>\n",
       "      <th>1417393591022936065</th>\n",
       "      <td>2021-07-20</td>\n",
       "      <td>2021-07-21</td>\n",
       "      <td>Allievo di Lombroso, è stato tra i padri della...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>83</th>\n",
       "      <th>1417393591022936065</th>\n",
       "      <td>2021-07-26</td>\n",
       "      <td>2021-07-26</td>\n",
       "      <td>Allievo di Lombroso, è stato tra i padri della...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>82</th>\n",
       "      <th>1417876798998732810</th>\n",
       "      <td>2021-07-20</td>\n",
       "      <td>2021-07-21</td>\n",
       "      <td>Roberta #Ferrero: Con il dl Sostegni bis andia...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>83</th>\n",
       "      <th>1417876798998732810</th>\n",
       "      <td>2021-07-26</td>\n",
       "      <td>2021-07-26</td>\n",
       "      <td>Roberta #Ferrero: Con il dl Sostegni bis andia...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>84</th>\n",
       "      <th>1417876798998732810</th>\n",
       "      <td>2021-07-28</td>\n",
       "      <td>2021-07-28</td>\n",
       "      <td>Roberta #Ferrero: Con il dl Sostegni bis andia...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>445 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                             spike_starting_date spike_window_date  \\\n",
       "spike_id tweet_id                                                    \n",
       "0        1163965966323109889          2019-08-20        2019-08-21   \n",
       "         1163965966323109889          2019-08-20        2019-08-22   \n",
       "         1163965966323109889          2019-08-20        2019-08-23   \n",
       "         1163965966323109889          2019-08-20        2019-08-24   \n",
       "         1163965966323109889          2019-08-20        2019-08-25   \n",
       "...                                          ...               ...   \n",
       "82       1417393591022936065          2021-07-20        2021-07-21   \n",
       "83       1417393591022936065          2021-07-26        2021-07-26   \n",
       "82       1417876798998732810          2021-07-20        2021-07-21   \n",
       "83       1417876798998732810          2021-07-26        2021-07-26   \n",
       "84       1417876798998732810          2021-07-28        2021-07-28   \n",
       "\n",
       "                                                                           text  \n",
       "spike_id tweet_id                                                                \n",
       "0        1163965966323109889  Ferrero, per la holding nuove munizioni da 134...  \n",
       "         1163965966323109889  Ferrero, per la holding nuove munizioni da 134...  \n",
       "         1163965966323109889  Ferrero, per la holding nuove munizioni da 134...  \n",
       "         1163965966323109889  Ferrero, per la holding nuove munizioni da 134...  \n",
       "         1163965966323109889  Ferrero, per la holding nuove munizioni da 134...  \n",
       "...                                                                         ...  \n",
       "82       1417393591022936065  Allievo di Lombroso, è stato tra i padri della...  \n",
       "83       1417393591022936065  Allievo di Lombroso, è stato tra i padri della...  \n",
       "82       1417876798998732810  Roberta #Ferrero: Con il dl Sostegni bis andia...  \n",
       "83       1417876798998732810  Roberta #Ferrero: Con il dl Sostegni bis andia...  \n",
       "84       1417876798998732810  Roberta #Ferrero: Con il dl Sostegni bis andia...  \n",
       "\n",
       "[445 rows x 3 columns]"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "spike_auth_tweets_date_included[['spike_starting_date', 'spike_window_date', 'text']]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Index:\n",
    "spike_id: the id of the spike with authority contribution. Needed to match data with the dataframe containing all the spikes.\n",
    "tweet_id: the id of the authority tweet. Needed to match it with the complete tweet dataframe.\n",
    "\n",
    "Values:\n",
    "spike_window_date: the spike day to which the tweet gets assigned the first time it gets paired with the spike.\n",
    "spike_starting_date: the starting date associated to the spike_id. \n",
    "spike_ending_date: the ending date associated to the spike_id. \n",
    "tweet_date: the date when the tweet has been published.\n",
    "text: the tweet textual content."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "spike_auth_tweets[['spike_window_date', 'spike_starting_date', 'spike_ending_date', 'tweet_date', 'text']].drop_duplicates().sort_index().to_csv('data/authority_filtered/spike_auth_tweets.csv')\n",
    "spike_auth_tweets_date_included[['spike_window_date', 'spike_starting_date', 'spike_ending_date', 'tweet_date', 'text']].drop_duplicates().sort_index().to_csv('data/authority_filtered/spike_auth_tweets_date_included.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>spike_starting_date</th>\n",
       "      <th>spike_ending_date</th>\n",
       "      <th>tweet_date</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>spike_id</th>\n",
       "      <th>tweet_id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"3\" valign=\"top\">0</th>\n",
       "      <th>1163965966323109889</th>\n",
       "      <td>2019-08-20</td>\n",
       "      <td>2019-08-25</td>\n",
       "      <td>2019-08-21</td>\n",
       "      <td>Ferrero, per la holding nuove munizioni da 134...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1164639655574196224</th>\n",
       "      <td>2019-08-20</td>\n",
       "      <td>2019-08-25</td>\n",
       "      <td>2019-08-22</td>\n",
       "      <td>Il 5S Barillari attacca la Nutella, Twitter no...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1164947272284672001</th>\n",
       "      <td>2019-08-20</td>\n",
       "      <td>2019-08-25</td>\n",
       "      <td>2019-08-23</td>\n",
       "      <td>Sampdoria, dopo un anno di tira e molla Ferrer...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <th>1174264995162599426</th>\n",
       "      <td>2019-09-18</td>\n",
       "      <td>2019-09-20</td>\n",
       "      <td>2019-09-18</td>\n",
       "      <td>Ferrero, premio di 2000 euro ai dipendenti nel...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <th>1174264995162599426</th>\n",
       "      <td>2019-09-22</td>\n",
       "      <td>2019-09-25</td>\n",
       "      <td>2019-09-18</td>\n",
       "      <td>Ferrero, premio di 2000 euro ai dipendenti nel...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>77</th>\n",
       "      <th>1399733841699520517</th>\n",
       "      <td>2021-06-07</td>\n",
       "      <td>2021-06-07</td>\n",
       "      <td>2021-06-01</td>\n",
       "      <td>Il Gruppo piemontese con la holding Cth ha fat...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>82</th>\n",
       "      <th>1417393591022936065</th>\n",
       "      <td>2021-07-20</td>\n",
       "      <td>2021-07-21</td>\n",
       "      <td>2021-07-20</td>\n",
       "      <td>Allievo di Lombroso, è stato tra i padri della...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">83</th>\n",
       "      <th>1417393591022936065</th>\n",
       "      <td>2021-07-26</td>\n",
       "      <td>2021-07-26</td>\n",
       "      <td>2021-07-20</td>\n",
       "      <td>Allievo di Lombroso, è stato tra i padri della...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1417876798998732810</th>\n",
       "      <td>2021-07-26</td>\n",
       "      <td>2021-07-26</td>\n",
       "      <td>2021-07-21</td>\n",
       "      <td>Roberta #Ferrero: Con il dl Sostegni bis andia...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>84</th>\n",
       "      <th>1417876798998732810</th>\n",
       "      <td>2021-07-28</td>\n",
       "      <td>2021-07-28</td>\n",
       "      <td>2021-07-21</td>\n",
       "      <td>Roberta #Ferrero: Con il dl Sostegni bis andia...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>176 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                             spike_starting_date spike_ending_date  \\\n",
       "spike_id tweet_id                                                    \n",
       "0        1163965966323109889          2019-08-20        2019-08-25   \n",
       "         1164639655574196224          2019-08-20        2019-08-25   \n",
       "         1164947272284672001          2019-08-20        2019-08-25   \n",
       "2        1174264995162599426          2019-09-18        2019-09-20   \n",
       "3        1174264995162599426          2019-09-22        2019-09-25   \n",
       "...                                          ...               ...   \n",
       "77       1399733841699520517          2021-06-07        2021-06-07   \n",
       "82       1417393591022936065          2021-07-20        2021-07-21   \n",
       "83       1417393591022936065          2021-07-26        2021-07-26   \n",
       "         1417876798998732810          2021-07-26        2021-07-26   \n",
       "84       1417876798998732810          2021-07-28        2021-07-28   \n",
       "\n",
       "                              tweet_date  \\\n",
       "spike_id tweet_id                          \n",
       "0        1163965966323109889  2019-08-21   \n",
       "         1164639655574196224  2019-08-22   \n",
       "         1164947272284672001  2019-08-23   \n",
       "2        1174264995162599426  2019-09-18   \n",
       "3        1174264995162599426  2019-09-18   \n",
       "...                                  ...   \n",
       "77       1399733841699520517  2021-06-01   \n",
       "82       1417393591022936065  2021-07-20   \n",
       "83       1417393591022936065  2021-07-20   \n",
       "         1417876798998732810  2021-07-21   \n",
       "84       1417876798998732810  2021-07-21   \n",
       "\n",
       "                                                                           text  \n",
       "spike_id tweet_id                                                                \n",
       "0        1163965966323109889  Ferrero, per la holding nuove munizioni da 134...  \n",
       "         1164639655574196224  Il 5S Barillari attacca la Nutella, Twitter no...  \n",
       "         1164947272284672001  Sampdoria, dopo un anno di tira e molla Ferrer...  \n",
       "2        1174264995162599426  Ferrero, premio di 2000 euro ai dipendenti nel...  \n",
       "3        1174264995162599426  Ferrero, premio di 2000 euro ai dipendenti nel...  \n",
       "...                                                                         ...  \n",
       "77       1399733841699520517  Il Gruppo piemontese con la holding Cth ha fat...  \n",
       "82       1417393591022936065  Allievo di Lombroso, è stato tra i padri della...  \n",
       "83       1417393591022936065  Allievo di Lombroso, è stato tra i padri della...  \n",
       "         1417876798998732810  Roberta #Ferrero: Con il dl Sostegni bis andia...  \n",
       "84       1417876798998732810  Roberta #Ferrero: Con il dl Sostegni bis andia...  \n",
       "\n",
       "[176 rows x 4 columns]"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_tmp = spike_auth_tweets[['spike_starting_date', 'spike_ending_date','tweet_date', 'text']].drop_duplicates()\n",
    "df_tmp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_tweets_in_date_range(df, \n",
    "                             date_series,\n",
    "                             range_type, \n",
    "                             date_range=7, \n",
    "                             range_start=7,\n",
    "                             range_end=7,\n",
    "                             include_date=False):\n",
    "    \n",
    "    range_types = ['until', 'beyond', 'in_range']\n",
    "    \n",
    "    if range_type not in range_types:\n",
    "        raise Exception('You must specify a date_range_type. The accepted values are [until, beyond, in_range]')\n",
    "    \n",
    "    output = []\n",
    "\n",
    "    if(range_type==range_types[0]):\n",
    "        for i, row in date_series.iterrows():\n",
    "            temp_df = filter_by_previous_dates(df, row['spike_starting_date'], date_range, 'tweet_date', include_date)\n",
    "            output.append(temp_df)\n",
    "    elif(range_type==range_types[1]):\n",
    "        for i, row in date_series.iterrows():\n",
    "            temp_df = filter_by_following_dates(df, row['spike_starting_date'], date_range, 'tweet_date', include_date)\n",
    "            output.append(temp_df)\n",
    "    elif(range_type==range_types[2]):\n",
    "        for i, row in date_series.iterrows():\n",
    "            start_date = row['spike_starting_date'] - timedelta(days=range_start)\n",
    "            end_date = row['spike_starting_date'] + timedelta(days=range_end)     \n",
    "            start_date = row['spike_starting_date'] - timedelta(days=range_start)\n",
    "            end_date = row['spike_starting_date'] + timedelta(days=range_end)\n",
    "            temp_df = filter_by_date_range(df, start_date, end_date, 'tweet_date')\n",
    "            output.append(temp_df)\n",
    "    return pd.concat(output)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "spikes_auth_tweets_before = get_tweets_in_date_range(df_tmp, df_tmp[['spike_starting_date']].drop_duplicates(),'until', 7).sort_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>spike_starting_date</th>\n",
       "      <th>spike_ending_date</th>\n",
       "      <th>tweet_date</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>spike_id</th>\n",
       "      <th>tweet_id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">2</th>\n",
       "      <th>1174264995162599426</th>\n",
       "      <td>2019-09-18</td>\n",
       "      <td>2019-09-20</td>\n",
       "      <td>2019-09-18</td>\n",
       "      <td>Ferrero, premio di 2000 euro ai dipendenti nel...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1174293258714808321</th>\n",
       "      <td>2019-09-18</td>\n",
       "      <td>2019-09-20</td>\n",
       "      <td>2019-09-18</td>\n",
       "      <td>Ferrero, via libera ai premi legati agli obiet...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1174293756381519872</th>\n",
       "      <td>2019-09-18</td>\n",
       "      <td>2019-09-20</td>\n",
       "      <td>2019-09-18</td>\n",
       "      <td>Ferrero, premio a tutti i dipendenti. In busta...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1174318674058121218</th>\n",
       "      <td>2019-09-18</td>\n",
       "      <td>2019-09-20</td>\n",
       "      <td>2019-09-18</td>\n",
       "      <td>Ferrero, 2mila euro come premio annuale a tutt...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1174347893072809984</th>\n",
       "      <td>2019-09-18</td>\n",
       "      <td>2019-09-20</td>\n",
       "      <td>2019-09-18</td>\n",
       "      <td>Centrati gli obiettivi alla Ferrero: oltre 2 m...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"3\" valign=\"top\">83</th>\n",
       "      <th>1417393591022936065</th>\n",
       "      <td>2021-07-26</td>\n",
       "      <td>2021-07-26</td>\n",
       "      <td>2021-07-20</td>\n",
       "      <td>Allievo di Lombroso, è stato tra i padri della...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1417876798998732810</th>\n",
       "      <td>2021-07-26</td>\n",
       "      <td>2021-07-26</td>\n",
       "      <td>2021-07-21</td>\n",
       "      <td>Roberta #Ferrero: Con il dl Sostegni bis andia...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1417876798998732810</th>\n",
       "      <td>2021-07-26</td>\n",
       "      <td>2021-07-26</td>\n",
       "      <td>2021-07-21</td>\n",
       "      <td>Roberta #Ferrero: Con il dl Sostegni bis andia...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">84</th>\n",
       "      <th>1417876798998732810</th>\n",
       "      <td>2021-07-28</td>\n",
       "      <td>2021-07-28</td>\n",
       "      <td>2021-07-21</td>\n",
       "      <td>Roberta #Ferrero: Con il dl Sostegni bis andia...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1417876798998732810</th>\n",
       "      <td>2021-07-28</td>\n",
       "      <td>2021-07-28</td>\n",
       "      <td>2021-07-21</td>\n",
       "      <td>Roberta #Ferrero: Con il dl Sostegni bis andia...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>159 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                             spike_starting_date spike_ending_date  \\\n",
       "spike_id tweet_id                                                    \n",
       "2        1174264995162599426          2019-09-18        2019-09-20   \n",
       "         1174293258714808321          2019-09-18        2019-09-20   \n",
       "         1174293756381519872          2019-09-18        2019-09-20   \n",
       "         1174318674058121218          2019-09-18        2019-09-20   \n",
       "         1174347893072809984          2019-09-18        2019-09-20   \n",
       "...                                          ...               ...   \n",
       "83       1417393591022936065          2021-07-26        2021-07-26   \n",
       "         1417876798998732810          2021-07-26        2021-07-26   \n",
       "         1417876798998732810          2021-07-26        2021-07-26   \n",
       "84       1417876798998732810          2021-07-28        2021-07-28   \n",
       "         1417876798998732810          2021-07-28        2021-07-28   \n",
       "\n",
       "                              tweet_date  \\\n",
       "spike_id tweet_id                          \n",
       "2        1174264995162599426  2019-09-18   \n",
       "         1174293258714808321  2019-09-18   \n",
       "         1174293756381519872  2019-09-18   \n",
       "         1174318674058121218  2019-09-18   \n",
       "         1174347893072809984  2019-09-18   \n",
       "...                                  ...   \n",
       "83       1417393591022936065  2021-07-20   \n",
       "         1417876798998732810  2021-07-21   \n",
       "         1417876798998732810  2021-07-21   \n",
       "84       1417876798998732810  2021-07-21   \n",
       "         1417876798998732810  2021-07-21   \n",
       "\n",
       "                                                                           text  \n",
       "spike_id tweet_id                                                                \n",
       "2        1174264995162599426  Ferrero, premio di 2000 euro ai dipendenti nel...  \n",
       "         1174293258714808321  Ferrero, via libera ai premi legati agli obiet...  \n",
       "         1174293756381519872  Ferrero, premio a tutti i dipendenti. In busta...  \n",
       "         1174318674058121218  Ferrero, 2mila euro come premio annuale a tutt...  \n",
       "         1174347893072809984  Centrati gli obiettivi alla Ferrero: oltre 2 m...  \n",
       "...                                                                         ...  \n",
       "83       1417393591022936065  Allievo di Lombroso, è stato tra i padri della...  \n",
       "         1417876798998732810  Roberta #Ferrero: Con il dl Sostegni bis andia...  \n",
       "         1417876798998732810  Roberta #Ferrero: Con il dl Sostegni bis andia...  \n",
       "84       1417876798998732810  Roberta #Ferrero: Con il dl Sostegni bis andia...  \n",
       "         1417876798998732810  Roberta #Ferrero: Con il dl Sostegni bis andia...  \n",
       "\n",
       "[159 rows x 4 columns]"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "spikes_auth_tweets_before"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "spikes_auth_tweets_after = get_tweets_in_date_range(df_tmp, df_tmp[['spike_starting_date']].drop_duplicates(), 'beyond', 7).sort_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>spike_starting_date</th>\n",
       "      <th>spike_ending_date</th>\n",
       "      <th>tweet_date</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>spike_id</th>\n",
       "      <th>tweet_id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"3\" valign=\"top\">0</th>\n",
       "      <th>1163965966323109889</th>\n",
       "      <td>2019-08-20</td>\n",
       "      <td>2019-08-25</td>\n",
       "      <td>2019-08-21</td>\n",
       "      <td>Ferrero, per la holding nuove munizioni da 134...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1164639655574196224</th>\n",
       "      <td>2019-08-20</td>\n",
       "      <td>2019-08-25</td>\n",
       "      <td>2019-08-22</td>\n",
       "      <td>Il 5S Barillari attacca la Nutella, Twitter no...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1164947272284672001</th>\n",
       "      <td>2019-08-20</td>\n",
       "      <td>2019-08-25</td>\n",
       "      <td>2019-08-23</td>\n",
       "      <td>Sampdoria, dopo un anno di tira e molla Ferrer...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">2</th>\n",
       "      <th>1174609247679778816</th>\n",
       "      <td>2019-09-18</td>\n",
       "      <td>2019-09-20</td>\n",
       "      <td>2019-09-19</td>\n",
       "      <td>2.200 euro in più sullo stipendio. La vostra a...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1174618145786650624</th>\n",
       "      <td>2019-09-18</td>\n",
       "      <td>2019-09-20</td>\n",
       "      <td>2019-09-19</td>\n",
       "      <td>La Ferrero riconosce a 6000 dipendenti un prem...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">72</th>\n",
       "      <th>1382427302752047109</th>\n",
       "      <td>2021-04-20</td>\n",
       "      <td>2021-04-21</td>\n",
       "      <td>2021-04-14</td>\n",
       "      <td>\"Ambrogio ho un leggero languorino, la mia non...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1382704030212886536</th>\n",
       "      <td>2021-04-20</td>\n",
       "      <td>2021-04-21</td>\n",
       "      <td>2021-04-15</td>\n",
       "      <td>Commissario Figliuolo al centro vaccinale in F...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75</th>\n",
       "      <th>1396407064595927040</th>\n",
       "      <td>2021-05-20</td>\n",
       "      <td>2021-05-24</td>\n",
       "      <td>2021-05-23</td>\n",
       "      <td>Qualcuno si chiede: \"ma i diciottenni sono tan...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>83</th>\n",
       "      <th>1417876798998732810</th>\n",
       "      <td>2021-07-26</td>\n",
       "      <td>2021-07-26</td>\n",
       "      <td>2021-07-21</td>\n",
       "      <td>Roberta #Ferrero: Con il dl Sostegni bis andia...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>84</th>\n",
       "      <th>1417876798998732810</th>\n",
       "      <td>2021-07-28</td>\n",
       "      <td>2021-07-28</td>\n",
       "      <td>2021-07-21</td>\n",
       "      <td>Roberta #Ferrero: Con il dl Sostegni bis andia...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>133 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                             spike_starting_date spike_ending_date  \\\n",
       "spike_id tweet_id                                                    \n",
       "0        1163965966323109889          2019-08-20        2019-08-25   \n",
       "         1164639655574196224          2019-08-20        2019-08-25   \n",
       "         1164947272284672001          2019-08-20        2019-08-25   \n",
       "2        1174609247679778816          2019-09-18        2019-09-20   \n",
       "         1174618145786650624          2019-09-18        2019-09-20   \n",
       "...                                          ...               ...   \n",
       "72       1382427302752047109          2021-04-20        2021-04-21   \n",
       "         1382704030212886536          2021-04-20        2021-04-21   \n",
       "75       1396407064595927040          2021-05-20        2021-05-24   \n",
       "83       1417876798998732810          2021-07-26        2021-07-26   \n",
       "84       1417876798998732810          2021-07-28        2021-07-28   \n",
       "\n",
       "                              tweet_date  \\\n",
       "spike_id tweet_id                          \n",
       "0        1163965966323109889  2019-08-21   \n",
       "         1164639655574196224  2019-08-22   \n",
       "         1164947272284672001  2019-08-23   \n",
       "2        1174609247679778816  2019-09-19   \n",
       "         1174618145786650624  2019-09-19   \n",
       "...                                  ...   \n",
       "72       1382427302752047109  2021-04-14   \n",
       "         1382704030212886536  2021-04-15   \n",
       "75       1396407064595927040  2021-05-23   \n",
       "83       1417876798998732810  2021-07-21   \n",
       "84       1417876798998732810  2021-07-21   \n",
       "\n",
       "                                                                           text  \n",
       "spike_id tweet_id                                                                \n",
       "0        1163965966323109889  Ferrero, per la holding nuove munizioni da 134...  \n",
       "         1164639655574196224  Il 5S Barillari attacca la Nutella, Twitter no...  \n",
       "         1164947272284672001  Sampdoria, dopo un anno di tira e molla Ferrer...  \n",
       "2        1174609247679778816  2.200 euro in più sullo stipendio. La vostra a...  \n",
       "         1174618145786650624  La Ferrero riconosce a 6000 dipendenti un prem...  \n",
       "...                                                                         ...  \n",
       "72       1382427302752047109  \"Ambrogio ho un leggero languorino, la mia non...  \n",
       "         1382704030212886536  Commissario Figliuolo al centro vaccinale in F...  \n",
       "75       1396407064595927040  Qualcuno si chiede: \"ma i diciottenni sono tan...  \n",
       "83       1417876798998732810  Roberta #Ferrero: Con il dl Sostegni bis andia...  \n",
       "84       1417876798998732810  Roberta #Ferrero: Con il dl Sostegni bis andia...  \n",
       "\n",
       "[133 rows x 4 columns]"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "spikes_auth_tweets_after"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "spikes_auth_tweets_range = get_tweets_in_date_range(df_tmp, df_tmp[['spike_starting_date']].drop_duplicates(),'in_range', range_start=1, range_end=1,).sort_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>spike_starting_date</th>\n",
       "      <th>spike_ending_date</th>\n",
       "      <th>tweet_date</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>spike_id</th>\n",
       "      <th>tweet_id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <th>1163965966323109889</th>\n",
       "      <td>2019-08-20</td>\n",
       "      <td>2019-08-25</td>\n",
       "      <td>2019-08-21</td>\n",
       "      <td>Ferrero, per la holding nuove munizioni da 134...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"4\" valign=\"top\">2</th>\n",
       "      <th>1174264995162599426</th>\n",
       "      <td>2019-09-18</td>\n",
       "      <td>2019-09-20</td>\n",
       "      <td>2019-09-18</td>\n",
       "      <td>Ferrero, premio di 2000 euro ai dipendenti nel...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1174293258714808321</th>\n",
       "      <td>2019-09-18</td>\n",
       "      <td>2019-09-20</td>\n",
       "      <td>2019-09-18</td>\n",
       "      <td>Ferrero, via libera ai premi legati agli obiet...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1174293756381519872</th>\n",
       "      <td>2019-09-18</td>\n",
       "      <td>2019-09-20</td>\n",
       "      <td>2019-09-18</td>\n",
       "      <td>Ferrero, premio a tutti i dipendenti. In busta...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1174318674058121218</th>\n",
       "      <td>2019-09-18</td>\n",
       "      <td>2019-09-20</td>\n",
       "      <td>2019-09-18</td>\n",
       "      <td>Ferrero, 2mila euro come premio annuale a tutt...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>73</th>\n",
       "      <th>1389296628213878787</th>\n",
       "      <td>2021-05-03</td>\n",
       "      <td>2021-05-04</td>\n",
       "      <td>2021-05-03</td>\n",
       "      <td>Napoleone, il romanzo di Ferrero in edicola co...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>82</th>\n",
       "      <th>1417393591022936065</th>\n",
       "      <td>2021-07-20</td>\n",
       "      <td>2021-07-21</td>\n",
       "      <td>2021-07-20</td>\n",
       "      <td>Allievo di Lombroso, è stato tra i padri della...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">83</th>\n",
       "      <th>1417393591022936065</th>\n",
       "      <td>2021-07-26</td>\n",
       "      <td>2021-07-26</td>\n",
       "      <td>2021-07-20</td>\n",
       "      <td>Allievo di Lombroso, è stato tra i padri della...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1417876798998732810</th>\n",
       "      <td>2021-07-26</td>\n",
       "      <td>2021-07-26</td>\n",
       "      <td>2021-07-21</td>\n",
       "      <td>Roberta #Ferrero: Con il dl Sostegni bis andia...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>84</th>\n",
       "      <th>1417876798998732810</th>\n",
       "      <td>2021-07-28</td>\n",
       "      <td>2021-07-28</td>\n",
       "      <td>2021-07-21</td>\n",
       "      <td>Roberta #Ferrero: Con il dl Sostegni bis andia...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>135 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                             spike_starting_date spike_ending_date  \\\n",
       "spike_id tweet_id                                                    \n",
       "0        1163965966323109889          2019-08-20        2019-08-25   \n",
       "2        1174264995162599426          2019-09-18        2019-09-20   \n",
       "         1174293258714808321          2019-09-18        2019-09-20   \n",
       "         1174293756381519872          2019-09-18        2019-09-20   \n",
       "         1174318674058121218          2019-09-18        2019-09-20   \n",
       "...                                          ...               ...   \n",
       "73       1389296628213878787          2021-05-03        2021-05-04   \n",
       "82       1417393591022936065          2021-07-20        2021-07-21   \n",
       "83       1417393591022936065          2021-07-26        2021-07-26   \n",
       "         1417876798998732810          2021-07-26        2021-07-26   \n",
       "84       1417876798998732810          2021-07-28        2021-07-28   \n",
       "\n",
       "                              tweet_date  \\\n",
       "spike_id tweet_id                          \n",
       "0        1163965966323109889  2019-08-21   \n",
       "2        1174264995162599426  2019-09-18   \n",
       "         1174293258714808321  2019-09-18   \n",
       "         1174293756381519872  2019-09-18   \n",
       "         1174318674058121218  2019-09-18   \n",
       "...                                  ...   \n",
       "73       1389296628213878787  2021-05-03   \n",
       "82       1417393591022936065  2021-07-20   \n",
       "83       1417393591022936065  2021-07-20   \n",
       "         1417876798998732810  2021-07-21   \n",
       "84       1417876798998732810  2021-07-21   \n",
       "\n",
       "                                                                           text  \n",
       "spike_id tweet_id                                                                \n",
       "0        1163965966323109889  Ferrero, per la holding nuove munizioni da 134...  \n",
       "2        1174264995162599426  Ferrero, premio di 2000 euro ai dipendenti nel...  \n",
       "         1174293258714808321  Ferrero, via libera ai premi legati agli obiet...  \n",
       "         1174293756381519872  Ferrero, premio a tutti i dipendenti. In busta...  \n",
       "         1174318674058121218  Ferrero, 2mila euro come premio annuale a tutt...  \n",
       "...                                                                         ...  \n",
       "73       1389296628213878787  Napoleone, il romanzo di Ferrero in edicola co...  \n",
       "82       1417393591022936065  Allievo di Lombroso, è stato tra i padri della...  \n",
       "83       1417393591022936065  Allievo di Lombroso, è stato tra i padri della...  \n",
       "         1417876798998732810  Roberta #Ferrero: Con il dl Sostegni bis andia...  \n",
       "84       1417876798998732810  Roberta #Ferrero: Con il dl Sostegni bis andia...  \n",
       "\n",
       "[135 rows x 4 columns]"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "spikes_auth_tweets_range"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "spikes_auth_tweets_before.to_csv('data/authority_filtered/spikes_auth_tweets_before.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Adds a new column counting the length of each spike in days to the input dataframe\n",
    "def get_spikes_days_length(df, id_col='spike_id', start_date_col='spike_starting_date', end_date_col='spike_ending_date'):\n",
    "    output = df.copy()\n",
    "    days_deltas = []\n",
    "    for i, row in output.iterrows():\n",
    "        days_delta = ((row[end_date_col] - row[start_date_col]) + timedelta(days=1))\n",
    "        days_deltas.append(days_delta.days)\n",
    "    output['spike_length'] = days_deltas\n",
    "    return output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_temp = (spikes_auth_tweets_before.reset_index().drop_duplicates(['spike_id']).reset_index()[['spike_starting_date', 'spike_ending_date']])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_spike_window_dates_df = merged_spikes_date_included_df[['spike_id', 'spike_window_date']].drop_duplicates().reset_index().sort_values('spike_id')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>index</th>\n",
       "      <th>spike_id</th>\n",
       "      <th>spike_window_date</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2019-08-20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2019-08-21</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2019-08-22</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>2019-08-23</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>7</td>\n",
       "      <td>4</td>\n",
       "      <td>2019-08-24</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>160</th>\n",
       "      <td>491</td>\n",
       "      <td>160</td>\n",
       "      <td>2021-08-19</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>161</th>\n",
       "      <td>492</td>\n",
       "      <td>161</td>\n",
       "      <td>2021-08-20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>162</th>\n",
       "      <td>493</td>\n",
       "      <td>162</td>\n",
       "      <td>2021-08-23</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>163</th>\n",
       "      <td>494</td>\n",
       "      <td>163</td>\n",
       "      <td>2021-08-24</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>164</th>\n",
       "      <td>495</td>\n",
       "      <td>164</td>\n",
       "      <td>2021-08-25</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>165 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     index  spike_id spike_window_date\n",
       "0        0         0        2019-08-20\n",
       "1        1         1        2019-08-21\n",
       "2        2         2        2019-08-22\n",
       "3        4         3        2019-08-23\n",
       "4        7         4        2019-08-24\n",
       "..     ...       ...               ...\n",
       "160    491       160        2021-08-19\n",
       "161    492       161        2021-08-20\n",
       "162    493       162        2021-08-23\n",
       "163    494       163        2021-08-24\n",
       "164    495       164        2021-08-25\n",
       "\n",
       "[165 rows x 3 columns]"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_spike_window_dates_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "There are not next lines in the DataFrame\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\alxau\\anaconda3\\lib\\site-packages\\pandas\\core\\indexing.py:1637: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  self._setitem_single_block(indexer, value, name)\n"
     ]
    }
   ],
   "source": [
    "all_spike_dates_df = collapse_consecutive_spike_days(all_spike_window_dates_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>index</th>\n",
       "      <th>spike_id</th>\n",
       "      <th>spike_window_date</th>\n",
       "      <th>spike_starting_date</th>\n",
       "      <th>spike_ending_date</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2019-08-20</td>\n",
       "      <td>2019-08-20</td>\n",
       "      <td>2019-08-25</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>2019-08-21</td>\n",
       "      <td>2019-08-20</td>\n",
       "      <td>2019-08-25</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>2019-08-22</td>\n",
       "      <td>2019-08-20</td>\n",
       "      <td>2019-08-25</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>2019-08-23</td>\n",
       "      <td>2019-08-20</td>\n",
       "      <td>2019-08-25</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>7</td>\n",
       "      <td>0</td>\n",
       "      <td>2019-08-24</td>\n",
       "      <td>2019-08-20</td>\n",
       "      <td>2019-08-25</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>160</th>\n",
       "      <td>491</td>\n",
       "      <td>86</td>\n",
       "      <td>2021-08-19</td>\n",
       "      <td>2021-08-19</td>\n",
       "      <td>2021-08-20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>161</th>\n",
       "      <td>492</td>\n",
       "      <td>86</td>\n",
       "      <td>2021-08-20</td>\n",
       "      <td>2021-08-19</td>\n",
       "      <td>2021-08-20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>162</th>\n",
       "      <td>493</td>\n",
       "      <td>87</td>\n",
       "      <td>2021-08-23</td>\n",
       "      <td>2021-08-23</td>\n",
       "      <td>2021-08-25</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>163</th>\n",
       "      <td>494</td>\n",
       "      <td>87</td>\n",
       "      <td>2021-08-24</td>\n",
       "      <td>2021-08-23</td>\n",
       "      <td>2021-08-25</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>164</th>\n",
       "      <td>495</td>\n",
       "      <td>87</td>\n",
       "      <td>2021-08-25</td>\n",
       "      <td>2021-08-23</td>\n",
       "      <td>2021-08-25</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>165 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     index  spike_id spike_window_date spike_starting_date spike_ending_date\n",
       "0        0         0        2019-08-20          2019-08-20        2019-08-25\n",
       "1        1         0        2019-08-21          2019-08-20        2019-08-25\n",
       "2        2         0        2019-08-22          2019-08-20        2019-08-25\n",
       "3        4         0        2019-08-23          2019-08-20        2019-08-25\n",
       "4        7         0        2019-08-24          2019-08-20        2019-08-25\n",
       "..     ...       ...               ...                 ...               ...\n",
       "160    491        86        2021-08-19          2021-08-19        2021-08-20\n",
       "161    492        86        2021-08-20          2021-08-19        2021-08-20\n",
       "162    493        87        2021-08-23          2021-08-23        2021-08-25\n",
       "163    494        87        2021-08-24          2021-08-23        2021-08-25\n",
       "164    495        87        2021-08-25          2021-08-23        2021-08-25\n",
       "\n",
       "[165 rows x 5 columns]"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_spike_dates_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "    spike_id spike_starting_date spike_ending_date\n",
      "0          0          2019-08-20        2019-08-25\n",
      "1          1          2019-09-15        2019-09-15\n",
      "2          2          2019-09-18        2019-09-20\n",
      "3          3          2019-09-22        2019-09-25\n",
      "4          4          2019-10-07        2019-10-10\n",
      "..       ...                 ...               ...\n",
      "83        83          2021-07-26        2021-07-26\n",
      "84        84          2021-07-28        2021-07-28\n",
      "85        85          2021-08-04        2021-08-05\n",
      "86        86          2021-08-19        2021-08-20\n",
      "87        87          2021-08-23        2021-08-25\n",
      "\n",
      "[88 rows x 3 columns]\n"
     ]
    }
   ],
   "source": [
    "all_spike_dates_df = all_spike_dates_df[['spike_id', 'spike_starting_date', 'spike_ending_date']].drop_duplicates().reset_index().drop(columns=['index']).sort_values('spike_ending_date')\n",
    "print(all_spike_dates_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>spike_id</th>\n",
       "      <th>tweet_id</th>\n",
       "      <th>spike_starting_date</th>\n",
       "      <th>spike_ending_date</th>\n",
       "      <th>tweet_date</th>\n",
       "      <th>text</th>\n",
       "      <th>spike_length</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2</td>\n",
       "      <td>1174264995162599426</td>\n",
       "      <td>2019-09-18</td>\n",
       "      <td>2019-09-20</td>\n",
       "      <td>2019-09-18</td>\n",
       "      <td>Ferrero, premio di 2000 euro ai dipendenti nel...</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>1174293258714808321</td>\n",
       "      <td>2019-09-18</td>\n",
       "      <td>2019-09-20</td>\n",
       "      <td>2019-09-18</td>\n",
       "      <td>Ferrero, via libera ai premi legati agli obiet...</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>1174293756381519872</td>\n",
       "      <td>2019-09-18</td>\n",
       "      <td>2019-09-20</td>\n",
       "      <td>2019-09-18</td>\n",
       "      <td>Ferrero, premio a tutti i dipendenti. In busta...</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2</td>\n",
       "      <td>1174318674058121218</td>\n",
       "      <td>2019-09-18</td>\n",
       "      <td>2019-09-20</td>\n",
       "      <td>2019-09-18</td>\n",
       "      <td>Ferrero, 2mila euro come premio annuale a tutt...</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2</td>\n",
       "      <td>1174347893072809984</td>\n",
       "      <td>2019-09-18</td>\n",
       "      <td>2019-09-20</td>\n",
       "      <td>2019-09-18</td>\n",
       "      <td>Centrati gli obiettivi alla Ferrero: oltre 2 m...</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>152</th>\n",
       "      <td>77</td>\n",
       "      <td>1399733841699520517</td>\n",
       "      <td>2021-06-07</td>\n",
       "      <td>2021-06-07</td>\n",
       "      <td>2021-06-01</td>\n",
       "      <td>Il Gruppo piemontese con la holding Cth ha fat...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>153</th>\n",
       "      <td>82</td>\n",
       "      <td>1417393591022936065</td>\n",
       "      <td>2021-07-20</td>\n",
       "      <td>2021-07-21</td>\n",
       "      <td>2021-07-20</td>\n",
       "      <td>Allievo di Lombroso, è stato tra i padri della...</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>154</th>\n",
       "      <td>83</td>\n",
       "      <td>1417393591022936065</td>\n",
       "      <td>2021-07-26</td>\n",
       "      <td>2021-07-26</td>\n",
       "      <td>2021-07-20</td>\n",
       "      <td>Allievo di Lombroso, è stato tra i padri della...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>155</th>\n",
       "      <td>83</td>\n",
       "      <td>1417876798998732810</td>\n",
       "      <td>2021-07-26</td>\n",
       "      <td>2021-07-26</td>\n",
       "      <td>2021-07-21</td>\n",
       "      <td>Roberta #Ferrero: Con il dl Sostegni bis andia...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>157</th>\n",
       "      <td>84</td>\n",
       "      <td>1417876798998732810</td>\n",
       "      <td>2021-07-28</td>\n",
       "      <td>2021-07-28</td>\n",
       "      <td>2021-07-21</td>\n",
       "      <td>Roberta #Ferrero: Con il dl Sostegni bis andia...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>133 rows × 7 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     spike_id             tweet_id spike_starting_date spike_ending_date  \\\n",
       "0           2  1174264995162599426          2019-09-18        2019-09-20   \n",
       "1           2  1174293258714808321          2019-09-18        2019-09-20   \n",
       "2           2  1174293756381519872          2019-09-18        2019-09-20   \n",
       "3           2  1174318674058121218          2019-09-18        2019-09-20   \n",
       "4           2  1174347893072809984          2019-09-18        2019-09-20   \n",
       "..        ...                  ...                 ...               ...   \n",
       "152        77  1399733841699520517          2021-06-07        2021-06-07   \n",
       "153        82  1417393591022936065          2021-07-20        2021-07-21   \n",
       "154        83  1417393591022936065          2021-07-26        2021-07-26   \n",
       "155        83  1417876798998732810          2021-07-26        2021-07-26   \n",
       "157        84  1417876798998732810          2021-07-28        2021-07-28   \n",
       "\n",
       "     tweet_date                                               text  \\\n",
       "0    2019-09-18  Ferrero, premio di 2000 euro ai dipendenti nel...   \n",
       "1    2019-09-18  Ferrero, via libera ai premi legati agli obiet...   \n",
       "2    2019-09-18  Ferrero, premio a tutti i dipendenti. In busta...   \n",
       "3    2019-09-18  Ferrero, 2mila euro come premio annuale a tutt...   \n",
       "4    2019-09-18  Centrati gli obiettivi alla Ferrero: oltre 2 m...   \n",
       "..          ...                                                ...   \n",
       "152  2021-06-01  Il Gruppo piemontese con la holding Cth ha fat...   \n",
       "153  2021-07-20  Allievo di Lombroso, è stato tra i padri della...   \n",
       "154  2021-07-20  Allievo di Lombroso, è stato tra i padri della...   \n",
       "155  2021-07-21  Roberta #Ferrero: Con il dl Sostegni bis andia...   \n",
       "157  2021-07-21  Roberta #Ferrero: Con il dl Sostegni bis andia...   \n",
       "\n",
       "     spike_length  \n",
       "0               3  \n",
       "1               3  \n",
       "2               3  \n",
       "3               3  \n",
       "4               3  \n",
       "..            ...  \n",
       "152             1  \n",
       "153             2  \n",
       "154             1  \n",
       "155             1  \n",
       "157             1  \n",
       "\n",
       "[133 rows x 7 columns]"
      ]
     },
     "execution_count": 71,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_spikes_days_length_df = get_spikes_days_length(spikes_auth_tweets_before.reset_index())\n",
    "all_spikes_days_length_df.drop_duplicates(inplace=True)\n",
    "all_spikes_days_length_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [],
   "source": [
    "random_col = np.random.randint(0, 2, len(all_spikes_days_length_df))\n",
    "all_spikes_days_length_df['is_relevant'] = random_col"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>spike_id</th>\n",
       "      <th>tweet_id</th>\n",
       "      <th>spike_starting_date</th>\n",
       "      <th>spike_ending_date</th>\n",
       "      <th>tweet_date</th>\n",
       "      <th>text</th>\n",
       "      <th>spike_length</th>\n",
       "      <th>is_relevant</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2</td>\n",
       "      <td>1174264995162599426</td>\n",
       "      <td>2019-09-18</td>\n",
       "      <td>2019-09-20</td>\n",
       "      <td>2019-09-18</td>\n",
       "      <td>Ferrero, premio di 2000 euro ai dipendenti nel...</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>1174293258714808321</td>\n",
       "      <td>2019-09-18</td>\n",
       "      <td>2019-09-20</td>\n",
       "      <td>2019-09-18</td>\n",
       "      <td>Ferrero, via libera ai premi legati agli obiet...</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>1174293756381519872</td>\n",
       "      <td>2019-09-18</td>\n",
       "      <td>2019-09-20</td>\n",
       "      <td>2019-09-18</td>\n",
       "      <td>Ferrero, premio a tutti i dipendenti. In busta...</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2</td>\n",
       "      <td>1174318674058121218</td>\n",
       "      <td>2019-09-18</td>\n",
       "      <td>2019-09-20</td>\n",
       "      <td>2019-09-18</td>\n",
       "      <td>Ferrero, 2mila euro come premio annuale a tutt...</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2</td>\n",
       "      <td>1174347893072809984</td>\n",
       "      <td>2019-09-18</td>\n",
       "      <td>2019-09-20</td>\n",
       "      <td>2019-09-18</td>\n",
       "      <td>Centrati gli obiettivi alla Ferrero: oltre 2 m...</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>152</th>\n",
       "      <td>77</td>\n",
       "      <td>1399733841699520517</td>\n",
       "      <td>2021-06-07</td>\n",
       "      <td>2021-06-07</td>\n",
       "      <td>2021-06-01</td>\n",
       "      <td>Il Gruppo piemontese con la holding Cth ha fat...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>153</th>\n",
       "      <td>82</td>\n",
       "      <td>1417393591022936065</td>\n",
       "      <td>2021-07-20</td>\n",
       "      <td>2021-07-21</td>\n",
       "      <td>2021-07-20</td>\n",
       "      <td>Allievo di Lombroso, è stato tra i padri della...</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>154</th>\n",
       "      <td>83</td>\n",
       "      <td>1417393591022936065</td>\n",
       "      <td>2021-07-26</td>\n",
       "      <td>2021-07-26</td>\n",
       "      <td>2021-07-20</td>\n",
       "      <td>Allievo di Lombroso, è stato tra i padri della...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>155</th>\n",
       "      <td>83</td>\n",
       "      <td>1417876798998732810</td>\n",
       "      <td>2021-07-26</td>\n",
       "      <td>2021-07-26</td>\n",
       "      <td>2021-07-21</td>\n",
       "      <td>Roberta #Ferrero: Con il dl Sostegni bis andia...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>157</th>\n",
       "      <td>84</td>\n",
       "      <td>1417876798998732810</td>\n",
       "      <td>2021-07-28</td>\n",
       "      <td>2021-07-28</td>\n",
       "      <td>2021-07-21</td>\n",
       "      <td>Roberta #Ferrero: Con il dl Sostegni bis andia...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>133 rows × 8 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     spike_id             tweet_id spike_starting_date spike_ending_date  \\\n",
       "0           2  1174264995162599426          2019-09-18        2019-09-20   \n",
       "1           2  1174293258714808321          2019-09-18        2019-09-20   \n",
       "2           2  1174293756381519872          2019-09-18        2019-09-20   \n",
       "3           2  1174318674058121218          2019-09-18        2019-09-20   \n",
       "4           2  1174347893072809984          2019-09-18        2019-09-20   \n",
       "..        ...                  ...                 ...               ...   \n",
       "152        77  1399733841699520517          2021-06-07        2021-06-07   \n",
       "153        82  1417393591022936065          2021-07-20        2021-07-21   \n",
       "154        83  1417393591022936065          2021-07-26        2021-07-26   \n",
       "155        83  1417876798998732810          2021-07-26        2021-07-26   \n",
       "157        84  1417876798998732810          2021-07-28        2021-07-28   \n",
       "\n",
       "     tweet_date                                               text  \\\n",
       "0    2019-09-18  Ferrero, premio di 2000 euro ai dipendenti nel...   \n",
       "1    2019-09-18  Ferrero, via libera ai premi legati agli obiet...   \n",
       "2    2019-09-18  Ferrero, premio a tutti i dipendenti. In busta...   \n",
       "3    2019-09-18  Ferrero, 2mila euro come premio annuale a tutt...   \n",
       "4    2019-09-18  Centrati gli obiettivi alla Ferrero: oltre 2 m...   \n",
       "..          ...                                                ...   \n",
       "152  2021-06-01  Il Gruppo piemontese con la holding Cth ha fat...   \n",
       "153  2021-07-20  Allievo di Lombroso, è stato tra i padri della...   \n",
       "154  2021-07-20  Allievo di Lombroso, è stato tra i padri della...   \n",
       "155  2021-07-21  Roberta #Ferrero: Con il dl Sostegni bis andia...   \n",
       "157  2021-07-21  Roberta #Ferrero: Con il dl Sostegni bis andia...   \n",
       "\n",
       "     spike_length  is_relevant  \n",
       "0               3            0  \n",
       "1               3            1  \n",
       "2               3            1  \n",
       "3               3            0  \n",
       "4               3            0  \n",
       "..            ...          ...  \n",
       "152             1            0  \n",
       "153             2            1  \n",
       "154             1            0  \n",
       "155             1            0  \n",
       "157             1            0  \n",
       "\n",
       "[133 rows x 8 columns]"
      ]
     },
     "execution_count": 73,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_spikes_days_length_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>spike_id</th>\n",
       "      <th>tweet_id</th>\n",
       "      <th>spike_starting_date</th>\n",
       "      <th>spike_ending_date</th>\n",
       "      <th>tweet_date</th>\n",
       "      <th>text</th>\n",
       "      <th>spike_length</th>\n",
       "      <th>is_relevant</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2</td>\n",
       "      <td>1174264995162599426</td>\n",
       "      <td>2019-09-18</td>\n",
       "      <td>2019-09-20</td>\n",
       "      <td>2019-09-18</td>\n",
       "      <td>Ferrero, premio di 2000 euro ai dipendenti nel...</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>1174293258714808321</td>\n",
       "      <td>2019-09-18</td>\n",
       "      <td>2019-09-20</td>\n",
       "      <td>2019-09-18</td>\n",
       "      <td>Ferrero, via libera ai premi legati agli obiet...</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>1174293756381519872</td>\n",
       "      <td>2019-09-18</td>\n",
       "      <td>2019-09-20</td>\n",
       "      <td>2019-09-18</td>\n",
       "      <td>Ferrero, premio a tutti i dipendenti. In busta...</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2</td>\n",
       "      <td>1174318674058121218</td>\n",
       "      <td>2019-09-18</td>\n",
       "      <td>2019-09-20</td>\n",
       "      <td>2019-09-18</td>\n",
       "      <td>Ferrero, 2mila euro come premio annuale a tutt...</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2</td>\n",
       "      <td>1174347893072809984</td>\n",
       "      <td>2019-09-18</td>\n",
       "      <td>2019-09-20</td>\n",
       "      <td>2019-09-18</td>\n",
       "      <td>Centrati gli obiettivi alla Ferrero: oltre 2 m...</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>152</th>\n",
       "      <td>77</td>\n",
       "      <td>1399733841699520517</td>\n",
       "      <td>2021-06-07</td>\n",
       "      <td>2021-06-07</td>\n",
       "      <td>2021-06-01</td>\n",
       "      <td>Il Gruppo piemontese con la holding Cth ha fat...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>153</th>\n",
       "      <td>82</td>\n",
       "      <td>1417393591022936065</td>\n",
       "      <td>2021-07-20</td>\n",
       "      <td>2021-07-21</td>\n",
       "      <td>2021-07-20</td>\n",
       "      <td>Allievo di Lombroso, è stato tra i padri della...</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>154</th>\n",
       "      <td>83</td>\n",
       "      <td>1417393591022936065</td>\n",
       "      <td>2021-07-26</td>\n",
       "      <td>2021-07-26</td>\n",
       "      <td>2021-07-20</td>\n",
       "      <td>Allievo di Lombroso, è stato tra i padri della...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>155</th>\n",
       "      <td>83</td>\n",
       "      <td>1417876798998732810</td>\n",
       "      <td>2021-07-26</td>\n",
       "      <td>2021-07-26</td>\n",
       "      <td>2021-07-21</td>\n",
       "      <td>Roberta #Ferrero: Con il dl Sostegni bis andia...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>157</th>\n",
       "      <td>84</td>\n",
       "      <td>1417876798998732810</td>\n",
       "      <td>2021-07-28</td>\n",
       "      <td>2021-07-28</td>\n",
       "      <td>2021-07-21</td>\n",
       "      <td>Roberta #Ferrero: Con il dl Sostegni bis andia...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>133 rows × 8 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     spike_id             tweet_id spike_starting_date spike_ending_date  \\\n",
       "0           2  1174264995162599426          2019-09-18        2019-09-20   \n",
       "1           2  1174293258714808321          2019-09-18        2019-09-20   \n",
       "2           2  1174293756381519872          2019-09-18        2019-09-20   \n",
       "3           2  1174318674058121218          2019-09-18        2019-09-20   \n",
       "4           2  1174347893072809984          2019-09-18        2019-09-20   \n",
       "..        ...                  ...                 ...               ...   \n",
       "152        77  1399733841699520517          2021-06-07        2021-06-07   \n",
       "153        82  1417393591022936065          2021-07-20        2021-07-21   \n",
       "154        83  1417393591022936065          2021-07-26        2021-07-26   \n",
       "155        83  1417876798998732810          2021-07-26        2021-07-26   \n",
       "157        84  1417876798998732810          2021-07-28        2021-07-28   \n",
       "\n",
       "     tweet_date                                               text  \\\n",
       "0    2019-09-18  Ferrero, premio di 2000 euro ai dipendenti nel...   \n",
       "1    2019-09-18  Ferrero, via libera ai premi legati agli obiet...   \n",
       "2    2019-09-18  Ferrero, premio a tutti i dipendenti. In busta...   \n",
       "3    2019-09-18  Ferrero, 2mila euro come premio annuale a tutt...   \n",
       "4    2019-09-18  Centrati gli obiettivi alla Ferrero: oltre 2 m...   \n",
       "..          ...                                                ...   \n",
       "152  2021-06-01  Il Gruppo piemontese con la holding Cth ha fat...   \n",
       "153  2021-07-20  Allievo di Lombroso, è stato tra i padri della...   \n",
       "154  2021-07-20  Allievo di Lombroso, è stato tra i padri della...   \n",
       "155  2021-07-21  Roberta #Ferrero: Con il dl Sostegni bis andia...   \n",
       "157  2021-07-21  Roberta #Ferrero: Con il dl Sostegni bis andia...   \n",
       "\n",
       "     spike_length  is_relevant  \n",
       "0               3            0  \n",
       "1               3            1  \n",
       "2               3            1  \n",
       "3               3            0  \n",
       "4               3            0  \n",
       "..            ...          ...  \n",
       "152             1            0  \n",
       "153             2            1  \n",
       "154             1            0  \n",
       "155             1            0  \n",
       "157             1            0  \n",
       "\n",
       "[133 rows x 8 columns]"
      ]
     },
     "execution_count": 74,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "auth_tweets_before_df = all_spikes_days_length_df.merge(auth_filtered_df[['id', 'user_username']], left_on='tweet_id', right_on='id')\n",
    "# all_spikes_days_length_df = all_spikes_days_length_df.drop('id')\n",
    "all_spikes_days_length_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0        GiorgiaMeloni\n",
       "1        GuidoCrosetto\n",
       "4          LegaSalvini\n",
       "5         ElioLannutti\n",
       "6      matteosalvinimi\n",
       "8          NFratoianni\n",
       "16     TeresaBellanova\n",
       "34      borghi_claudio\n",
       "39     stanzaselvaggia\n",
       "41      petergomezblog\n",
       "42        FranFerrante\n",
       "43            gonufrio\n",
       "45      ap_legambiente\n",
       "46     FurlanAnnamaria\n",
       "47            fai_cisl\n",
       "51             fiomnet\n",
       "52        LuciaCuffaro\n",
       "53        LucaBizzarri\n",
       "54         QRepubblica\n",
       "55               ADUC1\n",
       "56      ilmessaggeroit\n",
       "57     FattoAlimentare\n",
       "58            Corriere\n",
       "59      SecolodItalia1\n",
       "60     fattoquotidiano\n",
       "61       MilanoFinanza\n",
       "62          repubblica\n",
       "64            LaStampa\n",
       "68       Affaritaliani\n",
       "71        VanityFairIt\n",
       "73          Gazzettino\n",
       "74          greenMe_it\n",
       "82        Agenzia_Ansa\n",
       "83              GDS_it\n",
       "91           Linkiesta\n",
       "100     giornalettismo\n",
       "103     ilSalvagenteit\n",
       "105       lamescolanza\n",
       "116    alimentandonews\n",
       "127      ItaliaaTavola\n",
       "156        wireditalia\n",
       "197     Agenzia_Italia\n",
       "220         ilgiornale\n",
       "265      rinnovabiliit\n",
       "267      Qualenergiait\n",
       "Name: user_username, dtype: object"
      ]
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "relevant_auths = auth_filtered_df['user_username'].drop_duplicates()\n",
    "relevant_auths"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_coherency_index(df, auth_list):\n",
    "    output = []\n",
    "    tmp = df.copy()\n",
    "    tmp['count'] = 1\n",
    "    df_sum = tmp.groupby(pd.Grouper('user_username')).sum()\n",
    "    print(df_sum)\n",
    "    df_count = tmp.groupby(pd.Grouper('user_username')).count()\n",
    "    print(df_count)\n",
    "    output.append(df_sum/df_count)\n",
    "    output = pd.concat(output).reset_index()\n",
    "    return output[['user_username', 'is_relevant']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 139,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                 spike_id      tweet_id  spike_length  is_relevant  \\\n",
      "user_username                                                        \n",
      "Corriere            655.0  1.950579e+19          39.0            8   \n",
      "MilanoFinanza       375.0  1.048187e+19          17.0            5   \n",
      "Affaritaliani       700.0  2.200762e+19          36.0            8   \n",
      "fattoquotidiano     257.0  7.779937e+18          16.0            3   \n",
      "repubblica          706.0  2.202297e+19          36.0            9   \n",
      "VanityFairIt         16.0  4.723819e+18          10.0            1   \n",
      "GiorgiaMeloni         5.0  2.349236e+18           7.0            1   \n",
      "Gazzettino           11.0  2.374747e+18           3.0            1   \n",
      "Agenzia_Ansa        249.0  7.768618e+18          10.0            4   \n",
      "GDS_it                6.0  1.187680e+18           1.0            1   \n",
      "ilmessaggeroit        7.0  1.189249e+18           1.0            0   \n",
      "giornalettismo       21.0  2.399264e+18           2.0            1   \n",
      "SecolodItalia1       21.0  2.399434e+18           2.0            1   \n",
      "ilSalvagenteit      154.0  7.471689e+18          12.0            3   \n",
      "stanzaselvaggia      23.0  2.401546e+18           5.0            0   \n",
      "LaStampa            365.0  1.163910e+19          17.0            4   \n",
      "ElioLannutti         37.0  2.449263e+18           5.0            1   \n",
      "ItaliaaTavola       400.0  9.378381e+18          16.0            5   \n",
      "greenMe_it           95.0  4.962548e+18           6.0            2   \n",
      "QRepubblica          20.0  1.226980e+18           1.0            1   \n",
      "lamescolanza        122.0  3.884546e+18           6.0            2   \n",
      "LegaSalvini         413.0  1.062157e+19          11.0            4   \n",
      "GuidoCrosetto        81.0  2.582208e+18           6.0            1   \n",
      "Agenzia_Italia       47.0  1.314259e+18           1.0            1   \n",
      "FattoAlimentare      61.0  1.357606e+18           4.0            1   \n",
      "ilgiornale          308.0  5.599642e+18           8.0            2   \n",
      "\n",
      "                           id  count  \n",
      "user_username                         \n",
      "Corriere         1.950579e+19   15.0  \n",
      "MilanoFinanza    1.048187e+19    8.0  \n",
      "Affaritaliani    2.200762e+19   17.0  \n",
      "fattoquotidiano  7.779937e+18    6.0  \n",
      "repubblica       2.202297e+19   17.0  \n",
      "VanityFairIt     4.723819e+18    4.0  \n",
      "GiorgiaMeloni    2.349236e+18    2.0  \n",
      "Gazzettino       2.374747e+18    2.0  \n",
      "Agenzia_Ansa     7.768618e+18    6.0  \n",
      "GDS_it           1.187680e+18    1.0  \n",
      "ilmessaggeroit   1.189249e+18    1.0  \n",
      "giornalettismo   2.399264e+18    2.0  \n",
      "SecolodItalia1   2.399434e+18    2.0  \n",
      "ilSalvagenteit   7.471689e+18    6.0  \n",
      "stanzaselvaggia  2.401546e+18    2.0  \n",
      "LaStampa         1.163910e+19    9.0  \n",
      "ElioLannutti     2.449263e+18    2.0  \n",
      "ItaliaaTavola    9.378381e+18    7.0  \n",
      "greenMe_it       4.962548e+18    4.0  \n",
      "QRepubblica      1.226980e+18    1.0  \n",
      "lamescolanza     3.884546e+18    3.0  \n",
      "LegaSalvini      1.062157e+19    8.0  \n",
      "GuidoCrosetto    2.582208e+18    2.0  \n",
      "Agenzia_Italia   1.314259e+18    1.0  \n",
      "FattoAlimentare  1.357606e+18    1.0  \n",
      "ilgiornale       5.599642e+18    4.0  \n",
      "                 spike_id  tweet_id  spike_starting_date  spike_ending_date  \\\n",
      "user_username                                                                 \n",
      "Corriere               15        15                   15                 15   \n",
      "MilanoFinanza           8         8                    8                  8   \n",
      "Affaritaliani          17        17                   17                 17   \n",
      "fattoquotidiano         6         6                    6                  6   \n",
      "repubblica             17        17                   17                 17   \n",
      "VanityFairIt            4         4                    4                  4   \n",
      "GiorgiaMeloni           2         2                    2                  2   \n",
      "Gazzettino              2         2                    2                  2   \n",
      "Agenzia_Ansa            6         6                    6                  6   \n",
      "GDS_it                  1         1                    1                  1   \n",
      "ilmessaggeroit          1         1                    1                  1   \n",
      "giornalettismo          2         2                    2                  2   \n",
      "SecolodItalia1          2         2                    2                  2   \n",
      "ilSalvagenteit          6         6                    6                  6   \n",
      "stanzaselvaggia         2         2                    2                  2   \n",
      "LaStampa                9         9                    9                  9   \n",
      "ElioLannutti            2         2                    2                  2   \n",
      "ItaliaaTavola           7         7                    7                  7   \n",
      "greenMe_it              4         4                    4                  4   \n",
      "QRepubblica             1         1                    1                  1   \n",
      "lamescolanza            3         3                    3                  3   \n",
      "LegaSalvini             8         8                    8                  8   \n",
      "GuidoCrosetto           2         2                    2                  2   \n",
      "Agenzia_Italia          1         1                    1                  1   \n",
      "FattoAlimentare         1         1                    1                  1   \n",
      "ilgiornale              4         4                    4                  4   \n",
      "\n",
      "                 tweet_date  text  spike_length  is_relevant  id  count  \n",
      "user_username                                                            \n",
      "Corriere                 15    15            15           15  15     15  \n",
      "MilanoFinanza             8     8             8            8   8      8  \n",
      "Affaritaliani            17    17            17           17  17     17  \n",
      "fattoquotidiano           6     6             6            6   6      6  \n",
      "repubblica               17    17            17           17  17     17  \n",
      "VanityFairIt              4     4             4            4   4      4  \n",
      "GiorgiaMeloni             2     2             2            2   2      2  \n",
      "Gazzettino                2     2             2            2   2      2  \n",
      "Agenzia_Ansa              6     6             6            6   6      6  \n",
      "GDS_it                    1     1             1            1   1      1  \n",
      "ilmessaggeroit            1     1             1            1   1      1  \n",
      "giornalettismo            2     2             2            2   2      2  \n",
      "SecolodItalia1            2     2             2            2   2      2  \n",
      "ilSalvagenteit            6     6             6            6   6      6  \n",
      "stanzaselvaggia           2     2             2            2   2      2  \n",
      "LaStampa                  9     9             9            9   9      9  \n",
      "ElioLannutti              2     2             2            2   2      2  \n",
      "ItaliaaTavola             7     7             7            7   7      7  \n",
      "greenMe_it                4     4             4            4   4      4  \n",
      "QRepubblica               1     1             1            1   1      1  \n",
      "lamescolanza              3     3             3            3   3      3  \n",
      "LegaSalvini               8     8             8            8   8      8  \n",
      "GuidoCrosetto             2     2             2            2   2      2  \n",
      "Agenzia_Italia            1     1             1            1   1      1  \n",
      "FattoAlimentare           1     1             1            1   1      1  \n",
      "ilgiornale                4     4             4            4   4      4  \n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_username</th>\n",
       "      <th>is_relevant</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Corriere</td>\n",
       "      <td>0.533333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>MilanoFinanza</td>\n",
       "      <td>0.625000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Affaritaliani</td>\n",
       "      <td>0.470588</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>fattoquotidiano</td>\n",
       "      <td>0.500000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>repubblica</td>\n",
       "      <td>0.529412</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>VanityFairIt</td>\n",
       "      <td>0.250000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>GiorgiaMeloni</td>\n",
       "      <td>0.500000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Gazzettino</td>\n",
       "      <td>0.500000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Agenzia_Ansa</td>\n",
       "      <td>0.666667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>GDS_it</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>ilmessaggeroit</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>giornalettismo</td>\n",
       "      <td>0.500000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>SecolodItalia1</td>\n",
       "      <td>0.500000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>ilSalvagenteit</td>\n",
       "      <td>0.500000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>stanzaselvaggia</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>LaStampa</td>\n",
       "      <td>0.444444</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>ElioLannutti</td>\n",
       "      <td>0.500000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>ItaliaaTavola</td>\n",
       "      <td>0.714286</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>greenMe_it</td>\n",
       "      <td>0.500000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>QRepubblica</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>lamescolanza</td>\n",
       "      <td>0.666667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>LegaSalvini</td>\n",
       "      <td>0.500000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>GuidoCrosetto</td>\n",
       "      <td>0.500000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>Agenzia_Italia</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>FattoAlimentare</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>ilgiornale</td>\n",
       "      <td>0.500000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      user_username  is_relevant\n",
       "0          Corriere     0.533333\n",
       "1     MilanoFinanza     0.625000\n",
       "2     Affaritaliani     0.470588\n",
       "3   fattoquotidiano     0.500000\n",
       "4        repubblica     0.529412\n",
       "5      VanityFairIt     0.250000\n",
       "6     GiorgiaMeloni     0.500000\n",
       "7        Gazzettino     0.500000\n",
       "8      Agenzia_Ansa     0.666667\n",
       "9            GDS_it     1.000000\n",
       "10   ilmessaggeroit     0.000000\n",
       "11   giornalettismo     0.500000\n",
       "12   SecolodItalia1     0.500000\n",
       "13   ilSalvagenteit     0.500000\n",
       "14  stanzaselvaggia     0.000000\n",
       "15         LaStampa     0.444444\n",
       "16     ElioLannutti     0.500000\n",
       "17    ItaliaaTavola     0.714286\n",
       "18       greenMe_it     0.500000\n",
       "19      QRepubblica     1.000000\n",
       "20     lamescolanza     0.666667\n",
       "21      LegaSalvini     0.500000\n",
       "22    GuidoCrosetto     0.500000\n",
       "23   Agenzia_Italia     1.000000\n",
       "24  FattoAlimentare     1.000000\n",
       "25       ilgiornale     0.500000"
      ]
     },
     "execution_count": 139,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "get_coherency_index(auth_tweets_before_df, relevant_auths)"
   ]
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "f446ed4898e137face024438143ad86cf21c8cccb3ccfa44cb119e4a51b03028"
  },
  "kernelspec": {
   "display_name": "Python 3.8.3 64-bit ('base': conda)",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}