src/Data-Stats.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "0ca2ac08",
   "metadata": {},
   "source": [
    "# Data stats"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0b2c9d63",
   "metadata": {},
   "outputs": [],
   "source": [
    "%%javascript\n",
    "$('<div id=\"toc\"></div>').css({position: 'fixed', top: '120px', left: 0}).appendTo(document.body);\n",
    "$.getScript('https://kmahelona.github.io/ipython_notebook_goodies/ipython_notebook_toc.js');\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "17630718",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import re\n",
    "import json\n",
    "from tqdm import tqdm, trange\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3f5549e0",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import StratifiedKFold\n",
    "skf = StratifiedKFold(n_splits=5)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0fcb18d4",
   "metadata": {},
   "outputs": [],
   "source": [
    "# change matplotlib parameters\n",
    "plt.rcParams['figure.figsize'] = [15, 10]\n",
    "plt.rcParams.update({'font.size': 30})"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f51e00a4",
   "metadata": {},
   "source": [
    "# COVIDSenti - Sentiments"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "82545365",
   "metadata": {},
   "outputs": [],
   "source": [
    "# sentiment\n",
    "# https://www.ncbi.nlm.nih.gov/pmc/articles/PMC8545013/pdf/tcss-razzak-3051189.pdf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "311e6004",
   "metadata": {},
   "outputs": [],
   "source": [
    "path = \"../data/COVIDSenti/\"\n",
    "\n",
    "df = pd.read_csv(path+'COVIDSenti.csv')\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bfc484d6",
   "metadata": {},
   "outputs": [],
   "source": [
    "skf = StratifiedKFold(n_splits=5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "47f2c2e1",
   "metadata": {},
   "outputs": [],
   "source": [
    "y = df['label'].tolist()\n",
    "X = df['tweet'].tolist()\n",
    "for i, (train_index, test_index) in enumerate(skf.split(X, y)):\n",
    "    count = [0,0,0]\n",
    "    for ind in train_index:\n",
    "        if y[ind]=='neu':\n",
    "            count[1]+=1\n",
    "        elif y[ind]=='neg':\n",
    "            count[0]+=1\n",
    "        else:\n",
    "            count[2]+=1\n",
    "            \n",
    "    sub_df = df.iloc[test_index]\n",
    "    sub_df.to_csv('../data/COVIDSenti/fold'+str(i)+'.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "86e8b3be",
   "metadata": {},
   "outputs": [],
   "source": [
    "sub = df['label'].tolist()\n",
    "counts = [sub.count(a) for a in ['neg', 'neu', 'pos']]\n",
    "counts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b0a79a7c",
   "metadata": {},
   "outputs": [],
   "source": [
    "Y = counts\n",
    "COLORS = [\"red\", \"yellow\", \"green\"]\n",
    "\n",
    "names = ['Negative', 'Neutral', 'Positive']\n",
    "sns.set_style(\"darkgrid\")\n",
    "ax = sns.barplot(x=names, y=Y, palette=COLORS)\n",
    "for bars in ax.containers:\n",
    "    ax.bar_label(bars, fmt=\"%.f\")\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fe805ae4",
   "metadata": {},
   "source": [
    "# stance-detection-in-covid-19-tweets - Stance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "79e551e4",
   "metadata": {},
   "outputs": [],
   "source": [
    "# stance towards a topic + sentiment\n",
    "# https://aclanthology.org/2021.acl-long.127.pdf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "22ca540a",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "path = '../data/stance-detection-in-covid-19-tweets/dataset/'\n",
    "\n",
    "tweetIds = []\n",
    "df = pd.DataFrame()\n",
    "df['Tweet Id'] = []\n",
    "df['Target'] = []\n",
    "df['Stance'] = []\n",
    "df['Opinion Towards'] = []\n",
    "df['Sentiment'] = []\n",
    "for f in os.listdir(path):\n",
    "    if 'noisy' not in f:\n",
    "        df2 = pd.read_csv(path+f)\n",
    "        df = pd.concat([df, df2])\n",
    "        tweetIds.extend(df2['Tweet Id'].tolist())\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b0f71ad2",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "tweetIds = []\n",
    "for t_id in df['Tweet Id'].tolist():\n",
    "    tweetIds.append(int(format(t_id, '.0f')))\n",
    "df['Tweet Id'] = tweetIds\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0a1ccbec",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('../data/stance-detection-in-covid-19-tweets/tweets3.jsonl', 'r') as f:\n",
    "    json_list = list(f)\n",
    "\n",
    "tweets_list = []\n",
    "   \n",
    "for json_str in json_list:\n",
    "    tweets = json.loads(json_str)\n",
    "    for t in tweets['data']:\n",
    "        tweets_list.append(t)\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "841808c8",
   "metadata": {},
   "outputs": [],
   "source": [
    "t_ids = [int(t['id']) for t in tweets_list]\n",
    "tweets = [t['text'] for t in tweets_list]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e439b5ea",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "sub = df[df['Tweet Id'].isin(t_ids)]\n",
    "sub['tweet'] = tweets\n",
    "sub"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c867e977",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "for target in sub['Target'].unique():\n",
    "    tmp = sub[sub['Target']== target]\n",
    "    y = tmp['Stance'].tolist()\n",
    "    X = tmp['tweet'].tolist()\n",
    "    for i, (train_index, test_index) in enumerate(skf.split(X, y)):\n",
    "        sub_df = tmp.iloc[test_index]\n",
    "        sub_df.to_csv('../data/stance-detection-in-covid-19-tweets/'+str(target)+'/fold'+str(i)+'.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "343d1699",
   "metadata": {},
   "outputs": [],
   "source": [
    "tmp = sub[sub['Target']=='stay_at_home_orders']['Stance'].tolist()\n",
    "counts = [tmp.count(a) for a in ['AGAINST', 'NONE', 'FAVOR']]\n",
    "counts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "63d47b1a",
   "metadata": {},
   "outputs": [],
   "source": [
    "stances = ['stay_at_home_orders', 'fauci', 'face_masks', 'school_closures']\n",
    "\n",
    "Y = counts\n",
    "COLORS = [\"red\", \"yellow\", \"green\"]\n",
    "\n",
    "names = ['Against', 'None', 'Favor']\n",
    "sns.set_style(\"darkgrid\")\n",
    "ax = sns.barplot(x=names, y=Y, palette=COLORS)\n",
    "for bars in ax.containers:\n",
    "    ax.bar_label(bars, fmt=\"%.f\")\n",
    "\n",
    "#ax.set(title=\"Stance towards 'stay_at_home_orders'\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "08da6d5f",
   "metadata": {},
   "source": [
    "# COVID-19 Twitter Dataset with Latent Topics - Emotion"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1e13dfe3",
   "metadata": {},
   "outputs": [],
   "source": [
    "# emotion\n",
    "# https://arxiv.org/pdf/2007.06954.pdf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "eac1b4ca",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "path = '../data/covid-latent/Twitter-COVID-dataset---June2022/'\n",
    "\n",
    "df = pd.read_csv(path+'tweetid_userid_keyword_sentiments_emotions.csv')\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7d92a804",
   "metadata": {},
   "outputs": [],
   "source": [
    "ems = df['emotion'].tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "26e8d4b5",
   "metadata": {},
   "outputs": [],
   "source": [
    "set(ems)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "05966b31",
   "metadata": {},
   "outputs": [],
   "source": [
    "tweetIds = df['tweet_ID'].tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6568c69c",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('../data/covid-latent/Twitter-COVID-dataset---June2022/tweets.jsonl', 'r') as f:\n",
    "    json_list = list(f)\n",
    "\n",
    "tweets_list = []\n",
    "t_ids = []\n",
    "for json_str in tqdm(json_list):\n",
    "    tweets = json.loads(json_str)\n",
    "    \n",
    "    \n",
    "    for t in tweets['data']:\n",
    "        tweets_list.append(t)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f0a46986",
   "metadata": {},
   "outputs": [],
   "source": [
    "t_ids = [int(t['id']) for t in tweets_list]\n",
    "tweets = [t['text'] for t in tweets_list]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a1d1a194",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_data = df[df['tweet_ID'].isin(t_ids)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f290e7d1",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ec270cea",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_data['tweet'] = tweets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "eb741894",
   "metadata": {},
   "outputs": [],
   "source": [
    "num = 100000\n",
    "\n",
    "A = 75824459\n",
    "F = 64633650\n",
    "H = 59444056\n",
    "S = 14856610\n",
    "N = len(df)-A-F-H-S\n",
    "\n",
    "A = A/len(df)*num\n",
    "F = F/len(df)*num\n",
    "H = H/len(df)*num\n",
    "S = S/len(df)*num\n",
    "N = N/len(df)*num\n",
    "\n",
    "L = [A, F, H, S]\n",
    "txt = ['A', 'F', 'H', 'S']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fd4f266f",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "tmp_df = pd.DataFrame(columns=df_data.columns)\n",
    "\n",
    "for x in range(0, len(L)):\n",
    "    sam = df_data[df_data['emotion']==txt[x]].sample(round(L[x]))\n",
    "    tmp_df = pd.concat([tmp_df, sam])\n",
    "\n",
    "sam = df_data[df_data['emotion'].isna()].sample(round(N))\n",
    "tmp_df = pd.concat([tmp_df, sam])\n",
    "\n",
    "tmp_df = tmp_df.sample(frac=1).reset_index(drop=True)\n",
    "\n",
    "tmp_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0b939b83",
   "metadata": {},
   "outputs": [],
   "source": [
    "emotions = ['N' if not e==e else e for e in tmp_df['emotion'].tolist()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c5da10a5",
   "metadata": {},
   "outputs": [],
   "source": [
    "ids = []\n",
    "counts = {'N':0,\n",
    "         'H':0,\n",
    "         'A':0, \n",
    "         'S':0,\n",
    "         'F':0}\n",
    "for i in range(0, len(emotions)):\n",
    "    e = emotions[i]\n",
    "    if counts[e]<5000:\n",
    "        ids.append(i)\n",
    "        counts[e]+=1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1e6415bf",
   "metadata": {},
   "outputs": [],
   "source": [
    "subset = sub.iloc[ids]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "59d4f45d",
   "metadata": {},
   "outputs": [],
   "source": [
    "y = ['N' if not e==e else e for e in subset['emotion'].tolist()]\n",
    "X = subset['tweet'].tolist()\n",
    "for i, (train_index, test_index) in enumerate(skf.split(X, y)):\n",
    "    sub_df = subset.iloc[test_index]\n",
    "    sub_df.to_csv('../data/covid-latent/undersampling/fold'+str(i)+'.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "87485435",
   "metadata": {},
   "outputs": [],
   "source": [
    "emotions = ['N' if not e==e else e for e in sub['emotion'].tolist()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6fd34f5e",
   "metadata": {},
   "outputs": [],
   "source": [
    "counts = []\n",
    "E = ['H', 'A', 'S', 'F', 'N']\n",
    "for e in E:\n",
    "    counts.append(emotions.count(e))\n",
    "counts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b08e86a4",
   "metadata": {},
   "outputs": [],
   "source": [
    "Y = counts\n",
    "COLORS = [\"green\", \"red\", \"yellow\", \"blue\", \"purple\"]\n",
    "\n",
    "names = ['Happiness', 'Anger', 'Sadness', 'Fear', 'None']\n",
    "sns.set_style(\"darkgrid\")\n",
    "ax = sns.barplot(x=names, y=Y, palette=COLORS)\n",
    "for bars in ax.containers:\n",
    "    ax.bar_label(bars, fmt=\"%.f\")\n",
    "\n",
    "#ax.set(title='Emotion attribute distribution')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b14019ca",
   "metadata": {},
   "source": [
    "# Birdwatch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "54945188",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "df = pd.read_csv('../data/birdwatch/notes-00000.tsv', sep='\\t')\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b8bfcad7",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('../data/birdwatch/idtweetVALUES.jsonl', 'r') as f:\n",
    "    json_list = list(f)\n",
    "\n",
    "tweets_list = []\n",
    "ids_list = []\n",
    "   \n",
    "for json_str in json_list:\n",
    "    tweets = json.loads(json_str)\n",
    "    tweets_list.append(tweets['full_text'])\n",
    "    ids_list.append(tweets['id'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0cb09a53",
   "metadata": {},
   "outputs": [],
   "source": [
    "c = [0, 0, 0, 0, 0, 0]\n",
    "labels = ['MISINFORMED_OR_POTENTIALLY_MISLEADING', 'NOT_MISLEADING']\n",
    "\n",
    "relevant_tweets = []\n",
    "relevant_ids = []\n",
    "relevant_notes = []\n",
    "\n",
    "for i in range(0, len(ids_list)):\n",
    "    tid = ids_list[i]\n",
    "    sub = df[df['tweetId']==tid]\n",
    "    m = len(sub) if len(sub)<7 else 6\n",
    "    c[m-1]+=1\n",
    "    notes = sub['classification'].tolist()\n",
    "    counts_lab = [notes.count(l) for l in labels]\n",
    "    \n",
    "    if counts_lab[0]!=counts_lab[1]:\n",
    "        relevant_tweets.append(tweets_list[i])\n",
    "        relevant_ids.append(tid)\n",
    "        relevant_notes.append(labels[np.argmax(counts_lab)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ef28a3ab",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "new_df = pd.DataFrame()\n",
    "new_df['id'] = relevant_ids\n",
    "new_df['tweet'] = relevant_tweets\n",
    "new_df['note'] = relevant_notes\n",
    "new_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a6955560",
   "metadata": {},
   "outputs": [],
   "source": [
    "y = new_df['note'].tolist()\n",
    "X = new_df['tweet'].tolist()\n",
    "for i, (train_index, test_index) in enumerate(skf.split(X, y)):\n",
    "    sub_df = new_df.iloc[test_index]\n",
    "    sub_df.to_csv('../data/birdwatch/fold'+str(i)+'.csv', index=False)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8d23e6fb",
   "metadata": {},
   "outputs": [],
   "source": [
    "counts = [test_df['note'].tolist().count(a) for a in range(0, 2)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "23f2853e",
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "Y = counts\n",
    "COLORS = [\"lightcoral\", \"lightgreen\"]\n",
    "\n",
    "names = ['Misleading', 'Not Misleading']\n",
    "sns.set_style(\"darkgrid\")\n",
    "ax = sns.barplot(x=names, y=Y, palette=COLORS)\n",
    "for bars in ax.containers:\n",
    "    ax.bar_label(bars, fmt=\"%.f\")\n",
    "\n",
    "#ax.set(title='Emotion attribute distribution')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "792f7738",
   "metadata": {},
   "source": [
    "# Russian Troll"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b95ede81",
   "metadata": {},
   "outputs": [],
   "source": [
    "sub = pd.read_csv('../data/russian-troll-tweets/IRAhandle_tweets_2.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fc71b2b4",
   "metadata": {},
   "outputs": [],
   "source": [
    "dfs = []\n",
    "\n",
    "for i in trange(1, 14):\n",
    "    sub = pd.read_csv('../data/russian-troll-tweets/IRAhandle_tweets_'+str(i)+'.csv')\n",
    "    dfs.append(sub)\n",
    "    \n",
    "df = pd.concat(dfs)\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "57b4b627",
   "metadata": {},
   "outputs": [],
   "source": [
    "sub = df['account_category'].tolist()\n",
    "counts = [sub.count(a) for a in df['account_category'].unique()]\n",
    "better_counts = [] # merge everything that is not left/right \n",
    "better_counts.append(counts[3])\n",
    "better_counts.append(sum(counts)-counts[3]-counts[0])\n",
    "better_counts.append(counts[0])\n",
    "better_counts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "996f7d0c",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "Y = better_counts\n",
    "COLORS = [\"blue\", \"grey\", \"red\"]\n",
    "\n",
    "names = ['Left', 'Other', 'Right']\n",
    "sns.set_style(\"darkgrid\")\n",
    "ax = sns.barplot(x=names, y=Y, palette=COLORS)\n",
    "for bars in ax.containers:\n",
    "    ax.bar_label(bars, fmt=\"%.f\")\n",
    "\n",
    "#ax.set(title=\"Stance towards 'face masks'\")\n",
    "plt.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}