Results.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "annual-hometown",
   "metadata": {},
   "source": [
    "# Results.ipynb\n",
    "\n",
    "### This notebook analyses the labeled Tweets for the Results section of the thesis. A table and figures are written to a local folder.\n",
    "\n",
    "Author: Erik Puijk <br>\n",
    "Date  : May 9, 2022"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "organic-conversion",
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "import pandas as pd\n",
    "import csv\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "plt.close(\"all\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "killing-scheme",
   "metadata": {},
   "outputs": [],
   "source": [
    "def read_json(path):\n",
    "    \"\"\" Read the Tweets from a given text file and return in JSON-format. \"\"\"\n",
    "    \n",
    "    content = \"\"\n",
    "    \n",
    "    try:\n",
    "        with open(path, 'r') as f:\n",
    "            content = json.loads(f.read())\n",
    "    except IOError:\n",
    "        print(\"I/O error\")\n",
    "        \n",
    "    return content"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "functioning-particular",
   "metadata": {},
   "outputs": [],
   "source": [
    "def write_csv(headers, rows, path):\n",
    "    \"\"\" Write results from the analysis to a csv-file in a specified path. \"\"\"\n",
    "    \n",
    "    with open(path, 'w') as f:\n",
    "        w = csv.writer(f, lineterminator='\\n')\n",
    "        \n",
    "        w.writerow(headers)\n",
    "        w.writerows(rows)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "greek-trainer",
   "metadata": {},
   "outputs": [],
   "source": [
    "def fix_date(date):\n",
    "    \"\"\" Takes a date in the format yyyy-mm-dd and returns date in format dd-mm. \"\"\"\n",
    "\n",
    "    return str(date)[5:10][3:] + '-' + str(date)[5:10][:2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "thorough-siemens",
   "metadata": {},
   "outputs": [],
   "source": [
    "def avg_engagement_per_category(tweets):\n",
    "    \"\"\" Count the occurrences and mean engagement of each category. \"\"\"\n",
    "    \n",
    "    cat_con = [['cat_con', x, sum(tweets.cat_con == x), \n",
    "                \"%.10f%%\" % round(sum(tweets.cat_con == x) / len(tweets) * 100, 10),\n",
    "                \"%.2f\" % round(tweets.loc[tweets['cat_con'] == x, 'total_engagement'].mean() * 1000, 2)]\n",
    "               for x in sorted(tweets.cat_con.unique())]\n",
    "    cat_act = [['cat_act', x, sum(tweets.cat_act == x), \n",
    "                \"%.10f%%\" % round(sum(tweets.cat_act == x) / len(tweets) * 100, 10),\n",
    "                \"%.2f\" % round(tweets.loc[tweets['cat_act'] == x, 'total_engagement'].mean() * 1000, 2)]\n",
    "               for x in sorted(tweets.cat_act.unique())]\n",
    "    media_type = [['media_type', x, sum(tweets.media_type == x), \n",
    "                   \"%.10f%%\" % round(sum(tweets.media_type == x) / len(tweets) * 100, 10), \n",
    "                   \"%.2f\" % round(tweets.loc[tweets['media_type'] == x, 'total_engagement'].mean() * 1000, 2)]\n",
    "               for x in sorted(tweets.media_type.unique())]\n",
    "    \n",
    "    # Write results to a csv file\n",
    "    write_csv(['Type', 'Category', 'N', 'Percentage', 'Mean engagement ($x10^{-3}$)'],\n",
    "             [*cat_con, *cat_act, *media_type],\n",
    "             'results/avg_engagement_per_category.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "another-activity",
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_overall(x, y1, y2, interval, auth_label):\n",
    "    \"\"\" Plot the number of Tweets (bar) and the average engagement (line) per time interval in a single figure. \"\"\"\n",
    "    \n",
    "    # Reformat date\n",
    "    x = [fix_date(date) for date in x]\n",
    "    \n",
    "    fig, ax1 = plt.subplots(figsize = (6, 4))\n",
    "    ax1.bar(x, y1, width = 0.5, label='Number of Tweets')\n",
    "    ax2 = ax1.twinx()\n",
    "    \n",
    "    # Multiply engagement by 1000\n",
    "    y2 = [number*1000 for number in y2]\n",
    "    ax2.plot(x, y2, '-r')\n",
    "    ax1.plot(np.nan, '-r', label = 'Mean engagement')\n",
    "    \n",
    "    ax1.legend(loc=2)\n",
    "    \n",
    "    if interval == 'day':\n",
    "        plt.xticks(np.arange(0, len(x), 7.0))\n",
    "    \n",
    "    # Set labels and title\n",
    "    ax1.set_xlabel(interval.capitalize() + \" start date\")\n",
    "    ax1.set_ylabel('Number of Tweets')\n",
    "    ax2.set_ylabel('Mean engagement ($x10^{-3}$)')\n",
    "    ax2.set_ylim(ymin=0, ymax=max(y2)*1.3)\n",
    "    plt.title('Number of Tweets and mean engagement')\n",
    "    \n",
    "    plt.savefig('results/overall_%s_%s.png' % (interval, auth_label))\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "seven-actor",
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_categories(ylabel, title, x, y, labels, interval, filename, ymax):\n",
    "    \"\"\" Plot the occurrence of messaging strategies relative to the total number of Tweets posted in the time\n",
    "    interval. \"\"\"\n",
    "\n",
    "    fig, ax = plt.subplots(figsize = (6, 4))\n",
    "    \n",
    "    # Reformat dates and multiply ratio's to get percentages\n",
    "    x = [fix_date(date) for date in x]\n",
    "    y = [[j*100 for j in y[i]] for i in range(len(y))]\n",
    "    \n",
    "    # Multiply ratios in y by 100 to get a percentage\n",
    "    if len(labels) == 0:\n",
    "        ax.plot(x, y)\n",
    "    else:\n",
    "        for i, label in enumerate(labels):\n",
    "            ax.plot(x,[lst[i] for lst in y], label=label)\n",
    "    \n",
    "    if interval == 'day':\n",
    "        plt.xticks(np.arange(0, len(x), 7.0))\n",
    "    \n",
    "    # Set labels and title\n",
    "    ax.set_xlabel(interval.capitalize() + \" start date\")\n",
    "    ax.set_ylabel(ylabel)\n",
    "    ax.set_ylim(ymin=0, ymax=ymax)\n",
    "    plt.title(title)\n",
    "    \n",
    "    # Only generate legend if more than one labels\n",
    "    if len(labels) > 0:\n",
    "        ax.legend()\n",
    "    \n",
    "    plt.savefig('results/%s.png' % filename)\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "blind-solution",
   "metadata": {},
   "outputs": [],
   "source": [
    "def engagement_per_interval(tweets, all_users, interval, file_label, authors):\n",
    "    \"\"\" Plot the number of Tweets, mean engagement and use of content, activation and media type strategies\n",
    "        per day or per week for the given authors. \"\"\"  \n",
    "    \n",
    "    # Generate list of all authors if given author list is empty\n",
    "    if len(authors) == 0:\n",
    "        authors = all_users['handle'].tolist()\n",
    "    \n",
    "    # Copy selection of data frame\n",
    "    df = tweets[tweets['author_handle'].isin(authors)]\n",
    "    df = df[['created_at', 'author_handle', 'cat_con', 'cat_act', 'media_type', 'total_engagement']].copy()\n",
    "    \n",
    "    # Insert temporary column with creation date (without time)\n",
    "    df.insert(0, 'temp', pd.to_datetime(df.created_at.str[:10], format='%Y-%m-%d'))\n",
    "    \n",
    "    # Creation date with time can be removed\n",
    "    del df['created_at']\n",
    "    \n",
    "    # List and sort all days\n",
    "    dates = pd.date_range(pd.to_datetime('2021-02-04', format='%Y-%m-%d'), periods=42).tolist()\n",
    "    \n",
    "    if interval == 'week':\n",
    "        # If interval is week, only keep week starting dates (Thursdays)\n",
    "        dates = [date for i, date in enumerate(dates) if i % 7 == 0]  \n",
    "    \n",
    "    df.insert(0, 'date', \"\")\n",
    "    \n",
    "    # Insert week start date as column\n",
    "    for i, date in enumerate(dates):\n",
    "\n",
    "        if interval == 'week':\n",
    "            # Compare if date is equal to or larger than the checked start date, use that start date\n",
    "            df.loc[df.temp >= dates[i], 'date'] = date.date()\n",
    "        else:\n",
    "            df.loc[df.temp == dates[i], 'date'] = date.date()\n",
    "\n",
    "    # Delete temporary column\n",
    "    del df['temp']\n",
    "                \n",
    "    num_tweets = []\n",
    "    avg_engagement = []\n",
    "    cat_con = []\n",
    "    cat_act = []\n",
    "    media_type = []\n",
    "    \n",
    "    # Create copy of original dates without time\n",
    "    dates2 = [date.date() for date in dates]\n",
    "    \n",
    "    print('Mean engagement: %s' % (df['total_engagement'].mean() * 1000))\n",
    "    print('Number of Tweets: %s' % len(df))\n",
    "        \n",
    "    for i, date in enumerate(dates2):\n",
    "        # Make lists that will be plotted on y-axes\n",
    "        num_tweets.append(len(df[(df['date'] == date)]))\n",
    "        avg_engagement.append(df[(df['date'] == date)]['total_engagement'].mean())\n",
    "        cat_con.append([len(df[(df['date'] == date) & (df['cat_con'] == 'CAM')]) / num_tweets[i],\n",
    "                       len(df[(df['date'] == date) & (df['cat_con'] == 'POL')]) / num_tweets[i],\n",
    "                       len(df[(df['date'] == date) & (df['cat_con'] == 'SOC')]) / num_tweets[i]])\n",
    "        cat_act.append([len(df[(df['date'] == date) & (df['cat_act'] == 'CON')]) / num_tweets[i],\n",
    "                       len(df[(df['date'] == date) & (df['cat_act'] == 'FOL')]) / num_tweets[i],\n",
    "                       len(df[(df['date'] == date) & (df['cat_act'] == 'NONE')]) / num_tweets[i],\n",
    "                       len(df[(df['date'] == date) & (df['cat_act'] == 'SUP')]) / num_tweets[i]])\n",
    "        media_type.append([len(df[(df['date'] == date) & (df['media_type'] == 'animated_gif')]) / num_tweets[i],\n",
    "                       len(df[(df['date'] == date) & (df['media_type'] == 'none')]) / num_tweets[i],\n",
    "                       len(df[(df['date'] == date) & (df['media_type'] == 'photo')]) / num_tweets[i],\n",
    "                       len(df[(df['date'] == date) & (df['media_type'] == 'video')]) / num_tweets[i]])\n",
    "    \n",
    "    # Plot the lists\n",
    "    plot_overall(dates2, num_tweets, avg_engagement, interval, file_label)\n",
    "    plot_categories('Percentage of Tweets', 'Use of content type strategies', dates2, cat_con, \n",
    "               ['CAM', 'POL', 'SOC'], interval, 'cat_con_' + interval + '_' + file_label, 100)\n",
    "    plot_categories('Percentage of Tweets', 'Use of activation type strategies', dates2, cat_act, \n",
    "               ['CON', 'FOL', 'NONE', 'SUP'], interval, 'cat_act_' + interval + '_' + file_label, 100)\n",
    "    plot_categories('Percentage of Tweets', 'Use of media type strategies', dates2, media_type, \n",
    "               ['animated GIF', 'none', 'photo', 'video'], interval, 'media_type_' + interval + '_' + file_label, \n",
    "                    80)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "express-array",
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "tweets = read_json('source/tweets_all_labeled.txt')\n",
    "tweets_pd = pd.DataFrame(tweets)\n",
    "users_pd = pd.read_csv('source/users.csv')\n",
    "\n",
    "# Show the number of Tweets and average engagement per messaging strategy\n",
    "avg_engagement_per_category(tweets_pd)\n",
    "\n",
    "# Run for all parties\n",
    "engagement_per_interval(tweets_pd, users_pd, 'week', 'all', [])\n",
    "\n",
    "# Run for winning parties\n",
    "engagement_per_interval(tweets_pd, users_pd, 'week', 'winners', ['VoltNederland', 'DassenLaurens',\n",
    "                                                       'D66', 'SigridKaag',\n",
    "                                                       'fvdemocratie', 'thierrybaudet'])\n",
    "\n",
    "# Run for losing parties\n",
    "engagement_per_interval(tweets_pd, users_pd, 'week', 'losers', ['cdavandaag', 'WBHoekstra',\n",
    "                                                      'SPnl', 'MarijnissenL',\n",
    "                                                      'groenlinks', 'jesseklaver'])\n",
    "\n",
    "# Run for largest parties\n",
    "engagement_per_interval(tweets_pd, users_pd, 'week', 'largest', ['VVD', 'markrutte',\n",
    "                                                                 'D66', 'SigridKaag',\n",
    "                                                                 'geertwilderspvv'])\n",
    "\n",
    "# Run for smallest parties\n",
    "engagement_per_interval(tweets_pd, users_pd, 'week', 'smallest', ['PolitiekBIJ1', 'SylvanaBIJ1',\n",
    "                                                                  'BoerBurgerB', 'lientje1967',\n",
    "                                                                  '50pluspartij', 'LianedenHaan'])\n",
    "\n",
    "# # Run for parties\n",
    "# engagement_per_interval(tweets_pd, users_pd, 'week', 'parties', \n",
    "#                        users_pd.loc[users_pd['type'] == 'party', 'handle'].tolist())\n",
    "\n",
    "# # Run for party leaders\n",
    "# engagement_per_interval(tweets_pd, users_pd, 'week', 'leaders', \n",
    "#                        users_pd.loc[users_pd['type'] == 'person', 'handle'].tolist())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "latest-secondary",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}