BitinfoScraper - 701 Assets (Twitter & GTrend).ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "AlgoTrading - BitInfoChart Scraper",
      "provenance": [],
      "collapsed_sections": [
        "WPCbkMgDQ8vY",
        "uMnAGdCZPQK1",
        "45zzLEyIPdqx",
        "i3rmddoTRGBP"
      ]
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "L_wX4yZMQxa0"
      },
      "source": [
        "# User Settings"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EcLOU9WsQw-_"
      },
      "source": [
        "# What folder do you want to save the results in?\n",
        "# Individual CSV's for each coin saved with coin name as filename\n",
        "RESULTS_FOLDER = 'results'"
      ],
      "execution_count": 1,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "WPCbkMgDQ8vY"
      },
      "source": [
        "# Setup Notebook"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "F1eiojR90fXb",
        "outputId": "2240d802-c78c-4e89-f878-e447f1f2721e"
      },
      "source": [
        "!pip install fastcore\n",
        "!pip install aiohttp\n"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting fastcore\n",
            "  Downloading fastcore-1.3.27-py3-none-any.whl (56 kB)\n",
            "\u001b[?25l\r\u001b[K     |█████▊                          | 10 kB 17.2 MB/s eta 0:00:01\r\u001b[K     |███████████▌                    | 20 kB 20.0 MB/s eta 0:00:01\r\u001b[K     |█████████████████▎              | 30 kB 12.1 MB/s eta 0:00:01\r\u001b[K     |███████████████████████         | 40 kB 7.4 MB/s eta 0:00:01\r\u001b[K     |████████████████████████████▉   | 51 kB 4.9 MB/s eta 0:00:01\r\u001b[K     |████████████████████████████████| 56 kB 2.4 MB/s \n",
            "\u001b[?25hRequirement already satisfied: pip in /usr/local/lib/python3.7/dist-packages (from fastcore) (21.1.3)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.7/dist-packages (from fastcore) (21.2)\n",
            "Requirement already satisfied: pyparsing<3,>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging->fastcore) (2.4.7)\n",
            "Installing collected packages: fastcore\n",
            "Successfully installed fastcore-1.3.27\n",
            "Collecting aiohttp\n",
            "  Downloading aiohttp-3.8.1-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (1.1 MB)\n",
            "\u001b[K     |████████████████████████████████| 1.1 MB 5.0 MB/s \n",
            "\u001b[?25hCollecting multidict<7.0,>=4.5\n",
            "  Downloading multidict-5.2.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (160 kB)\n",
            "\u001b[K     |████████████████████████████████| 160 kB 53.9 MB/s \n",
            "\u001b[?25hCollecting yarl<2.0,>=1.0\n",
            "  Downloading yarl-1.7.2-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (271 kB)\n",
            "\u001b[K     |████████████████████████████████| 271 kB 55.7 MB/s \n",
            "\u001b[?25hCollecting aiosignal>=1.1.2\n",
            "  Downloading aiosignal-1.2.0-py3-none-any.whl (8.2 kB)\n",
            "Collecting async-timeout<5.0,>=4.0.0a3\n",
            "  Downloading async_timeout-4.0.1-py3-none-any.whl (5.7 kB)\n",
            "Collecting frozenlist>=1.1.1\n",
            "  Downloading frozenlist-1.2.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (192 kB)\n",
            "\u001b[K     |████████████████████████████████| 192 kB 66.9 MB/s \n",
            "\u001b[?25hRequirement already satisfied: typing-extensions>=3.7.4 in /usr/local/lib/python3.7/dist-packages (from aiohttp) (3.10.0.2)\n",
            "Collecting asynctest==0.13.0\n",
            "  Downloading asynctest-0.13.0-py3-none-any.whl (26 kB)\n",
            "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.7/dist-packages (from aiohttp) (21.2.0)\n",
            "Requirement already satisfied: charset-normalizer<3.0,>=2.0 in /usr/local/lib/python3.7/dist-packages (from aiohttp) (2.0.7)\n",
            "Requirement already satisfied: idna>=2.0 in /usr/local/lib/python3.7/dist-packages (from yarl<2.0,>=1.0->aiohttp) (2.10)\n",
            "Installing collected packages: multidict, frozenlist, yarl, asynctest, async-timeout, aiosignal, aiohttp\n",
            "Successfully installed aiohttp-3.8.1 aiosignal-1.2.0 async-timeout-4.0.1 asynctest-0.13.0 frozenlist-1.2.0 multidict-5.2.0 yarl-1.7.2\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "J-fVLY3YVc-e"
      },
      "source": [
        "from fastprogress import progress_bar\n",
        "import requests \n",
        "from bs4 import BeautifulSoup\n",
        "import pandas as pd\n",
        "import re\n",
        "import humanize\n",
        "from IPython.display import clear_output, display\n",
        "from pathlib import Path\n",
        "import numpy as np\n",
        "from fastcore.all import *\n",
        "\n",
        "import asyncio\n",
        "import aiohttp\n",
        "\n",
        "# this is to allow it to run in google colab environment\n",
        "import nest_asyncio\n",
        "nest_asyncio.apply()\n"
      ],
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DXcICVt5RM6B"
      },
      "source": [
        "Path(RESULTS_FOLDER).mkdir(exist_ok=True, parents=True)"
      ],
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "uMnAGdCZPQK1"
      },
      "source": [
        "#Helper Functions"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3VzQeEpqN7ow"
      },
      "source": [
        "def parse_strlist(sl):\n",
        "    clean = re.sub(\"[\\[\\],\\s]\",\"\",sl)\n",
        "    splitted = re.split(\"[\\'\\\"]\",clean)\n",
        "    values_only = [s for s in splitted if s != '']\n",
        "    return values_only\n",
        "\n",
        "def get_bitinfochart_graph_values(url, var_name):\n",
        "  response = requests.get(url)\n",
        "  soup = BeautifulSoup(response.text, 'html.parser')\n",
        "\n",
        "  scripts = soup.find_all('script')\n",
        "  for script in scripts:\n",
        "      if 'd = new Dygraph(document.getElementById(\"container\")' in script.text:\n",
        "          StrList = script.text\n",
        "          StrList = '[[' + StrList.split('[[')[-1]\n",
        "          StrList = StrList.split(']]')[0] +']]'\n",
        "          StrList = StrList.replace(\"new Date(\", '').replace(')','')\n",
        "          dataList = parse_strlist(StrList)\n",
        "\n",
        "  date = []\n",
        "  value = []\n",
        "  for each in dataList:\n",
        "      if (dataList.index(each) % 2) == 0:\n",
        "          date.append(each)\n",
        "      else:\n",
        "          value.append(each)\n",
        "\n",
        "  df = pd.DataFrame(list(zip(date, value)), columns=[\"date\",var_name])\n",
        "  return df\n"
      ],
      "execution_count": 54,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BVcqBoD0SRSD"
      },
      "source": [
        "# This function takes a list of df's then merge's them on the date field with an outer join\n",
        "\n",
        "def merge_dfs(df_list):\n",
        "  df_merged = None\n",
        "  for i in range(len(df_list)-1):\n",
        "    if i == 0:\n",
        "      df_merged = df_list[i].merge(df_list[i+1], on='date', how='outer')\n",
        "    else:\n",
        "      df_merged = df_merged.merge(df_list[i+1], on='date', how='outer')\n",
        "\n",
        "  return df_merged\n"
      ],
      "execution_count": 55,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "45zzLEyIPdqx"
      },
      "source": [
        "# Generate List of Pages to Scrape"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ytI06cncID7i"
      },
      "source": [
        "# Manually copy/pasted to use as a starting point\n",
        "\n",
        "chart_dict_list = [\n",
        "                    {'url': 'https://bitinfocharts.com/comparison/tweets-btc.html', 'name': 'tweets'},\n",
        "                    {'url': 'https://bitinfocharts.com/comparison/google_trends-btc.html', 'name': 'google_trends'},\n",
        "\n",
        "                    ]"
      ],
      "execution_count": 59,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2KHKd7INWJj8",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "4178a6b7-e44d-450d-ea48-185b54dc30a0"
      },
      "source": [
        "# Get list of all available coins\n",
        "\n",
        "url = 'https://bitinfocharts.com/comparison/tweets.html'\n",
        "response = requests.get(url)\n",
        "soup = BeautifulSoup(response.text, 'html.parser')\n",
        "\n",
        "coin_dict_list = []\n",
        "\n",
        "\n",
        "for span in soup.find_all('span'):\n",
        "  \n",
        "  if 's_coins' in str(span.get('class')):\n",
        "    name = span.get('title').lower()\n",
        "    coin = span.get('data-coin')\n",
        "    coin_dict_list.append({'full_name': name,\n",
        "                'coin': coin})\n",
        "\n",
        "coin_dict_list[:3]"
      ],
      "execution_count": 60,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[{'coin': 'btc', 'full_name': 'bitcoin'},\n",
              " {'coin': 'eth', 'full_name': 'ethereum'},\n",
              " {'coin': 'busd', 'full_name': 'busd'}]"
            ]
          },
          "metadata": {},
          "execution_count": 60
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "CWB7Jr2M6mk_",
        "outputId": "ee918af9-ece5-40ec-f943-c2feed6e543c"
      },
      "source": [
        "# Combine the list of coins with the list of url's to create a master dictionary with coins and url's to scrape\n",
        "\n",
        "for coin_dict in coin_dict_list:\n",
        "  coin_dict['scrape_details'] = []\n",
        "  for chart_dict in chart_dict_list:\n",
        "    temp_dict = chart_dict.copy()\n",
        "    \n",
        "    url = temp_dict['url']\n",
        "    url = url.replace('bitcoin', coin_dict['full_name'])\n",
        "    url = url.replace('btc', coin_dict['coin'])\n",
        "    url = url.replace(' ', '%20')\n",
        "\n",
        "    temp_dict['url'] = url\n",
        "    coin_dict['scrape_details'].append(temp_dict)\n",
        "\n",
        "coin_dict_list[:2]"
      ],
      "execution_count": 70,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[{'coin': 'btc',\n",
              "  'full_name': 'bitcoin',\n",
              "  'scrape_details': [{'name': 'tweets',\n",
              "    'url': 'https://bitinfocharts.com/comparison/tweets-btc.html'},\n",
              "   {'name': 'google_trends',\n",
              "    'url': 'https://bitinfocharts.com/comparison/google_trends-btc.html'}]},\n",
              " {'coin': 'eth',\n",
              "  'full_name': 'ethereum',\n",
              "  'scrape_details': [{'name': 'tweets',\n",
              "    'url': 'https://bitinfocharts.com/comparison/tweets-eth.html'},\n",
              "   {'name': 'google_trends',\n",
              "    'url': 'https://bitinfocharts.com/comparison/google_trends-eth.html'}]}]"
            ]
          },
          "metadata": {},
          "execution_count": 70
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "CoVeCJkEcsj-"
      },
      "source": [
        "# Async Helpers"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vjGg_JxScxiY"
      },
      "source": [
        "async def fetch(session, url, full_name, coin, var_name):\n",
        "\n",
        "  try:\n",
        "    async with session.get(url) as resp:\n",
        "        return await resp.text(), full_name, coin, var_name\n",
        "  except:\n",
        "    return 'failed', full_name, coin, var_name\n",
        "        # Catch HTTP errors/exceptions here\n"
      ],
      "execution_count": 63,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_wweoRbEdYtz"
      },
      "source": [
        "async def fetch_concurrent(coin_dictionary):\n",
        "  page_results = []\n",
        "  loop = asyncio.get_event_loop()\n",
        "\n",
        "\n",
        "  async with aiohttp.ClientSession() as session:\n",
        "    tasks = []\n",
        "    for coin_dict in coin_dictionary:\n",
        "      for page in coin_dict['scrape_details']:\n",
        "\n",
        "        tasks.append(loop.create_task(fetch(session, \n",
        "                                            page['url'], \n",
        "                                            coin_dict['full_name'], \n",
        "                                            coin_dict['coin'],\n",
        "                                            page['name'])))\n",
        "\n",
        "    for result in asyncio.as_completed(tasks):\n",
        "      text, full_name, coin, var_name = await result\n",
        "      result_dict = {'response_text': text,\n",
        "                     'full_name': full_name,\n",
        "                     'coin': coin,\n",
        "                     'var_name': var_name}\n",
        "      page_results.append(result_dict)\n",
        "            \n",
        "      \n",
        "    return page_results\n",
        "\n"
      ],
      "execution_count": 64,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "L4d9BQ6FoOr5"
      },
      "source": [
        "def extract_info_from_response(response_dict):\n",
        "\n",
        "  response_text = response_dict['response_text']\n",
        "  var_name = response_dict['var_name']\n",
        "  \n",
        "  soup = BeautifulSoup(response_text, 'html.parser')\n",
        "\n",
        "  scripts = soup.find_all('script')\n",
        "  for script in scripts:\n",
        "      if 'd = new Dygraph(document.getElementById(\"container\")' in script.text:\n",
        "          StrList = script.text\n",
        "          StrList = '[[' + StrList.split('[[')[-1]\n",
        "          StrList = StrList.split(']]')[0] +']]'\n",
        "          StrList = StrList.replace(\"new Date(\", '').replace(')','')\n",
        "          dataList = parse_strlist(StrList)\n",
        "\n",
        "  date = []\n",
        "  value = []\n",
        "  for each in dataList:\n",
        "      if (dataList.index(each) % 2) == 0:\n",
        "          date.append(each)\n",
        "      else:\n",
        "          value.append(each)\n",
        "\n",
        "  df = pd.DataFrame(list(zip(date, value)), columns=[\"date\",var_name])\n",
        "\n",
        "\n",
        "  if df[var_name].str.isnumeric().any():\n",
        "    if df[var_name].str.contains('nullnull').any():\n",
        "      df[var_name] = np.nan\n",
        "\n",
        "    return df\n",
        "  \n",
        "  else:\n",
        "    return pd.DataFrame(columns=['date', var_name])\n"
      ],
      "execution_count": 65,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ZxMocesf2RLv"
      },
      "source": [
        "# Async Code Run\n",
        "\n",
        "- Takes about 13s for all the requests\n",
        "- then takes 6 minutes to convert everything to dataframes"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "oUU3XSUBjXyQ",
        "outputId": "6142b13b-2c66-449e-d4c2-5363b957be51"
      },
      "source": [
        "%%time\n",
        "http_responses = asyncio.run(fetch_concurrent(coin_dict_list))\n",
        "len(http_responses)"
      ],
      "execution_count": 66,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "CPU times: user 5.16 s, sys: 589 ms, total: 5.74 s\n",
            "Wall time: 11.6 s\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 37
        },
        "id": "jHeViMYdlh8w",
        "outputId": "6d6ec976-c967-46a5-ba9c-7a5d8eccac14"
      },
      "source": [
        "coin_merged_df_list = []\n",
        "\n",
        "for coin_dict in progress_bar(coin_dict_list):\n",
        "  print(f\"Now Processing - {coin_dict['full_name']}\")\n",
        "  coin_df_list = []\n",
        "  for response in progress_bar(http_responses):\n",
        "    if coin_dict['full_name'] == response['full_name']:\n",
        "      coin_df_list.append(extract_info_from_response(response))\n",
        "  \n",
        "  coin_df = merge_dfs(coin_df_list)\n",
        "  coin_df['full_name'] = coin_dict['full_name']\n",
        "  coin_df['coin'] = coin_dict['coin']\n",
        "  coin_merged_df_list.append(coin_df)\n",
        "\n",
        "  file_path = RESULTS_FOLDER + '/' +coin_dict['full_name'] + '__' + coin_dict['coin'] + '.csv'\n",
        "  coin_df.to_csv(file_path)\n",
        "\n",
        "  clear_output()\n",
        "\n",
        "      \n",
        "combined_df = pd.concat(coin_merged_df_list, ignore_index=True, sort=False)\n",
        "\n",
        "combined_df.to_csv('all_bitcoininfochart_data.csv')\n",
        "combined_df.to_pickle('all_bitcoininfochart_data.pkl')"
      ],
      "execution_count": 71,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "\n",
              "    <div>\n",
              "        <style>\n",
              "            /* Turns off some styling */\n",
              "            progress {\n",
              "                /* gets rid of default border in Firefox and Opera. */\n",
              "                border: none;\n",
              "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
              "                background-size: auto;\n",
              "            }\n",
              "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
              "                background: #F44336;\n",
              "            }\n",
              "        </style>\n",
              "      <progress value='701' class='' max='701' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      100.00% [701/701 02:04<00:00]\n",
              "    </div>\n",
              "    "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "hlnEiNdiuiRd",
        "outputId": "ab60e4ce-3974-42f5-854c-611a675e12ea"
      },
      "source": [
        "print('Creating Zip File...')\n",
        "!zip -r -q results.zip {RESULTS_FOLDER}\n",
        "print('Complete!')"
      ],
      "execution_count": 72,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Creating Zip File...\n",
            "Complete!\n"
          ]
        }
      ]
    }
  ]
}