BitinfoScraper - 41 Assets (full).ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "AlgoTrading - BitInfoChart Scraper",
      "provenance": [],
      "collapsed_sections": [
        "WPCbkMgDQ8vY",
        "uMnAGdCZPQK1",
        "45zzLEyIPdqx",
        "i3rmddoTRGBP"
      ]
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "L_wX4yZMQxa0"
      },
      "source": [
        "# User Settings"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EcLOU9WsQw-_"
      },
      "source": [
        "# What folder do you want to save the results in?\n",
        "# Individual CSV's for each coin saved with coin name as filename\n",
        "RESULTS_FOLDER = 'results'"
      ],
      "execution_count": 1,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "WPCbkMgDQ8vY"
      },
      "source": [
        "# Setup Notebook"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "F1eiojR90fXb",
        "outputId": "1ed31ae1-08d8-46ae-dac7-6ceff4f37f3e"
      },
      "source": [
        "!pip install fastcore\n",
        "!pip install aiohttp\n"
      ],
      "execution_count": 78,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Requirement already satisfied: fastcore in /usr/local/lib/python3.7/dist-packages (1.3.27)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.7/dist-packages (from fastcore) (21.2)\n",
            "Requirement already satisfied: pip in /usr/local/lib/python3.7/dist-packages (from fastcore) (21.1.3)\n",
            "Requirement already satisfied: pyparsing<3,>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging->fastcore) (2.4.7)\n",
            "Requirement already satisfied: aiohttp in /usr/local/lib/python3.7/dist-packages (3.8.1)\n",
            "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.7/dist-packages (from aiohttp) (1.2.0)\n",
            "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.7/dist-packages (from aiohttp) (5.2.0)\n",
            "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.7/dist-packages (from aiohttp) (21.2.0)\n",
            "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.7/dist-packages (from aiohttp) (1.7.2)\n",
            "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.7/dist-packages (from aiohttp) (1.2.0)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4 in /usr/local/lib/python3.7/dist-packages (from aiohttp) (3.10.0.2)\n",
            "Requirement already satisfied: charset-normalizer<3.0,>=2.0 in /usr/local/lib/python3.7/dist-packages (from aiohttp) (2.0.7)\n",
            "Requirement already satisfied: asynctest==0.13.0 in /usr/local/lib/python3.7/dist-packages (from aiohttp) (0.13.0)\n",
            "Requirement already satisfied: async-timeout<5.0,>=4.0.0a3 in /usr/local/lib/python3.7/dist-packages (from aiohttp) (4.0.1)\n",
            "Requirement already satisfied: idna>=2.0 in /usr/local/lib/python3.7/dist-packages (from yarl<2.0,>=1.0->aiohttp) (2.10)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "J-fVLY3YVc-e"
      },
      "source": [
        "from fastprogress import progress_bar\n",
        "import requests \n",
        "from bs4 import BeautifulSoup\n",
        "import pandas as pd\n",
        "import re\n",
        "import humanize\n",
        "from IPython.display import clear_output, display\n",
        "from pathlib import Path\n",
        "import numpy as np\n",
        "from fastcore.all import *\n",
        "\n",
        "import asyncio\n",
        "import aiohttp\n",
        "\n",
        "# this is to allow it to run in google colab environment\n",
        "import nest_asyncio\n",
        "nest_asyncio.apply()\n"
      ],
      "execution_count": 79,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DXcICVt5RM6B"
      },
      "source": [
        "Path(RESULTS_FOLDER).mkdir(exist_ok=True, parents=True)"
      ],
      "execution_count": 80,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "uMnAGdCZPQK1"
      },
      "source": [
        "#Helper Functions"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3VzQeEpqN7ow"
      },
      "source": [
        "def parse_strlist(sl):\n",
        "    clean = re.sub(\"[\\[\\],\\s]\",\"\",sl)\n",
        "    splitted = re.split(\"[\\'\\\"]\",clean)\n",
        "    values_only = [s for s in splitted if s != '']\n",
        "    return values_only\n",
        "\n",
        "def get_bitinfochart_graph_values(url, var_name):\n",
        "  response = requests.get(url)\n",
        "  soup = BeautifulSoup(response.text, 'html.parser')\n",
        "\n",
        "  scripts = soup.find_all('script')\n",
        "  for script in scripts:\n",
        "      if 'd = new Dygraph(document.getElementById(\"container\")' in script.text:\n",
        "          StrList = script.text\n",
        "          StrList = '[[' + StrList.split('[[')[-1]\n",
        "          StrList = StrList.split(']]')[0] +']]'\n",
        "          StrList = StrList.replace(\"new Date(\", '').replace(')','')\n",
        "          dataList = parse_strlist(StrList)\n",
        "\n",
        "  date = []\n",
        "  value = []\n",
        "  for each in dataList:\n",
        "      if (dataList.index(each) % 2) == 0:\n",
        "          date.append(each)\n",
        "      else:\n",
        "          value.append(each)\n",
        "\n",
        "  df = pd.DataFrame(list(zip(date, value)), columns=[\"date\",var_name])\n",
        "  return df\n"
      ],
      "execution_count": 81,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BVcqBoD0SRSD"
      },
      "source": [
        "# This function takes a list of df's then merge's them on the date field with an outer join\n",
        "\n",
        "def merge_dfs(df_list):\n",
        "  df_merged = None\n",
        "  for i in range(len(df_list)-1):\n",
        "    if i == 0:\n",
        "      df_merged = df_list[i].merge(df_list[i+1], on='date', how='outer')\n",
        "    else:\n",
        "      df_merged = df_merged.merge(df_list[i+1], on='date', how='outer')\n",
        "\n",
        "  return df_merged\n"
      ],
      "execution_count": 82,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "45zzLEyIPdqx"
      },
      "source": [
        "# Generate List of Pages to Scrape"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Vh6yCLGeOBcw"
      },
      "source": [
        "# Manually copy/pasted to use as a starting point\n",
        "\n",
        "chart_dict_list = [{'url': 'https://bitinfocharts.com/comparison/bitcoin-transactions.html', 'name': 'transactions'},\n",
        "                    {'url': 'https://bitinfocharts.com/comparison/size-btc.html', 'name': 'block_size'},\n",
        "                    {'url': 'https://bitinfocharts.com/comparison/sentbyaddress-btc.html', 'name': 'sent_addresses'},\n",
        "                    {'url': 'https://bitinfocharts.com/comparison/bitcoin-difficulty.html', 'name': 'difficulty'},\n",
        "                    {'url': 'https://bitinfocharts.com/comparison/bitcoin-hashrate.html', 'name': 'hashrate'},\n",
        "                    {'url': 'https://bitinfocharts.com/comparison/bitcoin-mining_profitability.html', 'name': 'mining_profitability'},\n",
        "                   \n",
        "                    {'url': 'https://bitinfocharts.com/comparison/sentinusd-btc.html', 'name': 'send_usd'},\n",
        "                    {'url': 'https://bitinfocharts.com/comparison/bitcoin-transactionfees.html', 'name': 'av_transaction_size'},\n",
        "                    {'url': 'https://bitinfocharts.com/comparison/bitcoin-median_transaction_fee.html', 'name': 'median_transaction_size'},\n",
        "                    {'url': 'https://bitinfocharts.com/comparison/bitcoin-confirmationtime.html', 'name': 'confirmation_time'},\n",
        "                    {'url': 'https://bitinfocharts.com/comparison/bitcoin-marketcap.html', 'name': 'market_cap'},\n",
        "                    {'url': 'https://bitinfocharts.com/comparison/transactionvalue-btc.html', 'name': 'av_transaction_value'},\n",
        "                   \n",
        "                    {'url': 'https://bitinfocharts.com/comparison/mediantransactionvalue-btc.html', 'name': 'median_transaction_value'},\n",
        "                    {'url': 'https://bitinfocharts.com/comparison/tweets-btc.html', 'name': 'tweets'},\n",
        "                    {'url': 'https://bitinfocharts.com/comparison/google_trends-btc.html', 'name': 'google_trends'},\n",
        "                    {'url': 'https://bitinfocharts.com/comparison/activeaddresses-btc.html', 'name': 'active_addresses'},\n",
        "                    {'url': 'https://bitinfocharts.com/comparison/top100cap-btc.html', 'name': 'top_100_percent'},\n",
        "                    {'url': 'https://bitinfocharts.com/comparison/fee_to_reward-btc.html', 'name': 'fee_reward'},\n",
        "\n",
        "                    ]"
      ],
      "execution_count": 83,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2KHKd7INWJj8",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "a36fa5a6-4e68-42f2-d620-44183352c7d1"
      },
      "source": [
        "# Get list of all available coins\n",
        "\n",
        "url = 'https://bitinfocharts.com'\n",
        "response = requests.get(url)\n",
        "soup = BeautifulSoup(response.text, 'html.parser')\n",
        "\n",
        "coin_dict_list = []\n",
        "\n",
        "\n",
        "for span in soup.find_all('span'):\n",
        "  \n",
        "  if 's_coins' in str(span.get('class')):\n",
        "    name = span.get('title').lower()\n",
        "    coin = span.get('data-coin')\n",
        "    coin_dict_list.append({'full_name': name,\n",
        "                'coin': coin})\n",
        "\n",
        "coin_dict_list[:3]"
      ],
      "execution_count": 84,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[{'coin': 'btc', 'full_name': 'bitcoin'},\n",
              " {'coin': 'eth', 'full_name': 'ethereum'},\n",
              " {'coin': 'xrp', 'full_name': 'xrp'}]"
            ]
          },
          "metadata": {},
          "execution_count": 84
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "CWB7Jr2M6mk_",
        "outputId": "67f51d0a-c256-4d8d-8123-b18991de17ee"
      },
      "source": [
        "# Combine the list of coins with the list of url's to create a master dictionary with coins and url's to scrape\n",
        "\n",
        "for coin_dict in coin_dict_list:\n",
        "  coin_dict['scrape_details'] = []\n",
        "  for chart_dict in chart_dict_list:\n",
        "    temp_dict = chart_dict.copy()\n",
        "    \n",
        "    url = temp_dict['url']\n",
        "    url = url.replace('bitcoin', coin_dict['full_name'])\n",
        "    url = url.replace('btc', coin_dict['coin'])\n",
        "    url = url.replace(' ', '%20')\n",
        "\n",
        "    temp_dict['url'] = url\n",
        "    coin_dict['scrape_details'].append(temp_dict)\n",
        "\n",
        "coin_dict_list[:2]"
      ],
      "execution_count": 85,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[{'coin': 'btc',\n",
              "  'full_name': 'bitcoin',\n",
              "  'scrape_details': [{'name': 'transactions',\n",
              "    'url': 'https://bitinfocharts.com/comparison/bitcoin-transactions.html'},\n",
              "   {'name': 'block_size',\n",
              "    'url': 'https://bitinfocharts.com/comparison/size-btc.html'},\n",
              "   {'name': 'sent_addresses',\n",
              "    'url': 'https://bitinfocharts.com/comparison/sentbyaddress-btc.html'},\n",
              "   {'name': 'difficulty',\n",
              "    'url': 'https://bitinfocharts.com/comparison/bitcoin-difficulty.html'},\n",
              "   {'name': 'hashrate',\n",
              "    'url': 'https://bitinfocharts.com/comparison/bitcoin-hashrate.html'},\n",
              "   {'name': 'mining_profitability',\n",
              "    'url': 'https://bitinfocharts.com/comparison/bitcoin-mining_profitability.html'},\n",
              "   {'name': 'send_usd',\n",
              "    'url': 'https://bitinfocharts.com/comparison/sentinusd-btc.html'},\n",
              "   {'name': 'av_transaction_size',\n",
              "    'url': 'https://bitinfocharts.com/comparison/bitcoin-transactionfees.html'},\n",
              "   {'name': 'median_transaction_size',\n",
              "    'url': 'https://bitinfocharts.com/comparison/bitcoin-median_transaction_fee.html'},\n",
              "   {'name': 'confirmation_time',\n",
              "    'url': 'https://bitinfocharts.com/comparison/bitcoin-confirmationtime.html'},\n",
              "   {'name': 'market_cap',\n",
              "    'url': 'https://bitinfocharts.com/comparison/bitcoin-marketcap.html'},\n",
              "   {'name': 'av_transaction_value',\n",
              "    'url': 'https://bitinfocharts.com/comparison/transactionvalue-btc.html'},\n",
              "   {'name': 'median_transaction_value',\n",
              "    'url': 'https://bitinfocharts.com/comparison/mediantransactionvalue-btc.html'},\n",
              "   {'name': 'tweets',\n",
              "    'url': 'https://bitinfocharts.com/comparison/tweets-btc.html'},\n",
              "   {'name': 'google_trends',\n",
              "    'url': 'https://bitinfocharts.com/comparison/google_trends-btc.html'},\n",
              "   {'name': 'active_addresses',\n",
              "    'url': 'https://bitinfocharts.com/comparison/activeaddresses-btc.html'},\n",
              "   {'name': 'top_100_percent',\n",
              "    'url': 'https://bitinfocharts.com/comparison/top100cap-btc.html'},\n",
              "   {'name': 'fee_reward',\n",
              "    'url': 'https://bitinfocharts.com/comparison/fee_to_reward-btc.html'}]},\n",
              " {'coin': 'eth',\n",
              "  'full_name': 'ethereum',\n",
              "  'scrape_details': [{'name': 'transactions',\n",
              "    'url': 'https://bitinfocharts.com/comparison/ethereum-transactions.html'},\n",
              "   {'name': 'block_size',\n",
              "    'url': 'https://bitinfocharts.com/comparison/size-eth.html'},\n",
              "   {'name': 'sent_addresses',\n",
              "    'url': 'https://bitinfocharts.com/comparison/sentbyaddress-eth.html'},\n",
              "   {'name': 'difficulty',\n",
              "    'url': 'https://bitinfocharts.com/comparison/ethereum-difficulty.html'},\n",
              "   {'name': 'hashrate',\n",
              "    'url': 'https://bitinfocharts.com/comparison/ethereum-hashrate.html'},\n",
              "   {'name': 'mining_profitability',\n",
              "    'url': 'https://bitinfocharts.com/comparison/ethereum-mining_profitability.html'},\n",
              "   {'name': 'send_usd',\n",
              "    'url': 'https://bitinfocharts.com/comparison/sentinusd-eth.html'},\n",
              "   {'name': 'av_transaction_size',\n",
              "    'url': 'https://bitinfocharts.com/comparison/ethereum-transactionfees.html'},\n",
              "   {'name': 'median_transaction_size',\n",
              "    'url': 'https://bitinfocharts.com/comparison/ethereum-median_transaction_fee.html'},\n",
              "   {'name': 'confirmation_time',\n",
              "    'url': 'https://bitinfocharts.com/comparison/ethereum-confirmationtime.html'},\n",
              "   {'name': 'market_cap',\n",
              "    'url': 'https://bitinfocharts.com/comparison/ethereum-marketcap.html'},\n",
              "   {'name': 'av_transaction_value',\n",
              "    'url': 'https://bitinfocharts.com/comparison/transactionvalue-eth.html'},\n",
              "   {'name': 'median_transaction_value',\n",
              "    'url': 'https://bitinfocharts.com/comparison/mediantransactionvalue-eth.html'},\n",
              "   {'name': 'tweets',\n",
              "    'url': 'https://bitinfocharts.com/comparison/tweets-eth.html'},\n",
              "   {'name': 'google_trends',\n",
              "    'url': 'https://bitinfocharts.com/comparison/google_trends-eth.html'},\n",
              "   {'name': 'active_addresses',\n",
              "    'url': 'https://bitinfocharts.com/comparison/activeaddresses-eth.html'},\n",
              "   {'name': 'top_100_percent',\n",
              "    'url': 'https://bitinfocharts.com/comparison/top100cap-eth.html'},\n",
              "   {'name': 'fee_reward',\n",
              "    'url': 'https://bitinfocharts.com/comparison/fee_to_reward-eth.html'}]}]"
            ]
          },
          "metadata": {},
          "execution_count": 85
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "CoVeCJkEcsj-"
      },
      "source": [
        "# Async Helpers"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vjGg_JxScxiY"
      },
      "source": [
        "async def fetch(session, url, full_name, coin, var_name):\n",
        "\n",
        "  try:\n",
        "    async with session.get(url) as resp:\n",
        "        return await resp.text(), full_name, coin, var_name\n",
        "  except:\n",
        "    return 'failed', full_name, coin, var_name\n",
        "        # Catch HTTP errors/exceptions here\n"
      ],
      "execution_count": 86,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_wweoRbEdYtz"
      },
      "source": [
        "async def fetch_concurrent(coin_dictionary):\n",
        "  page_results = []\n",
        "  loop = asyncio.get_event_loop()\n",
        "\n",
        "\n",
        "  async with aiohttp.ClientSession() as session:\n",
        "    tasks = []\n",
        "    for coin_dict in coin_dictionary:\n",
        "      for page in coin_dict['scrape_details']:\n",
        "\n",
        "        tasks.append(loop.create_task(fetch(session, \n",
        "                                            page['url'], \n",
        "                                            coin_dict['full_name'], \n",
        "                                            coin_dict['coin'],\n",
        "                                            page['name'])))\n",
        "\n",
        "    for result in asyncio.as_completed(tasks):\n",
        "      text, full_name, coin, var_name = await result\n",
        "      result_dict = {'response_text': text,\n",
        "                     'full_name': full_name,\n",
        "                     'coin': coin,\n",
        "                     'var_name': var_name}\n",
        "      page_results.append(result_dict)\n",
        "            \n",
        "      \n",
        "    return page_results\n",
        "\n"
      ],
      "execution_count": 87,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "L4d9BQ6FoOr5"
      },
      "source": [
        "def extract_info_from_response(response_dict):\n",
        "\n",
        "  response_text = response_dict['response_text']\n",
        "  var_name = response_dict['var_name']\n",
        "  \n",
        "  soup = BeautifulSoup(response_text, 'html.parser')\n",
        "\n",
        "  scripts = soup.find_all('script')\n",
        "  for script in scripts:\n",
        "      if 'd = new Dygraph(document.getElementById(\"container\")' in script.text:\n",
        "          StrList = script.text\n",
        "          StrList = '[[' + StrList.split('[[')[-1]\n",
        "          StrList = StrList.split(']]')[0] +']]'\n",
        "          StrList = StrList.replace(\"new Date(\", '').replace(')','')\n",
        "          dataList = parse_strlist(StrList)\n",
        "\n",
        "  date = []\n",
        "  value = []\n",
        "  for each in dataList:\n",
        "      if (dataList.index(each) % 2) == 0:\n",
        "          date.append(each)\n",
        "      else:\n",
        "          value.append(each)\n",
        "\n",
        "  df = pd.DataFrame(list(zip(date, value)), columns=[\"date\",var_name])\n",
        "\n",
        "\n",
        "  if df[var_name].str.isnumeric().any():\n",
        "    if df[var_name].str.contains('nullnull').any():\n",
        "      df[var_name] = np.nan\n",
        "\n",
        "    return df\n",
        "  \n",
        "  else:\n",
        "    return pd.DataFrame(columns=['date', var_name])\n"
      ],
      "execution_count": 88,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ZxMocesf2RLv"
      },
      "source": [
        "# Async Code Run\n",
        "\n",
        "- Takes about 13s for all the requests\n",
        "- then takes 6 minutes to convert everything to dataframes"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "oUU3XSUBjXyQ",
        "outputId": "1f115365-2a28-44b4-c875-3dcf5d9d9105"
      },
      "source": [
        "%%time\n",
        "http_responses = asyncio.run(fetch_concurrent(coin_dict_list))\n",
        "len(http_responses)"
      ],
      "execution_count": 89,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "CPU times: user 3.61 s, sys: 461 ms, total: 4.07 s\n",
            "Wall time: 7.45 s\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 37
        },
        "id": "jHeViMYdlh8w",
        "outputId": "228d53d1-7e36-45c5-e592-a76c7c1074b1"
      },
      "source": [
        "coin_merged_df_list = []\n",
        "\n",
        "for coin_dict in progress_bar(coin_dict_list):\n",
        "  print(f\"Now Processing - {coin_dict['full_name']}\")\n",
        "  coin_df_list = []\n",
        "  for response in progress_bar(http_responses):\n",
        "    if coin_dict['full_name'] == response['full_name']:\n",
        "      coin_df_list.append(extract_info_from_response(response))\n",
        "  \n",
        "  coin_df = merge_dfs(coin_df_list)\n",
        "  coin_df['full_name'] = coin_dict['full_name']\n",
        "  coin_df['coin'] = coin_dict['coin']\n",
        "  coin_merged_df_list.append(coin_df)\n",
        "\n",
        "  file_path = RESULTS_FOLDER + '/' +coin_dict['full_name'] + '__' + coin_dict['coin'] + '.csv'\n",
        "  coin_df.to_csv(file_path)\n",
        "\n",
        "  clear_output()\n",
        "\n",
        "      \n",
        "combined_df = pd.concat(coin_merged_df_list, ignore_index=True, sort=False)\n",
        "\n",
        "combined_df.to_csv('all_bitcoininfochart_data.csv')\n",
        "combined_df.to_pickle('all_bitcoininfochart_data.pkl')"
      ],
      "execution_count": 90,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "\n",
              "    <div>\n",
              "        <style>\n",
              "            /* Turns off some styling */\n",
              "            progress {\n",
              "                /* gets rid of default border in Firefox and Opera. */\n",
              "                border: none;\n",
              "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
              "                background-size: auto;\n",
              "            }\n",
              "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
              "                background: #F44336;\n",
              "            }\n",
              "        </style>\n",
              "      <progress value='42' class='' max='42' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      100.00% [42/42 04:46<00:00]\n",
              "    </div>\n",
              "    "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "hlnEiNdiuiRd",
        "outputId": "ba077ab6-cddb-4f9f-d485-28a29e4836be"
      },
      "source": [
        "print('Creating Zip File...')\n",
        "!zip -r -q results.zip {RESULTS_FOLDER}\n",
        "print('Complete!')"
      ],
      "execution_count": 91,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Creating Zip File...\n",
            "Complete!\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1UHTkjySCYYF"
      },
      "source": [
        "df = coin_merged_df_list[len(coin_merged_df_list)-1]"
      ],
      "execution_count": 92,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8x8ihTAxGKa7"
      },
      "source": [
        "df_columns = df.columns.to_list()"
      ],
      "execution_count": 93,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 669
        },
        "id": "867xWCtiGLUs",
        "outputId": "ae8dd317-f764-432c-aa9f-db7f4150b25d"
      },
      "source": [
        "results_summary_dict_list = []\n",
        "\n",
        "for df in coin_merged_df_list:\n",
        "  full_name = df.loc[0,'full_name']\n",
        "  coin = df.loc[0,'coin']\n",
        "\n",
        "  total_count = 0\n",
        "  df_columns = df.columns.to_list()\n",
        "\n",
        "  for column in df_columns:\n",
        "    if column != 'date':\n",
        "      try:\n",
        "        col_count = df[column].str.isnumeric().sum()\n",
        "        total_count = col_count + total_count\n",
        "      except:\n",
        "        pass\n",
        "  results = {'full_name': full_name,\n",
        "             'coin': coin,\n",
        "             'values': total_count}\n",
        "  results_summary_dict_list.append(results)\n",
        "\n",
        "df_summary = pd.DataFrame(results_summary_dict_list)\n",
        "display(df_summary.sort_values('values', ascending=False).head(20))\n",
        "df_summary.to_csv('summary.csv')"
      ],
      "execution_count": 94,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>full_name</th>\n",
              "      <th>coin</th>\n",
              "      <th>values</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>bitcoin</td>\n",
              "      <td>btc</td>\n",
              "      <td>41155</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>litecoin</td>\n",
              "      <td>ltc</td>\n",
              "      <td>32933</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>dash</td>\n",
              "      <td>dash</td>\n",
              "      <td>26359</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>dogecoin</td>\n",
              "      <td>doge</td>\n",
              "      <td>25446</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13</th>\n",
              "      <td>vertcoin</td>\n",
              "      <td>vtc</td>\n",
              "      <td>22331</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>15</th>\n",
              "      <td>blackcoin</td>\n",
              "      <td>blk</td>\n",
              "      <td>21536</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14</th>\n",
              "      <td>feathercoin</td>\n",
              "      <td>ftc</td>\n",
              "      <td>20691</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>ethereum</td>\n",
              "      <td>eth</td>\n",
              "      <td>18759</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>12</th>\n",
              "      <td>reddcoin</td>\n",
              "      <td>rdd</td>\n",
              "      <td>17992</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>10</th>\n",
              "      <td>ethereum classic</td>\n",
              "      <td>etc</td>\n",
              "      <td>17171</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>bitcoin sv</td>\n",
              "      <td>bsv</td>\n",
              "      <td>16885</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>bitcoin cash</td>\n",
              "      <td>bch</td>\n",
              "      <td>15609</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>monero</td>\n",
              "      <td>xmr</td>\n",
              "      <td>15412</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>zcash</td>\n",
              "      <td>zec</td>\n",
              "      <td>14580</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11</th>\n",
              "      <td>bitcoin gold</td>\n",
              "      <td>btg</td>\n",
              "      <td>14482</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>xrp</td>\n",
              "      <td>xrp</td>\n",
              "      <td>7615</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>19</th>\n",
              "      <td>syscoin</td>\n",
              "      <td>syscoin</td>\n",
              "      <td>4383</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>34</th>\n",
              "      <td>potcoin</td>\n",
              "      <td>potcoin</td>\n",
              "      <td>4294</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>22</th>\n",
              "      <td>digibyte</td>\n",
              "      <td>digibyte</td>\n",
              "      <td>2933</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>37</th>\n",
              "      <td>vericoin</td>\n",
              "      <td>vericoin</td>\n",
              "      <td>2508</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "           full_name      coin  values\n",
              "0            bitcoin       btc   41155\n",
              "3           litecoin       ltc   32933\n",
              "8               dash      dash   26359\n",
              "5           dogecoin      doge   25446\n",
              "13          vertcoin       vtc   22331\n",
              "15         blackcoin       blk   21536\n",
              "14       feathercoin       ftc   20691\n",
              "1           ethereum       eth   18759\n",
              "12          reddcoin       rdd   17992\n",
              "10  ethereum classic       etc   17171\n",
              "7         bitcoin sv       bsv   16885\n",
              "4       bitcoin cash       bch   15609\n",
              "6             monero       xmr   15412\n",
              "9              zcash       zec   14580\n",
              "11      bitcoin gold       btg   14482\n",
              "2                xrp       xrp    7615\n",
              "19           syscoin   syscoin    4383\n",
              "34           potcoin   potcoin    4294\n",
              "22          digibyte  digibyte    2933\n",
              "37          vericoin  vericoin    2508"
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JqEERHgdHaeY"
      },
      "source": [
        ""
      ],
      "execution_count": 33,
      "outputs": []
    }
  ]
}