TaskNumberOne.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1544,
   "id": "arabic-assault",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import requests\n",
    "import urllib\n",
    "from bs4 import BeautifulSoup\n",
    "from datetime import datetime\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "municipal-chocolate",
   "metadata": {},
   "source": [
    "***\n",
    "#### I.  Part\n",
    "Определите наиболее удобный способ программно получать список поданных за день форм,\n",
    "автоматизируйте процесс загрузки данных и подготовки их для анализа.\n",
    "***\n",
    "Answer: An EDGAR database store daily filled filings in location /daily-index/, which will publish at the end of the day between 10:00 - 11:00PM. So we\"ll parse link below with dynamic link and get that date what we need.\n",
    "As an example I get and prepared Balance Sheet from quaterly report of HERMAN MILLER "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1545,
   "id": "vanilla-spare",
   "metadata": {},
   "outputs": [],
   "source": [
    "# year and quater\n",
    "year='2021'\n",
    "quater='1'\n",
    "\n",
    "# date format: yymmdd or '20210430'\n",
    "date='20210104'\n",
    "\n",
    "# it returns yymmdd format\n",
    "# date = str(datetime.now().date()).replace('-','') \n",
    "\n",
    "# it returns us link to filings\n",
    "def get_url(year, quater, date):\n",
    "    url ='https://www.sec.gov/Archives/edgar/daily-index/%s/QTR%s/master.%s.idx' % (year,quater,date)\n",
    "\n",
    "    return url"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1704,
   "id": "exclusive-mathematics",
   "metadata": {},
   "outputs": [],
   "source": [
    "# define a url, in this case I'll just pass the get_url function\n",
    "file_url = get_url(year,quater,date)\n",
    "\n",
    "# request that new content, this will not be a JSON STRUCTURE!\n",
    "content = requests.get(file_url).content\n",
    "\n",
    "# we're writing the content to a file, so we don't need to request it again.\n",
    "with open('master_20210104.txt', 'wb') as f:\n",
    "     f.write(content)\n",
    "        \n",
    "# opening dowloaded file\n",
    "with open('master_20210104.txt','rb') as f:\n",
    "     byte_data = f.read()\n",
    "\n",
    "# we have a byte stream that needs to be decoded and then split by double spaces.\n",
    "data = byte_data.decode(\"utf-8\").split('  ')\n",
    "\n",
    "# should remove the headers, so look for the end of the header and grab it's index\n",
    "for index, item in enumerate(data):\n",
    "    if \"ftp://ftp.sec.gov/edgar/\" in item:\n",
    "        start_ind = index\n",
    "\n",
    "# define a new dataset without the header info.\n",
    "data_format = data[start_ind + 1:]\n",
    "\n",
    "master_data = []\n",
    "\n",
    "for index, item in enumerate(data_format):\n",
    "    \n",
    "    if index == 0:\n",
    "        clean_item_data = item.replace('\\n','|').split('|')\n",
    "        clean_item_data = clean_item_data[8:]\n",
    "    else:\n",
    "        clean_item_data = item.replace('\\n','|').split('|')\n",
    "        \n",
    "    for index, row in enumerate(clean_item_data):\n",
    "        \n",
    "        if '.txt' in row:\n",
    "            mini_list = clean_item_data[(index - 4): index + 1]\n",
    "            \n",
    "            if len(mini_list) != 0:\n",
    "                mini_list[4] = \"https://www.sec.gov/Archives/\" + mini_list[4]\n",
    "                master_data.append(mini_list) \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1705,
   "id": "consecutive-breathing",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>CIK</th>\n",
       "      <th>Company Name</th>\n",
       "      <th>FormType</th>\n",
       "      <th>Date</th>\n",
       "      <th>Source</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1232579</td>\n",
       "      <td>SYRVALIN KRISTINE C</td>\n",
       "      <td>3</td>\n",
       "      <td>2021-01-04</td>\n",
       "      <td>https://www.sec.gov/Archives/edgar/data/123257...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1232582</td>\n",
       "      <td>ASHFORD HOSPITALITY TRUST INC</td>\n",
       "      <td>424B3</td>\n",
       "      <td>2021-01-04</td>\n",
       "      <td>https://www.sec.gov/Archives/edgar/data/123258...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1232582</td>\n",
       "      <td>ASHFORD HOSPITALITY TRUST INC</td>\n",
       "      <td>8-K</td>\n",
       "      <td>2021-01-04</td>\n",
       "      <td>https://www.sec.gov/Archives/edgar/data/123258...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1233152</td>\n",
       "      <td>CALHOUN DAVID L</td>\n",
       "      <td>4</td>\n",
       "      <td>2021-01-04</td>\n",
       "      <td>https://www.sec.gov/Archives/edgar/data/123315...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1233167</td>\n",
       "      <td>NEAL MICHAEL A</td>\n",
       "      <td>4</td>\n",
       "      <td>2021-01-04</td>\n",
       "      <td>https://www.sec.gov/Archives/edgar/data/123316...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4484</th>\n",
       "      <td>9631</td>\n",
       "      <td>BANK OF NOVA SCOTIA</td>\n",
       "      <td>FWP</td>\n",
       "      <td>2021-01-04</td>\n",
       "      <td>https://www.sec.gov/Archives/edgar/data/9631/0...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4485</th>\n",
       "      <td>9631</td>\n",
       "      <td>BANK OF NOVA SCOTIA</td>\n",
       "      <td>FWP</td>\n",
       "      <td>2021-01-04</td>\n",
       "      <td>https://www.sec.gov/Archives/edgar/data/9631/0...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4486</th>\n",
       "      <td>97216</td>\n",
       "      <td>TEREX CORP</td>\n",
       "      <td>3</td>\n",
       "      <td>2021-01-04</td>\n",
       "      <td>https://www.sec.gov/Archives/edgar/data/97216/...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4487</th>\n",
       "      <td>97517</td>\n",
       "      <td>TEXAS PACIFIC LAND TRUST</td>\n",
       "      <td>4</td>\n",
       "      <td>2021-01-04</td>\n",
       "      <td>https://www.sec.gov/Archives/edgar/data/97517/...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4488</th>\n",
       "      <td>99302</td>\n",
       "      <td>TRANSCAT INC</td>\n",
       "      <td>4</td>\n",
       "      <td>2021-01-04</td>\n",
       "      <td>https://www.sec.gov/Archives/edgar/data/99302/...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>4489 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          CIK                   Company Name FormType       Date  \\\n",
       "0     1232579            SYRVALIN KRISTINE C        3 2021-01-04   \n",
       "1     1232582  ASHFORD HOSPITALITY TRUST INC    424B3 2021-01-04   \n",
       "2     1232582  ASHFORD HOSPITALITY TRUST INC      8-K 2021-01-04   \n",
       "3     1233152                CALHOUN DAVID L        4 2021-01-04   \n",
       "4     1233167                 NEAL MICHAEL A        4 2021-01-04   \n",
       "...       ...                            ...      ...        ...   \n",
       "4484     9631            BANK OF NOVA SCOTIA      FWP 2021-01-04   \n",
       "4485     9631            BANK OF NOVA SCOTIA      FWP 2021-01-04   \n",
       "4486    97216                     TEREX CORP        3 2021-01-04   \n",
       "4487    97517       TEXAS PACIFIC LAND TRUST        4 2021-01-04   \n",
       "4488    99302                   TRANSCAT INC        4 2021-01-04   \n",
       "\n",
       "                                                 Source  \n",
       "0     https://www.sec.gov/Archives/edgar/data/123257...  \n",
       "1     https://www.sec.gov/Archives/edgar/data/123258...  \n",
       "2     https://www.sec.gov/Archives/edgar/data/123258...  \n",
       "3     https://www.sec.gov/Archives/edgar/data/123315...  \n",
       "4     https://www.sec.gov/Archives/edgar/data/123316...  \n",
       "...                                                 ...  \n",
       "4484  https://www.sec.gov/Archives/edgar/data/9631/0...  \n",
       "4485  https://www.sec.gov/Archives/edgar/data/9631/0...  \n",
       "4486  https://www.sec.gov/Archives/edgar/data/97216/...  \n",
       "4487  https://www.sec.gov/Archives/edgar/data/97517/...  \n",
       "4488  https://www.sec.gov/Archives/edgar/data/99302/...  \n",
       "\n",
       "[4489 rows x 5 columns]"
      ]
     },
     "execution_count": 1705,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# our data collecting into DataFrame\n",
    "df = pd.DataFrame(master_data, columns=['CIK','Company Name', 'FormType','Date', 'Source'])\n",
    "\n",
    "# Date column converting into datetime format\n",
    "df['Date']=df['Date'].apply(pd.to_datetime, format='%Y-%m-%d')\n",
    "\n",
    "# replacing .txt format to html\n",
    "df['Source'] = df['Source'].apply(lambda x: x.replace(\".txt\", \"-index.html\"))\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1706,
   "id": "weekly-pathology",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>CIK</th>\n",
       "      <th>Company Name</th>\n",
       "      <th>FormType</th>\n",
       "      <th>Date</th>\n",
       "      <th>Source</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>3629</th>\n",
       "      <td>66382</td>\n",
       "      <td>HERMAN MILLER INC</td>\n",
       "      <td>10-Q</td>\n",
       "      <td>2021-01-04</td>\n",
       "      <td>https://www.sec.gov/Archives/edgar/data/66382/...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        CIK       Company Name FormType       Date  \\\n",
       "3629  66382  HERMAN MILLER INC     10-Q 2021-01-04   \n",
       "\n",
       "                                                 Source  \n",
       "3629  https://www.sec.gov/Archives/edgar/data/66382/...  "
      ]
     },
     "execution_count": 1706,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# giving to function CIK number of Company\n",
    "# we will see type of Forms and source index which is first column\n",
    "def get_source(CIK):\n",
    "    return df[df['CIK']==CIK]\n",
    "\n",
    "get_source('66382')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1707,
   "id": "attractive-import",
   "metadata": {},
   "outputs": [],
   "source": [
    "# put the source index and form type from table above\n",
    "# this function returns source to report\n",
    "\n",
    "def report_source(source_index, form_type):\n",
    "    #reading source and filling NaN value as an empty \n",
    "    document_index = pd.read_html(df['Source'][source_index])\n",
    "    document_index = document_index[0].fillna('')\n",
    "    \n",
    "    # here we're getting access to each report\n",
    "    document_name = document_index[document_index['Type'].str.contains(form_type)]\n",
    "    document_name = document_name['Document'].str.split(' ')\n",
    "    document_name = document_name[0][0]\n",
    "    \n",
    "    report_f = df['Source'][source_index].replace('-','').replace('index.html','')\n",
    "    \n",
    "    return report_f + '/' + document_name"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1708,
   "id": "western-present",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "https://www.sec.gov/Archives/edgar/data/66382/000006638221000003/mlhr-20201128.htm\n"
     ]
    }
   ],
   "source": [
    "print(report_source(3629, '10-Q'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1709,
   "id": "directed-upgrade",
   "metadata": {},
   "outputs": [],
   "source": [
    "# This exaple of preparing 10-Q report\n",
    "# reading function returns source\n",
    "# it might right here some 403 forbidden problem should run several times\n",
    "\n",
    "url_fin = report_source(3629, '10-Q')\n",
    "balance_sheet = pd.read_html(url_fin)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1710,
   "id": "stock-cincinnati",
   "metadata": {},
   "outputs": [],
   "source": [
    "# finding balancesheet\n",
    "balance_sheet = balance_sheet[7][[0,4,10]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1711,
   "id": "sunrise-cincinnati",
   "metadata": {},
   "outputs": [],
   "source": [
    "#deleting unnecessary data\n",
    "balance_sheet.columns=['Accounts','28/11/2020','30/05/2020']\n",
    "balance_sheet = balance_sheet.dropna()\n",
    "balance_sheet = balance_sheet.drop([2,23])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1712,
   "id": "minute-preserve",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Accounts</th>\n",
       "      <th>28/11/2020</th>\n",
       "      <th>30/05/2020</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Cash and cash equivalents</td>\n",
       "      <td>377.9</td>\n",
       "      <td>454.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Short-term investments</td>\n",
       "      <td>7.2</td>\n",
       "      <td>7.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Accounts receivable, net of allowances of $5.2...</td>\n",
       "      <td>193.0</td>\n",
       "      <td>180.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Unbilled accounts receivable</td>\n",
       "      <td>33.0</td>\n",
       "      <td>19.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Inventories, net</td>\n",
       "      <td>191.0</td>\n",
       "      <td>197.3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Prepaid expenses</td>\n",
       "      <td>26.8</td>\n",
       "      <td>43.3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Other current assets</td>\n",
       "      <td>9.0</td>\n",
       "      <td>16.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Total current assets</td>\n",
       "      <td>837.9</td>\n",
       "      <td>917.1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Property and equipment, at cost</td>\n",
       "      <td>1124.0</td>\n",
       "      <td>1111.3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>Less — accumulated depreciation</td>\n",
       "      <td>(798.1)</td>\n",
       "      <td>(780.5)</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                            Accounts 28/11/2020 30/05/2020\n",
       "0                          Cash and cash equivalents      377.9      454.0\n",
       "1                             Short-term investments        7.2        7.0\n",
       "2  Accounts receivable, net of allowances of $5.2...      193.0      180.0\n",
       "3                       Unbilled accounts receivable       33.0       19.5\n",
       "4                                   Inventories, net      191.0      197.3\n",
       "5                                   Prepaid expenses       26.8       43.3\n",
       "6                               Other current assets        9.0       16.0\n",
       "7                               Total current assets      837.9      917.1\n",
       "8                    Property and equipment, at cost     1124.0     1111.3\n",
       "9                    Less — accumulated depreciation    (798.1)    (780.5)"
      ]
     },
     "execution_count": 1712,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "balance_sheet=balance_sheet.reset_index(drop=True)\n",
    "balance_sheet.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1713,
   "id": "suspected-creation",
   "metadata": {},
   "outputs": [],
   "source": [
    "# cleaning and preparing data\n",
    "balance_sheet['28/11/2020'] = balance_sheet['28/11/2020'].map(lambda x: x.replace('(', '-').replace(')', ''))\n",
    "balance_sheet['30/05/2020'] = balance_sheet['30/05/2020'].map(lambda x: x.replace('(', '-').replace(')', ''))\n",
    "balance_sheet['28/11/2020'] = balance_sheet['28/11/2020'].map(lambda x: x.replace('—', '0'))\n",
    "balance_sheet['30/05/2020'] = balance_sheet['30/05/2020'].map(lambda x: x.replace('—', '0'))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1717,
   "id": "north-hometown",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Accounts</th>\n",
       "      <th>28/11/2020</th>\n",
       "      <th>30/05/2020</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Cash and cash equivalents</td>\n",
       "      <td>377.9</td>\n",
       "      <td>454.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Short-term investments</td>\n",
       "      <td>7.2</td>\n",
       "      <td>7.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Accounts receivable, net of allowances of $5.2...</td>\n",
       "      <td>193.0</td>\n",
       "      <td>180.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Unbilled accounts receivable</td>\n",
       "      <td>33.0</td>\n",
       "      <td>19.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Inventories, net</td>\n",
       "      <td>191.0</td>\n",
       "      <td>197.3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Prepaid expenses</td>\n",
       "      <td>26.8</td>\n",
       "      <td>43.3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Other current assets</td>\n",
       "      <td>9.0</td>\n",
       "      <td>16.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Total current assets</td>\n",
       "      <td>837.9</td>\n",
       "      <td>917.1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Property and equipment, at cost</td>\n",
       "      <td>1124.0</td>\n",
       "      <td>1111.3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>Less — accumulated depreciation</td>\n",
       "      <td>-798.1</td>\n",
       "      <td>-780.5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                            Accounts  28/11/2020  30/05/2020\n",
       "0                          Cash and cash equivalents       377.9       454.0\n",
       "1                             Short-term investments         7.2         7.0\n",
       "2  Accounts receivable, net of allowances of $5.2...       193.0       180.0\n",
       "3                       Unbilled accounts receivable        33.0        19.5\n",
       "4                                   Inventories, net       191.0       197.3\n",
       "5                                   Prepaid expenses        26.8        43.3\n",
       "6                               Other current assets         9.0        16.0\n",
       "7                               Total current assets       837.9       917.1\n",
       "8                    Property and equipment, at cost      1124.0      1111.3\n",
       "9                    Less — accumulated depreciation      -798.1      -780.5"
      ]
     },
     "execution_count": 1717,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# if we want we can store balance sheet to csv\n",
    "balance_sheet[['28/11/2020', '30/05/2020']] = balance_sheet[['28/11/2020', '30/05/2020']].astype(float)\n",
    "balance_sheet.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1715,
   "id": "handed-outline",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total current assets: 837.8999999999999\n",
      "Total current liabilities: 507.5\n",
      "Current Ratio: 1.6510344827586205\n"
     ]
    }
   ],
   "source": [
    "# here exaple some calculations\n",
    "total_current_assets = balance_sheet['28/11/2020'][0:7].sum()\n",
    "print('Total current assets:',total_current_assets)\n",
    "print('Total current liabilities:', balance_sheet['28/11/2020'][23])\n",
    "print('Current Ratio:', total_current_assets/balance_sheet['28/11/2020'][23])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "checked-lloyd",
   "metadata": {},
   "source": [
    "***\n",
    "Because of daily filings will publish at the end of the day, we can use atom output format to getting live filings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1564,
   "id": "blocked-outdoors",
   "metadata": {},
   "outputs": [],
   "source": [
    "# control list\n",
    "control = []\n",
    "new_data = {}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1724,
   "id": "female-profit",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Live time getting filings\n",
    "# This parser let us know last 40 filing that filled in each time and if it is not in our dataframe it show us new filings\n",
    "# so then we can find by CIK or Source and pars it for quick analyze\n",
    "import feedparser\n",
    "feed = feedparser.parse('https://www.sec.gov/cgi-bin/browse-edgar?action=getcurrent&CIK=&type=&company=&dateb=&owner=include&start=0&count=40&output=atom')\n",
    "\n",
    "for entry in feed.entries:\n",
    "    new_data.setdefault('Title', [])\n",
    "    new_data.setdefault('Source', [])\n",
    "    if entry.title not in control:\n",
    "        control.append(entry.title)\n",
    "        new_data['Title'].append(entry.title)\n",
    "        new_data['Source'].append(entry.link)\n",
    "        print(entry.title)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1723,
   "id": "signal-literature",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Title</th>\n",
       "      <th>Source</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>176</th>\n",
       "      <td>4 - Thimsen John (0001842163) (Reporting)</td>\n",
       "      <td>https://www.sec.gov/Archives/edgar/data/184216...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>175</th>\n",
       "      <td>3 - Pavia Juan Carlos (0001860670) (Reporting)</td>\n",
       "      <td>https://www.sec.gov/Archives/edgar/data/186067...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>174</th>\n",
       "      <td>4 - NOWICKI JOSEPH M (0001238633) (Reporting)</td>\n",
       "      <td>https://www.sec.gov/Archives/edgar/data/123863...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>173</th>\n",
       "      <td>4 - Lumber Liquidators Holdings, Inc. (0001396...</td>\n",
       "      <td>https://www.sec.gov/Archives/edgar/data/139603...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>172</th>\n",
       "      <td>4 - BRAINERD MARY K (0001195918) (Reporting)</td>\n",
       "      <td>https://www.sec.gov/Archives/edgar/data/119591...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>424B2 - CITIGROUP INC (0000831001) (Filer)</td>\n",
       "      <td>https://www.sec.gov/Archives/edgar/data/831001...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>13F-HR - BRUNI J V &amp; CO /CO (0001078841) (Filer)</td>\n",
       "      <td>https://www.sec.gov/Archives/edgar/data/107884...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>8-K - Federal Home Loan Bank of Atlanta (00013...</td>\n",
       "      <td>https://www.sec.gov/Archives/edgar/data/133146...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>6-K - EMX Royalty Corp (0001285786) (Filer)</td>\n",
       "      <td>https://www.sec.gov/Archives/edgar/data/128578...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>X-17A-5 - LABRUNERIE FINANCIAL SERVICES, INC. ...</td>\n",
       "      <td>https://www.sec.gov/Archives/edgar/data/933297...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>177 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                 Title  \\\n",
       "176          4 - Thimsen John (0001842163) (Reporting)   \n",
       "175     3 - Pavia Juan Carlos (0001860670) (Reporting)   \n",
       "174      4 - NOWICKI JOSEPH M (0001238633) (Reporting)   \n",
       "173  4 - Lumber Liquidators Holdings, Inc. (0001396...   \n",
       "172       4 - BRAINERD MARY K (0001195918) (Reporting)   \n",
       "..                                                 ...   \n",
       "4           424B2 - CITIGROUP INC (0000831001) (Filer)   \n",
       "3     13F-HR - BRUNI J V & CO /CO (0001078841) (Filer)   \n",
       "2    8-K - Federal Home Loan Bank of Atlanta (00013...   \n",
       "1          6-K - EMX Royalty Corp (0001285786) (Filer)   \n",
       "0    X-17A-5 - LABRUNERIE FINANCIAL SERVICES, INC. ...   \n",
       "\n",
       "                                                Source  \n",
       "176  https://www.sec.gov/Archives/edgar/data/184216...  \n",
       "175  https://www.sec.gov/Archives/edgar/data/186067...  \n",
       "174  https://www.sec.gov/Archives/edgar/data/123863...  \n",
       "173  https://www.sec.gov/Archives/edgar/data/139603...  \n",
       "172  https://www.sec.gov/Archives/edgar/data/119591...  \n",
       "..                                                 ...  \n",
       "4    https://www.sec.gov/Archives/edgar/data/831001...  \n",
       "3    https://www.sec.gov/Archives/edgar/data/107884...  \n",
       "2    https://www.sec.gov/Archives/edgar/data/133146...  \n",
       "1    https://www.sec.gov/Archives/edgar/data/128578...  \n",
       "0    https://www.sec.gov/Archives/edgar/data/933297...  \n",
       "\n",
       "[177 rows x 2 columns]"
      ]
     },
     "execution_count": 1723,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.DataFrame(new_data)\n",
    "df[::-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "lesbian-affect",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "consolidated-warehouse",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "israeli-james",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}