Skip to content

Commit

Permalink
update
Browse files Browse the repository at this point in the history
  • Loading branch information
Prashant Tripathi authored and Prashant Tripathi committed Jul 24, 2024
1 parent 8dbc39f commit 928dbdb
Show file tree
Hide file tree
Showing 7 changed files with 365 additions and 306 deletions.
334 changes: 167 additions & 167 deletions DATA/GOLD/TradeHistory/TradeHistory_data.csv
View file

Large diffs are not rendered by default.

14 changes: 7 additions & 7 deletions NOTEBOOKS/01_ETL_Bronze2Silver_Layer_TradeHistory.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,12 +9,12 @@
},
{
"cell_type": "code",
"execution_count": 9,
"execution_count": 1,
"metadata": {},
"outputs": [],
"source": [
"import pandas as pd\n",
"from common_utilities import replace_punctuation_from_columns,global_paths"
"from common_utilities import replace_punctuation_from_columns, global_path"
]
},
{
Expand All @@ -26,7 +26,7 @@
},
{
"cell_type": "code",
"execution_count": 10,
"execution_count": 2,
"metadata": {},
"outputs": [
{
Expand All @@ -49,7 +49,7 @@
" 8 price 166 non-null float64 \n",
"dtypes: datetime64[ns](1), float64(3), object(5)\n",
"memory usage: 13.0+ KB\n",
"Silver Layer csv file for trade history successfully created at:\n",
"SILVER Layer csv file for trade history successfully created at:\n",
"C:\\Users\\prashant.tripathi\\Code\\Upstox\\DATA\\SILVER\\TradeHistory\\TradeHistory_data.csv\n"
]
}
Expand All @@ -59,7 +59,7 @@
"df_TradeHistory = []\n",
"\n",
"# Loop through List of all CSV files in the folder\n",
"for file_path in TRADEHISTORY_BRONZE_LAYER_PATH.glob(\"*.csv\"):\n",
"for file_path in global_path.tradehistory_bronze_layer_path.glob(\"*.csv\"):\n",
" # Read the CSV file\n",
" df = pd.read_csv(file_path)\n",
" # Append the DataFrame to the list\n",
Expand Down Expand Up @@ -124,11 +124,11 @@
"]\n",
"\n",
"# Save the result as a csv file\n",
"df_TradeHistory.to_csv(TradeHistory_SILVER_FILE_PATH, index=None)\n",
"df_TradeHistory.to_csv(global_path.tradehistory_silver_file_path, index=None)\n",
"df_TradeHistory.info()\n",
"\n",
"print(\"SILVER Layer csv file for trade history successfully created at:\")\n",
"print(TradeHistory_SILVER_FILE_PATH.resolve())"
"print(global_path.tradehistory_silver_file_path.resolve())"
]
}
],
Expand Down
14 changes: 7 additions & 7 deletions NOTEBOOKS/02_ETL_Bronze2Silver_Layer_BillSummary.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,12 +9,12 @@
},
{
"cell_type": "code",
"execution_count": 7,
"execution_count": 1,
"metadata": {},
"outputs": [],
"source": [
"import pandas as pd\n",
"from common_utilities import replace_punctuation_from_columns,global_paths"
"from common_utilities import replace_punctuation_from_columns,global_path"
]
},
{
Expand All @@ -26,7 +26,7 @@
},
{
"cell_type": "code",
"execution_count": 8,
"execution_count": 2,
"metadata": {},
"outputs": [
{
Expand All @@ -46,7 +46,7 @@
" 5 bill_number 53 non-null int32 \n",
"dtypes: float64(2), int32(1), object(3)\n",
"memory usage: 2.7+ KB\n",
"Silver Layer csv file for Bill Summary successfully created at:\n",
"SILVER Layer csv file for Bill Summary successfully created at:\n",
"C:\\Users\\prashant.tripathi\\Code\\Upstox\\DATA\\SILVER\\BillSummary\\BillSummary_data.csv\n"
]
}
Expand All @@ -56,7 +56,7 @@
"df_BillSummary = []\n",
"\n",
"# Loop through List of all CSV files in the folder\n",
"for file_path in BillSummary_BRONZE_LAYER_PATH.glob(\"*.csv\"):\n",
"for file_path in global_path.billsummary_bronze_layer_path.glob(\"*.csv\"):\n",
" # Read the CSV file\n",
" df = pd.read_csv(file_path)\n",
" # Append the DataFrame to the list\n",
Expand All @@ -81,11 +81,11 @@
"df_BillSummary = df_BillSummary[[\"date\", \"exchange\", \"segment\", \"debit_amount\", \"credit_amount\", \"bill_number\"]]\n",
"\n",
"# Save the result as a csv file\n",
"df_BillSummary.to_csv(BillSummary_SILVER_FILE_PATH, index=None)\n",
"df_BillSummary.to_csv(global_path.billsummary_silver_file_path, index=None)\n",
"df_BillSummary.info()\n",
"\n",
"print(\"SILVER Layer csv file for Bill Summary successfully created at:\")\n",
"print(BillSummary_SILVER_FILE_PATH.resolve())"
"print(global_path.billsummary_silver_file_path.resolve())"
]
}
],
Expand Down
14 changes: 7 additions & 7 deletions NOTEBOOKS/03_ETL_Bronze2Silver_Layer_StockPrice.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,12 +9,12 @@
},
{
"cell_type": "code",
"execution_count": 5,
"execution_count": 1,
"metadata": {},
"outputs": [],
"source": [
"import pandas as pd\n",
"from common_utilities import replace_punctuation_from_columns,global_paths"
"from common_utilities import replace_punctuation_from_columns,global_path"
]
},
{
Expand All @@ -26,7 +26,7 @@
},
{
"cell_type": "code",
"execution_count": 6,
"execution_count": 2,
"metadata": {},
"outputs": [
{
Expand All @@ -46,7 +46,7 @@
" 5 close 13688 non-null float64\n",
"dtypes: float64(4), object(2)\n",
"memory usage: 641.8+ KB\n",
"Silver Layer csv file for Stock Price history successfully created at:\n",
"SILVER Layer csv file for Stock Price history successfully created at:\n",
"C:\\Users\\prashant.tripathi\\Code\\Upstox\\DATA\\SILVER\\StockPrice\\StockPrice_data.csv\n"
]
}
Expand All @@ -56,7 +56,7 @@
"df_StockPrice = []\n",
"\n",
"# Loop through List of all CSV files in the folder\n",
"for file_path in StockPrice_BRONZE_LAYER_PATH.glob(\"*.NS.csv\"):\n",
"for file_path in global_path.stockprice_bronze_layer_path.glob(\"*.NS.csv\"):\n",
" # Read the CSV file\n",
" df = pd.read_csv(file_path)\n",
"\n",
Expand All @@ -79,11 +79,11 @@
"\n",
"# Save the result as a csv file\n",
"df_StockPrice = df_StockPrice[[\"date\", \"stock_name\", \"open\", \"high\", \"low\", \"close\"]]\n",
"df_StockPrice.to_csv(StockPrice_SILVER_FILE_PATH, index=None)\n",
"df_StockPrice.to_csv(global_path.stockprice_silver_file_path, index=None)\n",
"df_StockPrice.info()\n",
"\n",
"print(\"SILVER Layer csv file for Stock Price history successfully created at:\")\n",
"print(StockPrice_SILVER_FILE_PATH.resolve())"
"print(global_path.stockprice_silver_file_path.resolve())"
]
}
],
Expand Down
14 changes: 7 additions & 7 deletions NOTEBOOKS/04_ETL_Bronze2Silver_Layer_Symbol.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,12 +9,12 @@
},
{
"cell_type": "code",
"execution_count": 3,
"execution_count": 1,
"metadata": {},
"outputs": [],
"source": [
"import pandas as pd\n",
"from common_utilities import replace_punctuation_from_columns,global_paths"
"from common_utilities import replace_punctuation_from_columns,global_path"
]
},
{
Expand All @@ -26,7 +26,7 @@
},
{
"cell_type": "code",
"execution_count": 4,
"execution_count": 2,
"metadata": {},
"outputs": [
{
Expand All @@ -45,7 +45,7 @@
" 4 instrument_type 4328 non-null object\n",
"dtypes: object(5)\n",
"memory usage: 202.9+ KB\n",
"Silver Layer csv file for Symbol successfully created at:\n",
"SILVER Layer csv file for Symbol successfully created at:\n",
"C:\\Users\\prashant.tripathi\\Code\\Upstox\\DATA\\SILVER\\Symbol\\Symbol_data.csv\n"
]
}
Expand All @@ -55,7 +55,7 @@
"df_Symbol = []\n",
"\n",
"# Loop through List of all CSV files in the folder\n",
"for file_path in Symbol_BRONZE_LAYER_PATH.glob(\"*.csv\"):\n",
"for file_path in global_path.symbol_bronze_layer_path.glob(\"*.csv\"):\n",
" # Read the CSV file\n",
" df = pd.read_csv(file_path)\n",
" # Append the DataFrame to the list\n",
Expand All @@ -71,11 +71,11 @@
"df_Symbol = df_Symbol.sort_values(by=[\"scrip_code\"])\n",
"\n",
"# Save the result as a csv file\n",
"df_Symbol.to_csv(Symbol_SILVER_FILE_PATH, index=None)\n",
"df_Symbol.to_csv(global_path.symbol_silver_file_path, index=None)\n",
"df_Symbol.info()\n",
"\n",
"print(\"SILVER Layer csv file for Symbol successfully created at:\")\n",
"print(Symbol_SILVER_FILE_PATH.resolve())"
"print(global_path.symbol_silver_file_path.resolve())"
]
}
],
Expand Down
97 changes: 77 additions & 20 deletions NOTEBOOKS/05_ETL_Silver2Gold_Layer_TradeHistory.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,12 +9,12 @@
},
{
"cell_type": "code",
"execution_count": 1,
"execution_count": 11,
"metadata": {},
"outputs": [],
"source": [
"import pandas as pd\n",
"from common_utilities import Portfolio,global_paths"
"from common_utilities import Portfolio, global_path"
]
},
{
Expand All @@ -26,20 +26,30 @@
},
{
"cell_type": "code",
"execution_count": 2,
"execution_count": 12,
"metadata": {},
"outputs": [],
"source": [
"# read the csv file\n",
"df_TradeHistory = pd.read_csv(TradeHistory_SILVER_FILE_PATH)\n",
"\n",
"# replace scrip code to compnay name\n",
"df_Symbol = pd.read_csv(Symbol_SILVER_FILE_PATH)\n",
"df_Symbol[\"scrip_code\"] = df_Symbol[\"scrip_code\"].astype(str)\n",
"df_TradeHistory = pd.read_csv(global_path.tradehistory_silver_file_path)\n",
"\n",
"# Convert 'datetime' to datetime type\n",
"df_TradeHistory[\"datetime\"] = pd.to_datetime(df_TradeHistory[\"datetime\"])\n",
"\n",
"# sort the dataframe by date\n",
"df_TradeHistory = df_TradeHistory.sort_values(by=\"datetime\")"
]
},
{
"cell_type": "code",
"execution_count": 13,
"metadata": {},
"outputs": [],
"source": [
"# replace scrip code to compnay name\n",
"df_Symbol = pd.read_csv(global_path.symbol_silver_file_path)\n",
"df_Symbol[\"scrip_code\"] = df_Symbol[\"scrip_code\"].astype(str)\n",
"\n",
"# Merge df_TradeHistory with df_Symbol on the matching columns\n",
"df_TradeHistory = df_TradeHistory.merge(\n",
" df_Symbol[[\"scrip_code\", \"symbol\"]],\n",
Expand All @@ -49,21 +59,69 @@
")\n",
"\n",
"# Assign the new column 'stock_name' in df_TradeHistory to the values from 'symbol'\n",
"df_TradeHistory[\"stock_name\"] = df_TradeHistory[\"symbol\"].combine_first(df_TradeHistory[\"stock_name\"])\n",
"df_TradeHistory[\"stock_name\"] = df_TradeHistory[\"symbol\"].combine_first(df_TradeHistory[\"stock_name\"])"
]
},
{
"cell_type": "code",
"execution_count": 14,
"metadata": {},
"outputs": [],
"source": [
"portfolio = Portfolio()\n",
"\n",
"# sort the dataframe by date\n",
"df_TradeHistory = df_TradeHistory.sort_values(by=\"datetime\")\n",
"# Apply the function of trade logic to each row of the DataFrame\n",
"data = [portfolio.trade(row.to_dict()) for _, row in df_TradeHistory.iterrows()]\n",
"\n",
"df_TradeHistory = pd.DataFrame(data)\n",
"\n",
"portfolio = Portfolio()\n",
"data = [portfolio.trade(row) for _, row in df_TradeHistory.iterrows()]\n",
"# Select columns that end with 'price' or 'amount'\n",
"columns_to_round = [col for col in df_TradeHistory.columns if col.endswith('price') or col.endswith('amount')]\n",
"\n",
"df_TradeHistory = pd.DataFrame(data)\n",
"# df_TradeHistory = df_TradeHistory.round(2)\n",
"df_TradeHistory.columns()\n",
"# Round the values in the selected columns to two decimal places\n",
"df_TradeHistory[columns_to_round] = df_TradeHistory[columns_to_round].round(2)\n"
]
},
{
"cell_type": "code",
"execution_count": 15,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"<class 'pandas.core.frame.DataFrame'>\n",
"Index: 166 entries, 2 to 92\n",
"Data columns (total 14 columns):\n",
" # Column Non-Null Count Dtype \n",
"--- ------ -------------- ----- \n",
" 0 datetime 166 non-null datetime64[ns]\n",
" 1 exchange 166 non-null object \n",
" 2 segment 166 non-null object \n",
" 3 stock_name 166 non-null object \n",
" 4 side 166 non-null object \n",
" 5 buy_quantity 166 non-null int64 \n",
" 6 buy_price 166 non-null float64 \n",
" 7 buy_amount 166 non-null float64 \n",
" 8 sell_quantity 166 non-null int64 \n",
" 9 sell_price 166 non-null float64 \n",
" 10 sell_amount 166 non-null float64 \n",
" 11 holding_quantity 166 non-null int64 \n",
" 12 holding_price 166 non-null float64 \n",
" 13 holding_amount 166 non-null int64 \n",
"dtypes: datetime64[ns](1), float64(5), int64(4), object(4)\n",
"memory usage: 19.5+ KB\n",
"GOLD Layer csv file for trade history successfully created at:\n",
"C:\\Users\\prashant.tripathi\\Code\\Upstox\\DATA\\GOLD\\TradeHistory\\TradeHistory_data.csv\n"
]
}
],
"source": [
"df_TradeHistory = df_TradeHistory[\n",
" [\n",
" \"datetime\",\n",
" \"exchange\",\n",
" \"segment\",\n",
" \"stock_name\",\n",
" \"side\",\n",
Expand All @@ -73,9 +131,8 @@
" \"sell_quantity\",\n",
" \"sell_price\",\n",
" \"sell_amount\",\n",
" \"profit_loss_amount\",\n",
" \"holding_quantity\",\n",
" \"holding_price_avg\",\n",
" \"holding_price\",\n",
" \"holding_amount\",\n",
" ]\n",
"]\n",
Expand All @@ -84,10 +141,10 @@
"df_TradeHistory = df_TradeHistory.sort_values(by=[\"segment\", \"stock_name\", \"datetime\"])\n",
"\n",
"# Save the result as a csv file\n",
"df_TradeHistory.to_csv(TradeHistory_GOLD_FILE_PATH, index=None)\n",
"df_TradeHistory.to_csv(global_path.tradehistory_gold_file_path, index=None)\n",
"df_TradeHistory.info()\n",
"print(\"GOLD Layer csv file for trade history successfully created at:\")\n",
"print(TradeHistory_GOLD_FILE_PATH.resolve())"
"print(global_path.tradehistory_gold_file_path.resolve())"
]
}
],
Expand Down
Loading

0 comments on commit 928dbdb

Please sign in to comment.