v2.03 - Nifty Prediction added to GUI

- Nifty-50 Gap up/down prediction added - cosmetic updates for GUI - version bumped - tab added for similar stock search - chromadb similar search in progress - pyinstaller build and requirements updated - fixed similar search in CLI mode - similar search added to GUI - Update requirements.txt - chromadb added to top - try catch added for importing chromadb to resolve pytest errors - Release page updated
pranjal-joshi · Oct 3, 2023 · ce399a4 · ce399a4
1 parent 57759b5
commit ce399a4
Show file tree

Hide file tree

Showing 11 changed files with 205 additions and 40 deletions.
diff --git a/.github/dependencies/sqlite3.dll b/.github/dependencies/sqlite3.dll
diff --git a/.github/workflows/workflow-build-matrix.yml b/.github/workflows/workflow-build-matrix.yml
@@ -24,7 +24,7 @@ jobs:
           - os: windows-latest
             TARGET: Windows
             CMD_BUILD: |
-              pyinstaller --onefile --icon=src\icon.ico src\screenipy.py  --hidden-import cmath --hidden-import talib.stream --hidden-import numpy --hidden-import pandas --hidden-import alive-progress --hidden-import alive_progress
+              pyinstaller --onefile --icon=src\icon.ico src\screenipy.py  --hidden-import cmath --hidden-import talib.stream --hidden-import numpy --hidden-import pandas --hidden-import alive-progress --hidden-import alive_progress --hidden-import chromadb
             DEP_BUILD: |
               python -m pip install --upgrade pip
               echo Installing TA-lib...
@@ -44,7 +44,7 @@ jobs:
           - os: ubuntu-20.04
             TARGET: Linux
             CMD_BUILD: |
-              pyinstaller --onefile --icon=src/icon.ico src/screenipy.py  --hidden-import cmath --hidden-import talib.stream --hidden-import numpy --hidden-import pandas --hidden-import alive-progress --hidden-import alive_progress
+              pyinstaller --onefile --icon=src/icon.ico src/screenipy.py  --hidden-import cmath --hidden-import talib.stream --hidden-import numpy --hidden-import pandas --hidden-import alive-progress --hidden-import alive_progress --hidden-import chromadb
               mv /home/runner/work/Screeni-py/Screeni-py/dist/screenipy /home/runner/work/Screeni-py/Screeni-py/dist/screenipy.bin
               chmod +x /home/runner/work/Screeni-py/Screeni-py/dist/screenipy.bin
             DEP_BUILD: |
@@ -68,7 +68,7 @@ jobs:
           - os: macos-latest
             TARGET: MacOS
             CMD_BUILD: |
-              pyinstaller --onefile --windowed --icon=src/icon.ico src/screenipy.py  --hidden-import cmath --hidden-import talib.stream --hidden-import numpy --hidden-import pandas --hidden-import alive-progress --hidden-import alive_progress
+              pyinstaller --onefile --windowed --icon=src/icon.ico src/screenipy.py  --hidden-import cmath --hidden-import talib.stream --hidden-import numpy --hidden-import pandas --hidden-import alive-progress --hidden-import alive_progress --hidden-import chromadb
               mv /Users/runner/work/Screeni-py/Screeni-py/dist/screenipy /Users/runner/work/Screeni-py/Screeni-py/dist/screenipy.run
             DEP_BUILD: |
               brew install ta-lib

diff --git a/requirements.txt b/requirements.txt
@@ -18,7 +18,7 @@ pyinstaller==5.6.2
 pytest-mock
 pytoml
 retrying
-scipy==1.7.3
+scipy==1.11.2
 ta-lib
 tabulate
 yfinance==0.1.87
@@ -61,4 +61,5 @@ pandas_ta
 # protobuf==3.19.6
 protobuf
 streamlit==1.26.0
-tensorflow
+tensorflow
+chromadb==0.4.10
diff --git a/src/classes/Changelog.py b/src/classes/Changelog.py
@@ -7,7 +7,7 @@
 
 from classes.ColorText import colorText
 
-VERSION = "2.02"
+VERSION = "2.03"
 
 changelog = colorText.BOLD + '[ChangeLog]\n' + colorText.END + colorText.BLUE + '''
 [1.00 - Beta]
@@ -216,4 +216,8 @@
 3. Cosmetic improvements
 4. YouTube Video added to docs
 
+[2.03]
+1. AI based Nifty-50 Gap up/down prediction added to GUI
+2. Cosmetic updates and minor bug-fixes.
+
 ''' + colorText.END
diff --git a/src/classes/OtaUpdater.py b/src/classes/OtaUpdater.py
@@ -85,7 +85,8 @@ def showWhatsNew():
         md = requests.get(url)
         txt = md.text
         txt = txt.split("New?")[1]
-        txt = txt.split("## Downloads")[0]
+        # txt = txt.split("## Downloads")[0]
+        txt = txt.split("## Installation Guide")[0]
         txt = txt.replace('**','').replace('`','').strip()
         return (txt+"\n")
 

diff --git a/src/classes/ParallelProcessing.py b/src/classes/ParallelProcessing.py
@@ -59,7 +59,7 @@ def run(self):
             sys.exit(0)
 
     def screenStocks(self, executeOption, reversalOption, maLength, daysForLowestVolume, minRSI, maxRSI, respChartPattern, insideBarToLookback, totalSymbols,
-                     configManager, fetcher, screener, candlePatterns, stock, newlyListedOnly, downloadOnly, printCounter=False):
+                     configManager, fetcher, screener, candlePatterns, stock, newlyListedOnly, downloadOnly, vectorSearch, printCounter=False):
         screenResults = pd.DataFrame(columns=[
             'Stock', 'Consolidating', 'Breaking-Out', 'MA-Signal', 'Volume', 'LTP', 'RSI', 'Trend', 'Pattern'])
         screeningDictionary = {'Stock': "", 'Consolidating': "",  'Breaking-Out': "",
@@ -105,6 +105,11 @@ def screenStocks(self, executeOption, reversalOption, maLength, daysForLowestVol
 
             fullData, processedData = screener.preprocessData(
                 data, daysToLookback=configManager.daysToLookback)
+
+            if type(vectorSearch) != bool and type(vectorSearch) and vectorSearch[2] == True:
+                executeOption = 0
+                with self.screenCounter.get_lock():
+                    screener.addVector(fullData, stock, vectorSearch[1])
 
             if newlyListedOnly:
                 if not screener.validateNewlyListed(fullData, period):

diff --git a/src/classes/Screener.py b/src/classes/Screener.py
@@ -9,16 +9,22 @@
 import math
 import numpy as np
 import pandas as pd
-# import talib
 import joblib
 import keras
+import time
 import classes.Utility as Utility
+from classes.Utility import isGui
 from sklearn.preprocessing import StandardScaler
 from scipy.signal import argrelextrema
 from scipy.stats import linregress
 from classes.ColorText import colorText
 from classes.SuppressOutput import SuppressOutput
 from classes.ScreenipyTA import ScreenerTA
+try:
+    import chromadb
+    CHROMA_AVAILABLE = True
+except:
+    CHROMA_AVAILABLE = False
 
 
 # Exception for newly listed stocks with candle nos < daysToLookback
@@ -592,9 +598,11 @@ def getNiftyPrediction(self, data, proxyServer):
             out = colorText.BOLD + colorText.GREEN + "BULLISH" + colorText.END + colorText.BOLD
             sug = "Stay Bullish!"
         if not Utility.tools.isClosingHour():
-            print(colorText.BOLD + colorText.WARN + "Note: The AI prediction should be executed After 3 PM or Near to Closing time as the Prediction Accuracy is based on the Closing price!" + colorText.END)
+            print(colorText.BOLD + colorText.WARN + "Note: The AI prediction should be executed After 3 PM Around the Closing hours as the Prediction Accuracy is based on the Closing price!" + colorText.END)
         print(colorText.BOLD + colorText.BLUE + "\n" + "[+] Nifty AI Prediction -> " + colorText.END + colorText.BOLD + "Market may Open {} next day! {}".format(out, sug) + colorText.END)
         print(colorText.BOLD + colorText.BLUE + "\n" + "[+] Nifty AI Prediction -> " + colorText.END + "Probability/Strength of Prediction = {}%".format(Utility.tools.getSigmoidConfidence(pred[0])))
+        if isGui():
+            return pred, 'BULLISH' if pred <= 0.5 else 'BEARISH', Utility.tools.getSigmoidConfidence(pred[0])
         return pred
 
     def monitorFiveEma(self, proxyServer, fetcher, result_df, last_signal, risk_reward = 3):
@@ -668,7 +676,27 @@ def monitorFiveEma(self, proxyServer, fetcher, result_df, last_signal, risk_rewa
         result_df.drop_duplicates(keep='last', inplace=True)
         result_df.sort_values(by='Time', inplace=True)
         return result_df[::-1]
-
+
+    # Add data to vector database
+    def addVector(self, data, stockCode, daysToLookback):
+        data = data[::-1] # Reinverting preprocessedData for pct_change
+        data = data.pct_change()
+        # data = data[::-1]     # Do we need to invert again? No we dont - See operation after flatten
+        data = data[['Open', 'High', 'Low', 'Close']]
+        data = data.reset_index(drop=True)
+        data = data.dropna()
+        data = data.to_numpy().flatten().tolist()
+        data = data[(-4 * daysToLookback):]     # Keep only OHLC * daysToLookback samples
+        if len(data) == (4 * daysToLookback):
+            chroma_client = chromadb.PersistentClient(path="./chromadb_store/")
+            collection = chroma_client.get_or_create_collection(name="nse_stocks")
+            collection.upsert(
+                embeddings=[data],
+                documents=[stockCode],
+                ids=[stockCode]
+            )
+            return data
+
 
     '''
     # Find out trend for days to lookback

diff --git a/src/classes/Utility.py b/src/classes/Utility.py
@@ -286,6 +286,19 @@ def promptChartPatterns():
             input(colorText.BOLD + colorText.FAIL +
                   "\n[+] Invalid Option Selected. Press Any Key to Continue..." + colorText.END)
             return (None, None)
+
+    # Prompt for Similar stock search
+    def promptSimilarStockSearch():
+        try:
+            stockCode = str(input(colorText.BOLD + colorText.WARN +
+                                    "\n[+] Enter the Name of the stock to search similar stocks for: " + colorText.END)).upper()
+            candles = int(input(colorText.BOLD + colorText.WARN +
+                                    "\n[+] How many candles (TimeFrame) to look back for similarity? : " + colorText.END))
+            return stockCode, candles
+        except ValueError:
+            input(colorText.BOLD + colorText.FAIL +
+                "\n[+] Invalid Option Selected. Press Any Key to Continue..." + colorText.END)
+            return None, None
 
     def getProgressbarStyle():
         bar = 'smooth'

diff --git a/src/release.md b/src/release.md
@@ -16,12 +16,13 @@ Celebrating more than 7K+ Downloads - Thank You for your support :tada:
 
 [![Screeni-py - Detailed Installation Guide](https://markdown-videos-api.jorgenkh.no/url?url=https%3A%2F%2Fyoutu.be%2F2HMN0ac4H20)](https://youtu.be/2HMN0ac4H20)
 
-## Downloads
+## Downloads 
+#### Deprycated - Use Docker Method mentioned in next section
 | Operating System | Executable File |
 | :-: | --- |
-| ![Windows](https://img.shields.io/badge/Windows-0078D6?style=for-the-badge&logo=windows&logoColor=white) | **[screenipy.exe](https://github.com/pranjal-joshi/Screeni-py/releases/download/2.02/screenipy.exe)** |
-| ![Linux](https://img.shields.io/badge/Linux-FCC624?style=for-the-badge&logo=linux&logoColor=black) | **[screenipy.bin](https://github.com/pranjal-joshi/Screeni-py/releases/download/2.02/screenipy.bin)** |
-| ![Mac OS](https://img.shields.io/badge/mac%20os-D3D3D3?style=for-the-badge&logo=apple&logoColor=000000) | **[screenipy.run](https://github.com/pranjal-joshi/Screeni-py/releases/download/2.02/screenipy.run)** ([Read Installation Guide](https://github.com/pranjal-joshi/Screeni-py/blob/main/INSTALLATION.md#for-macos)) |
+| ![Windows](https://img.shields.io/badge/Windows-0078D6?style=for-the-badge&logo=windows&logoColor=white) | **[screenipy.exe](https://github.com/pranjal-joshi/Screeni-py/releases/download/2.03/screenipy.exe)** |
+| ![Linux](https://img.shields.io/badge/Linux-FCC624?style=for-the-badge&logo=linux&logoColor=black) | **[screenipy.bin](https://github.com/pranjal-joshi/Screeni-py/releases/download/2.03/screenipy.bin)** |
+| ![Mac OS](https://img.shields.io/badge/mac%20os-D3D3D3?style=for-the-badge&logo=apple&logoColor=000000) | **[screenipy.run](https://github.com/pranjal-joshi/Screeni-py/releases/download/2.03/screenipy.run)** ([Read Installation Guide](https://github.com/pranjal-joshi/Screeni-py/blob/main/INSTALLATION.md#for-macos)) |
 
 ## [Docker Releases](https://hub.docker.com/r/joshipranjal/screeni-py/tags)
 

diff --git a/src/screenipy.py b/src/screenipy.py
@@ -1,7 +1,7 @@
 #!/usr/bin/python3
 
-# Pyinstaller compile Windows: pyinstaller --onefile --icon=src\icon.ico src\screenipy.py  --hidden-import cmath --hidden-import talib.stream --hidden-import numpy --hidden-import pandas --hidden-import alive-progress
-# Pyinstaller compile Linux  : pyinstaller --onefile --icon=src/icon.ico src/screenipy.py  --hidden-import cmath --hidden-import talib.stream --hidden-import numpy --hidden-import pandas --hidden-import alive-progress
+# Pyinstaller compile Windows: pyinstaller --onefile --icon=src\icon.ico src\screenipy.py  --hidden-import cmath --hidden-import talib.stream --hidden-import numpy --hidden-import pandas --hidden-import alive-progress --hidden-import chromadb
+# Pyinstaller compile Linux  : pyinstaller --onefile --icon=src/icon.ico src/screenipy.py  --hidden-import cmath --hidden-import talib.stream --hidden-import numpy --hidden-import pandas --hidden-import alive-progress --hidden-import chromadb
 
 # Keep module imports prior to classes
 import os
@@ -28,6 +28,11 @@
 from tabulate import tabulate
 import multiprocessing
 multiprocessing.freeze_support()
+try:
+    import chromadb
+    CHROMA_AVAILABLE = True
+except:
+    CHROMA_AVAILABLE = False
 
 # Argument Parsing for test purpose
 argParser = argparse.ArgumentParser()
@@ -51,6 +56,9 @@
 loadCount = 0
 maLength = None
 newlyListedOnly = False
+vectorSearch = False
+
+CHROMADB_PATH = "chromadb_store/"
 
 configManager = ConfigManager.tools()
 fetcher = Fetcher.tools(configManager)
@@ -63,6 +71,15 @@
 except KeyError:
     proxyServer = ""
 
+# Clear chromadb store initially
+if CHROMA_AVAILABLE:
+    chroma_client = chromadb.PersistentClient(path=CHROMADB_PATH)
+    try:
+        chroma_client.delete_collection("nse_stocks")
+    except:
+        pass
+
+
 # Manage Execution flow
 
 
@@ -74,6 +91,7 @@ def initExecution():
      W > Screen stocks from my own Watchlist
      N > Nifty Prediction using Artifical Intelligence (Use for Gap-Up/Gap-Down/BTST/STBT)
      E > Live Index Scan : 5 EMA for Intraday
+     S > Search for Similar Stocks (forming Similar Chart Pattern)
 
      0 > Screen stocks by the stock names (NSE Stock Code)
      1 > Nifty 50               2 > Nifty Next 50           3 > Nifty 100
@@ -108,7 +126,7 @@ def initExecution():
         Utility.tools.clearScreen()
         return initExecution()
 
-    if tickerOption == 'N' or tickerOption == 'E':
+    if tickerOption == 'N' or tickerOption == 'E' or tickerOption == 'S':
         return tickerOption, 0
 
     if tickerOption and tickerOption != 'W':
@@ -153,7 +171,7 @@ def initExecution():
 
 # Main function
 def main(testing=False, testBuild=False, downloadOnly=False, execute_inputs:list = []):
-    global screenCounter, screenResultsCounter, stockDict, loadedStockData, keyboardInterruptEvent, loadCount, maLength, newlyListedOnly
+    global screenCounter, screenResultsCounter, stockDict, loadedStockData, keyboardInterruptEvent, loadCount, maLength, newlyListedOnly, vectorSearch
     screenCounter = multiprocessing.Value('i', 1)
     screenResultsCounter = multiprocessing.Value('i', 0)
     keyboardInterruptEvent = multiprocessing.Manager().Event()
@@ -184,7 +202,10 @@ def main(testing=False, testBuild=False, downloadOnly=False, execute_inputs:list
             if execute_inputs != []:
                 if not configManager.checkConfigFile():
                     configManager.setConfig(ConfigManager.parser, default=True, showFileCreatedText=False)
-                tickerOption, executeOption = int(execute_inputs[0]), int(execute_inputs[1])
+                try:
+                    tickerOption, executeOption = int(execute_inputs[0]), int(execute_inputs[1])
+                except:
+                    tickerOption, executeOption = str(execute_inputs[0]), int(execute_inputs[1])
                 if tickerOption == 13:
                     newlyListedOnly = True
                     tickerOption = 12
@@ -268,7 +289,7 @@ def main(testing=False, testBuild=False, downloadOnly=False, execute_inputs:list
               "[+] Press any key to Exit!" + colorText.END)
         sys.exit(0)
 
-    if tickerOption == 'W' or tickerOption == 'N' or tickerOption == 'E' or (tickerOption >= 0 and tickerOption < 15):
+    if tickerOption == 'W' or tickerOption == 'N' or tickerOption == 'E' or tickerOption == 'S' or (tickerOption >= 0 and tickerOption < 15):
         configManager.getConfig(ConfigManager.parser)
         try:
             if tickerOption == 'W':
@@ -316,6 +337,18 @@ def main(testing=False, testBuild=False, downloadOnly=False, execute_inputs:list
                     if not isGui():
                         input('\nPress any key to Continue...\n')
                     return
+            elif tickerOption == 'S':
+                if not CHROMA_AVAILABLE:
+                    print(colorText.BOLD + colorText.FAIL +
+                  "\n\n[+] ChromaDB not available in your environment! You can't use this feature!\n" + colorText.END)
+                else:
+                    if execute_inputs != []:
+                        stockCode, candles = execute_inputs[2], execute_inputs[3]
+                    else:
+                        stockCode, candles = Utility.tools.promptSimilarStockSearch()
+                    vectorSearch = [stockCode, candles, True]
+                    tickerOption, executeOption = 12, 1
+                    listStockCodes = fetcher.fetchStockCodes(tickerOption, proxyServer=proxyServer)
             else:
                 if tickerOption == 14:    # Override config for F&O Stocks
                     configManager.stageTwo = False
@@ -338,7 +371,7 @@ def main(testing=False, testBuild=False, downloadOnly=False, execute_inputs:list
               "[+] Starting Stock Screening.. Press Ctrl+C to stop!\n")
 
         items = [(executeOption, reversalOption, maLength, daysForLowestVolume, minRSI, maxRSI, respChartPattern, insideBarToLookback, len(listStockCodes),
-                  configManager, fetcher, screener, candlePatterns, stock, newlyListedOnly, downloadOnly)
+                  configManager, fetcher, screener, candlePatterns, stock, newlyListedOnly, downloadOnly, vectorSearch)
                  for stock in listStockCodes]
 
         tasks_queue = multiprocessing.JoinableQueue()
@@ -416,6 +449,25 @@ def main(testing=False, testBuild=False, downloadOnly=False, execute_inputs:list
             except Exception as e:
                 break
 
+        if CHROMA_AVAILABLE and type(vectorSearch) == list and vectorSearch[2]:
+            chroma_client = chromadb.PersistentClient(path=CHROMADB_PATH)
+            collection = chroma_client.get_collection(name="nse_stocks")
+            query_embeddings= collection.get(ids = [stockCode], include=["embeddings"])["embeddings"]
+            results = collection.query(
+                query_embeddings=query_embeddings,
+                n_results=4
+            )['ids'][0]
+            try:
+                results.remove(stockCode)
+            except ValueError:
+                pass
+            matchedScreenResults, matchedSaveResults = pd.DataFrame(columns=screenResults.columns), pd.DataFrame(columns=saveResults.columns)
+            for stk in results:
+                matchedScreenResults = matchedScreenResults.append(screenResults[screenResults['Stock'].str.contains(stk)])
+                matchedSaveResults = matchedSaveResults.append(saveResults[saveResults['Stock'].str.contains(stk)])
+            screenResults, saveResults = matchedScreenResults, matchedSaveResults
+
+
         screenResults.sort_values(by=['Stock'], ascending=True, inplace=True)
         saveResults.sort_values(by=['Stock'], ascending=True, inplace=True)
         screenResults.set_index('Stock', inplace=True)
@@ -456,6 +508,7 @@ def main(testing=False, testBuild=False, downloadOnly=False, execute_inputs:list
             if not isGui():
                 input('')
         newlyListedOnly = False
+        vectorSearch = False
 
 
 if __name__ == "__main__":
@@ -474,7 +527,7 @@ def main(testing=False, testBuild=False, downloadOnly=False, execute_inputs:list
             while True:
                 main()
         except Exception as e:
-            # raise e
+            raise e
             if isDevVersion == OTAUpdater.developmentVersion:
                 raise(e)
             input(colorText.BOLD + colorText.FAIL +