From 45ace3433642f9a8ed0b2560dd4f316c2754f9cb Mon Sep 17 00:00:00 2001 From: pranjal-joshi Date: Thu, 5 Oct 2023 14:30:50 +0530 Subject: [PATCH] v2.04 - Moved to TA-Lib-Precompiled (0.4.25), Progressbar for GUI - TA-Lib-Precompiled added instead of building from source for faster docker builds - TA-Lib-Precompiled(v0.4.25) now replaces TA-Lib installation using source for test and build workflows, test workflow moved to ubuntu runner - test runner downgraded to ubuntu-20.04 - OTA checking cached for every 1h to eliminate Github API quota - progressbar added to GUI, counter shared from env vars - progressbar errors handled --- .github/workflows/workflow-build-matrix.yml | 4 +- .github/workflows/workflow-test.yml | 17 +----- Dockerfile | 15 +++--- requirements.txt | 3 +- src/screenipy.py | 4 +- src/streamlit_app.py | 58 +++++++++++++++++---- 6 files changed, 64 insertions(+), 37 deletions(-) diff --git a/.github/workflows/workflow-build-matrix.yml b/.github/workflows/workflow-build-matrix.yml index 091be3f7..5195563c 100644 --- a/.github/workflows/workflow-build-matrix.yml +++ b/.github/workflows/workflow-build-matrix.yml @@ -14,8 +14,8 @@ on: jobs: # Job for builing packages - Build-Executables: - name: Build Packages (Disabled onwards v2.03) + Build-Executables-(Disabled onwards v2.03): + name: Build Packages #needs: Create-Release runs-on: ${{ matrix.os }} strategy: diff --git a/.github/workflows/workflow-test.yml b/.github/workflows/workflow-test.yml index 90792575..10952d98 100644 --- a/.github/workflows/workflow-test.yml +++ b/.github/workflows/workflow-test.yml @@ -17,7 +17,7 @@ jobs: # Test the source-code Test-Source: - runs-on: windows-latest + runs-on: ubuntu-20.04 steps: - uses: actions/checkout@v4 @@ -35,25 +35,10 @@ jobs: restore-keys: | ${{ runner.os }}-pip- - - name: Install dependencies for TA-Lib - run: | - python -m pip install --upgrade pip - cd .github/dependencies/ - echo %cd% - pip install TA_Lib-0.4.19-cp39-cp39-win_amd64.whl - - # - name: Install Numpy-MKL for Windows - # run: | - # echo Downloading Numpy-MKL (212MB), Please wait! - # powershell.exe -Command (new-object System.Net.WebClient).DownloadFile('https://download.lfd.uci.edu/pythonlibs/w4tscw6k/numpy-1.20.2+mkl-cp39-cp39-win_amd64.whl','numpy-1.20.2+mkl-cp39-cp39-win_amd64.whl') - # pip install numpy-1.20.2+mkl-cp39-cp39-win_amd64.whl - - name: Install dependencies using pip run: | python -m pip install --upgrade pip pip install flake8 pytest pytest-mock - pip install numpy==1.21.0 - pip install ta-lib pip install -r requirements.txt - name: Lint with flake8 diff --git a/Dockerfile b/Dockerfile index 30cc5a75..c2ba423d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -29,17 +29,18 @@ WORKDIR /opt/program RUN chmod +x * -WORKDIR /opt/program/.github/dependencies/ -RUN tar -xzf ta-lib-0.4.0-src.tar.gz +# Removed build ta-lib from source as we're using [pip3 install TA-Lib-Precompiled] for faster docker build -WORKDIR /opt/program/.github/dependencies/ta-lib/ -RUN ./configure --prefix=/usr --build=$(uname -m)-unknown-linux-gnu -RUN make -RUN make install +# WORKDIR /opt/program/.github/dependencies/ +# RUN tar -xzf ta-lib-0.4.0-src.tar.gz + +# WORKDIR /opt/program/.github/dependencies/ta-lib/ +# RUN ./configure --prefix=/usr --build=$(uname -m)-unknown-linux-gnu +# RUN make +# RUN make install WORKDIR /opt/program/ RUN python3 -m pip install --upgrade pip -# RUN pip3 install ta-lib==0.4.24 RUN pip3 install -r "requirements.txt" diff --git a/requirements.txt b/requirements.txt index b8521039..3e5d5c6b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -19,7 +19,8 @@ pytest-mock pytoml retrying scipy==1.11.2 -ta-lib +# ta-lib +TA-Lib-Precompiled tabulate yfinance==0.1.87 alive-progress==1.6.2 diff --git a/src/screenipy.py b/src/screenipy.py index c15d105f..d1cf2c31 100644 --- a/src/screenipy.py +++ b/src/screenipy.py @@ -407,7 +407,8 @@ def main(testing=False, testBuild=False, downloadOnly=False, execute_inputs:list for _ in range(multiprocessing.cpu_count()): tasks_queue.put(None) try: - numStocks = len(listStockCodes) + numStocks, totalStocks = len(listStockCodes), len(listStockCodes) + os.environ['SCREENIPY_TOTAL_STOCKS'] = str(totalStocks) print(colorText.END+colorText.BOLD) bar, spinner = Utility.tools.getProgressbarStyle() with alive_bar(numStocks, bar=bar, spinner=spinner) as progressbar: @@ -419,6 +420,7 @@ def main(testing=False, testBuild=False, downloadOnly=False, execute_inputs:list saveResults = saveResults.append( result[1], ignore_index=True) numStocks -= 1 + os.environ['SCREENIPY_SCREEN_COUNTER'] = str(int((totalStocks-numStocks)/totalStocks*100)) progressbar.text(colorText.BOLD + colorText.GREEN + f'Found {screenResultsCounter.value} Stocks' + colorText.END) progressbar() diff --git a/src/streamlit_app.py b/src/streamlit_app.py index c311b571..22cfaa7f 100644 --- a/src/streamlit_app.py +++ b/src/streamlit_app.py @@ -6,6 +6,8 @@ import urllib from time import sleep from pathlib import Path +from threading import Thread +from time import sleep import classes.ConfigManager as ConfigManager st.set_page_config(layout="wide", page_title="Screeni-py", page_icon="📈") @@ -25,7 +27,14 @@ except KeyError: proxyServer = "" -isDevVersion, guiUpdateMessage = OTAUpdater.checkForUpdate(proxyServer, VERSION) +isDevVersion, guiUpdateMessage = None, None + +@st.cache_data(ttl='1h', show_spinner=False) +def check_updates(): + isDevVersion, guiUpdateMessage = OTAUpdater.checkForUpdate(proxyServer, VERSION) + return isDevVersion, guiUpdateMessage + +isDevVersion, guiUpdateMessage = check_updates() execute_inputs = [] @@ -64,15 +73,44 @@ def on_start_button_click(): global execute_inputs if isDevVersion != None: st.info(f'Received inputs (Debug only): {execute_inputs}') - with st.spinner('Screening stocks for you...'): - try: - screenipy_main(execute_inputs=execute_inputs) - except StopIteration: - pass - except requests.exceptions.RequestException as e: - st.error('Failed to reach Screeni-py server!', icon='đŸĢ¤') - st.info('This issue is related with your Internet Service Provider (ISP) - Many **Jio** users faced this issue as the screeni-py data cache server appeared to be not reachable for them!\n\nPlease go through this thread carefully to resolve this error: https://github.com/pranjal-joshi/Screeni-py/issues/164', icon='ℹī¸') - st.exception(e) + + def dummy_call(): + try: + screenipy_main(execute_inputs=execute_inputs) + except StopIteration: + pass + except requests.exceptions.RequestException: + os.environ['SCREENIPY_REQ_ERROR'] = "TRUE" + + t = Thread(target=dummy_call) + t.start() + + st.markdown(""" + + """, unsafe_allow_html=True) + + progress_text = "🚀 Preparing Screener, Please Wait! " + progress_bar = st.progress(0, text=progress_text) + + os.environ['SCREENIPY_SCREEN_COUNTER'] = '0' + while int(os.environ.get('SCREENIPY_SCREEN_COUNTER')) < 100: + sleep(0.05) + cnt = int(os.environ.get('SCREENIPY_SCREEN_COUNTER')) + if cnt > 0: + progress_text = "🔍 Screening stocks for you... " + progress_bar.progress(cnt, text=progress_text + f"**:red[{cnt}%]** Done") + if os.environ.get('SCREENIPY_REQ_ERROR') and "TRUE" in os.environ.get('SCREENIPY_REQ_ERROR'): + st.error('Failed to reach Screeni-py server!', icon='đŸĢ¤') + st.info('This issue is related with your Internet Service Provider (ISP) - Many **Jio** users faced this issue as the screeni-py data cache server appeared to be not reachable for them!\n\nTry with another ISP/Network or go through this thread carefully to resolve this error: https://github.com/pranjal-joshi/Screeni-py/issues/164', icon='ℹī¸') + del os.environ['SCREENIPY_REQ_ERROR'] + break + + t.join() + progress_bar.empty() def nifty_predict(col): with col.container():