base-requirements.txt

# Core dependencies
llama-index==0.11.17
llama-index-llms-langchain==0.4.2
langchain==0.3.1
langchain-core==0.3.6
langchain-milvus==0.1.7rc2
langchain-chroma == 0.1.4
langchain-community==0.3.1
langchain-experimental==0.3.2
langgraph==0.2.28
pandasai==2.2.15
pydantic==2.9.2 
pydantic-core==2.23.4
python-dotenv==1.0.1
streamlit==1.37.0
st-pages==0.5.0
streamlit-extras==0.3.6
sseclient-py==1.8.0
torch==2.2.2

# Machine Learning and NLP
scikit-learn==1.4.1.post1
transformers==4.41.2
sentence-transformers==2.2.2
huggingface-hub==0.25.2
instructorembedding==1.0.1
datasets==2.20.0
evaluate==0.4.1
accelerate==0.27.2
peft==0.9.0

# Data processing and storage
pandas==1.5.3
numpy==1.26.0
sqlitedict==2.1.0
faiss-cpu==1.7.4
chromadb==0.4.24
PyMuPDF==1.23.4
PyMuPDFb==1.23.3
pypandoc==1.13
pypandoc_binary==1.13
python-docx==1.1.2
stqdm==0.0.5
xlrd==2.0.1

# Web and API
boto3==1.35.56
requests>=2.32.2
aiohttp==3.9.1
fastapi==0.110.1
uvicorn==0.23.2
google-search-results==2.4.2
nest_asyncio==1.6.0
httpx==0.27.0
backoff==2.2.1
streamlit-javascript==0.1.5

# Visualization
plotly==5.19.0
matplotlib==3.8.0
seaborn==0.12.2
librosa==0.10.1

# Text processing
beautifulsoup4==4.12.3
fpdf2==2.7.9
html2text==2024.2.26
pypdf>=4.0.1,<5.0.0
unstructured[pdf]==0.14.9
unstructured-client==0.18.0
unstructured-inference==0.7.36

# Audio and Video
yt-dlp==2024.8.6

# Miscellaneous
schedule==1.2.2
tqdm>=4.66.3
watchdog==4.0.0
pillow==10.2.0
pycountry==23.12.11
omegaconf==2.3.0
jsonlines==4.0.0

# Data sources
sec-downloader==0.11.1
yfinance==0.2.43

# Development tools
ipykernel==6.29.4
mypy==1.11.2
pre-commit==4.0.1
ruff==0.6.9
ruamel.yaml==0.18.6

# Schedule
schedule==1.2.2

#llm-eval
weave==0.51.1
wandb==0.18.5