Feature/add llama index (#19)

* fix prompt generator check

* onboard llama index

.gitignore

@@ -4,7 +4,7 @@ poetry.lock
 *.bak
 **/__pycache__/**
 **/.DS_Store
-
+storeage/
 # Local sandbox environments
 playground/
-outputs/
+outputs/

pyproject.toml

@@ -20,6 +20,7 @@ pandas = "^2.1.0"
 accelerate = { version = "^0.23.0", optional = true }
 anthropic = { version = "^0.3.10", optional = true }
 datasets = { version = "^2.14.5", optional = true }
+llama-index = { version = "^0.8.29.post1", optional = true }
 openai = { version = "0.27.8", optional = true }
 torch = { version = "^2.0.1", optional = true }
 transformers = { version = "^4.33.1", optional = true }
@@ -30,6 +31,7 @@ anthropic_support = ["anthropic"]
 hf_support = ["accelerate", "datasets", "torch", "transformers"]
 openai_support = ["openai"]
 vllm_support = ["accelerate", "torch", "vllm"]
+llama_index_support = ["llama-index"]
 
 [tool.poetry.group.dev.dependencies]
 black = "^23.3.0"

sciphi/core/base.py

@@ -8,4 +8,5 @@ class ProviderName(Enum):
     OPENAI = "openai"
     ANTHROPIC = "anthropic"
     HUGGING_FACE = "hugging-face"
+    LLAMA_INDEX = "llama-index"
     VLLM = "vllm"

sciphi/core/utils.py

@@ -50,6 +50,12 @@ def get_root_dir() -> str:
 
 
 def get_data_config_dir() -> str:
-    """Get the path to the root of the code repository."""
+    """Get the path to the root of the config directory."""
     script_dir = os.path.dirname(os.path.realpath(__file__))
     return os.path.join(script_dir, "..", "data", "stock_config")
+
+
+def get_data_raw_dir() -> str:
+    """Get the path to the root of the raw data directory."""
+    script_dir = os.path.dirname(os.path.realpath(__file__))
+    return os.path.join(script_dir, "..", "data", "stock_raw")

sciphi/data/stock_raw/learning_with_python/Makefile

@@ -0,0 +1,54 @@
+LATEX = latex
+
+DVIPS = dvips
+
+PDFFLAGS = -dCompatibilityLevel=1.4 -dPDFSETTINGS=/prepress \
+           -dCompressPages=true -dUseFlateCompression=true  \
+           -dEmbedAllFonts=true -dSubsetFonts=true -dMaxSubsetPct=100
+
+%.dvi: %.tex
+	$(LATEX) $<
+
+%.ps: %.dvi
+	$(DVIPS) -o $@ $<
+
+%.pdf: %.ps
+	ps2pdf $(PDFFLAGS) $<
+
+all:	top.tex
+	latex top
+	makeindex top
+	latex top
+	dvips -T 6.75in,9.25in -Ppdf -o thinkCSpy.ps top
+#	dvips -T 7.444in,9.681in -Ppdf -o thinkCSpy.ps top
+	ps2pdf $(PDFFLAGS) thinkCSpy.ps
+
+DEST = /home/downey/public_html/greent/thinkpython/thinkCSpy
+
+distrib:
+	rm -rf dist
+	mkdir dist dist/tex dist/html
+	rsync -a thinkCSpy.pdf thinkCSpy.ps dist
+	rsync -a Makefile *.tex dist/tex
+	rsync -a illustrations dist/tex
+	rm dist/tex/illustrations/*.bak
+	rsync -a images dist/html
+	mkdir dist/html/illustrations
+	rsync -a illustrations/*.png dist/html/illustrations
+	cd dist/tex; make clean
+	cd dist; tar -czf thinkCSpy.tex.tar.gz tex
+	cd ltoh; perl make_book_html.pl ../dist/tex ../dist/html
+	cd dist; tar -czf thinkCSpy.html.tar.gz html
+	cd dist; cp thinkCSpy.tex.tar.gz thinkCSpy.tex.tgz
+	cd dist; cp thinkCSpy.html.tar.gz thinkCSpy.html.tgz
+	rsync -a dist/* $(DEST)
+	chmod -R o+r $(DEST)/*
+
+clean:
+	rm -f *~ *.aux *.log *.dvi *.idx *.ilg *.ind *.toc
+
+
+prop:
+	zip downey_software_design.zip proposal/proposal.pdf proposal/table_of_contents.pdf proposal/chapters12-14.pdf
+
+