commit for now

nexa/gguf/lib_utils.py

@@ -5,27 +5,6 @@
 from pathlib import Path
 from typing import List, Optional
 import pathlib
-from nexa.utils import (
-    is_nexa_cuda_installed,
-    is_nexa_metal_installed,
-    try_add_cuda_lib_path,
-)
-
-
-# Determine the appropriate library folder
-def _determine_lib_folder(lib_folder: Optional[str]) -> str:
-    if lib_folder:
-        valid_lib_folders = ["cpu", "cuda", "metal"]
-        assert (
-            lib_folder in valid_lib_folders
-        ), f"lib_folder must be one of {valid_lib_folders}"
-        return lib_folder
-
-    if is_nexa_cuda_installed():
-        return "cuda"
-    elif is_nexa_metal_installed():
-        return "metal"
-    return "cpu"
 
 # Load the library
 def _load_shared_library(lib_base_name: str):

nexa/gguf/nexa_inference_vlm.py

@@ -210,6 +210,11 @@ def _chat(self, user_input: str, image_path: str = None) -> Iterator:
                 "content": content,
             },
         ]
+        print(messages)
+        print(model_path)
+        print(self.projector)
+        print(self.model)
+        print(self.stop_words)
 
         return self.model.create_chat_completion(
             messages=messages,

tests/test_image_generation.py

@@ -44,6 +44,6 @@ def test_img_to_img():
         output[0].save("output_img_to_img.png")
 
 # Main execution
-# if __name__ == "__main__":
-    # test_txt_to_img()
-    # test_img_to_img()
+if __name__ == "__main__":
+    test_txt_to_img()
+    test_img_to_img()

tests/test_vlm.py

@@ -16,20 +16,23 @@ def image_to_base64_data_uri(file_path):
 
 model_url = "https://nexa-model-hub-bucket.s3.us-west-1.amazonaws.com/public/nanoLLaVA/model-fp16.gguf"
 mmproj_url = "https://nexa-model-hub-bucket.s3.us-west-1.amazonaws.com/public/nanoLLaVA/projector-fp16.gguf"
+img_url = 'https://upload.wikimedia.org/wikipedia/commons/4/47/PNG_transparency_demonstration_1.png'
 
 # Download paths
 output_dir = os.getcwd()
 model_path = download_model(model_url, output_dir)
 mmproj_path = download_model(mmproj_url, output_dir)
+image_path = download_model(img_url, output_dir)
 print("Model downloaded to:", model_path)
 print("MMProj downloaded to:", mmproj_path)
+print("Image downloaded to:", image_path)
 
 chat_handler = NanoLlavaChatHandler(clip_model_path=mmproj_path)
 
 def test_image_generation():
     llm = llama.Llama(
         model_path=model_path,
-        chat_handler=chat_handler,
+        chat_handler=(chat_handler),
         n_ctx=2048,  # n_ctx should be increased to accommodate the image embedding
         n_gpu_layers=-1,  # Uncomment to use GPU acceleration
         verbose=False,
@@ -43,26 +46,27 @@ def test_image_generation():
             {
                 "role": "user",
                 "content": [
-                    {"type": "text", "text": "What's in this image?"},
                     {
                         "type": "image_url",
                         "image_url": {
-                            "url": "https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg"
+                            "url": image_to_base64_data_uri(image_path)
                         },
-                    },
+                    },                    
+                    {"type": "text", "text": "What's in this image?"},
                 ],
             },
         ],
-        stream=True,
+        # stream=True,
     )
-    for chunk in output:
-        delta = chunk["choices"][0]["delta"]
-        if "role" in delta:
-            print(delta["role"], end=": ")
-        elif "content" in delta:
-            print(delta["content"], end="")
+    # for chunk in output:
+    #     delta = chunk["choices"][0]["delta"]
+    #     if "role" in delta:
+    #         print(delta["role"], end=": ")
+    #     elif "content" in delta:
+    #         print(delta["content"], end="")
 
 
-# if __name__ == "__main__":
+if __name__ == "__main__":
+    test_image_generation()
 #     print("=== Testing 1 ===")
 #     test1()