fix text blob urls not replaced with file contents for non-LLM models

bpm-ai/bpm_ai/common/multimodal.py

@@ -1,3 +1,4 @@
+from bpm_ai_core.llm.common.blob import Blob
 from bpm_ai_core.ocr.ocr import OCR
 from bpm_ai_core.speech_recognition.asr import ASRModel
 from bpm_ai_core.util.file import is_supported_audio_file, is_file, is_supported_text_file
@@ -45,6 +46,14 @@ def prepare_text_blobs(input_data: dict):
     }
 
 
+async def replace_text_blobs(input_data: dict):
+    return {
+        k: (await Blob.from_path_or_url(v).as_bytes()).decode("utf-8")
+        if (isinstance(v, str) and is_supported_text_file(v))
+        else v for k, v in input_data.items()
+    }
+
+
 def assert_all_files_processed(input_data: dict):
     for v in input_data.values():
         if v and isinstance(v, str) and is_file(v):

bpm-ai/bpm_ai/decide/decide.py

@@ -12,7 +12,7 @@
 
 from bpm_ai.common.errors import MissingParameterError
 from bpm_ai.common.multimodal import transcribe_audio, prepare_images_for_llm_prompt, ocr_documents, prepare_text_blobs, \
-    assert_all_files_processed
+    assert_all_files_processed, replace_text_blobs
 from bpm_ai.decide.schema import get_cot_decision_output_schema, get_decision_output_schema, remove_order_prefix_from_keys
 
 
@@ -118,7 +118,7 @@ async def decide_classifier(
     else:  # text classifier
         input_data = await ocr_documents(input_data, ocr)
         input_data = await transcribe_audio(input_data, asr)
-        input_data = prepare_text_blobs(input_data)
+        input_data = await replace_text_blobs(input_data)
         assert_all_files_processed(input_data)
 
         input_md = dict_to_md(input_data).strip()

bpm-ai/bpm_ai/extract/extract.py

@@ -16,7 +16,7 @@
 
 from bpm_ai.common.errors import MissingParameterError
 from bpm_ai.common.multimodal import transcribe_audio, prepare_images_for_llm_prompt, ocr_documents, prepare_text_blobs, \
-    assert_all_files_processed
+    assert_all_files_processed, replace_text_blobs
 from bpm_ai.extract.util import merge_dicts, strip_non_numeric_chars, create_json_object
 
 
@@ -94,7 +94,7 @@ async def extract_qa(
     else:
         input_data = await ocr_documents(input_data, ocr)
     input_data = await transcribe_audio(input_data, asr)
-    input_data = prepare_text_blobs(input_data)
+    input_data = await replace_text_blobs(input_data)
     assert_all_files_processed(input_data)
 
     if not output_schema:
@@ -131,7 +131,7 @@ async def extract_value(text: str, field_name: str, field_type: str, description
 
         if field_type == "integer":
             try:
-                return int(strip_non_numeric_chars(qa_result.answer))
+                return int(strip_non_numeric_chars(qa_result.answer))  # todo should also accept and round floats
             except ValueError:
                 return None
         elif field_type == "number":

bpm-ai/bpm_ai/translate/translate.py

@@ -7,7 +7,7 @@
 
 from bpm_ai.common.errors import MissingParameterError, LanguageNotFoundError
 from bpm_ai.common.multimodal import transcribe_audio, prepare_images_for_llm_prompt, ocr_documents, prepare_text_blobs, \
-    assert_all_files_processed
+    assert_all_files_processed, replace_text_blobs
 from bpm_ai.translate.util import get_translation_output_schema, get_lang_code
 
 
@@ -69,7 +69,7 @@ async def translate_nmt(
 
     input_items = await ocr_documents(input_items, ocr)
     input_items = await transcribe_audio(input_items, asr)
-    input_data = prepare_text_blobs(input_data)
+    input_data = await replace_text_blobs(input_data)
     assert_all_files_processed(input_data)
 
     try:

bpm-ai/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "bpm-ai"
-version = "1.7.1"
+version = "1.7.2"
 description = "AI task automation for BPM engines."
 authors = ["Bennet Krause <[email protected]>"]
 repository = "https://github.com/holunda-io/bpm-ai"
@@ -22,7 +22,7 @@ azure-ai-documentintelligence = "^1.0.0b2"
 av = "^11.0.0"
 
 [tool.poetry.group.dev.dependencies]
-bpm-ai-inference = "0.3.0"
+bpm-ai-inference = "0.3.3"
 ctranslate2 = "4.1.0"
 
 

bpm-ai/tests/conftest.py

@@ -6,6 +6,7 @@
 from bpm_ai_core.llm.anthropic_chat.anthropic_chat import ChatAnthropic
 from bpm_ai_core.llm.openai_chat.openai_chat import ChatOpenAI
 from bpm_ai_core.util.rpc import remote_object
+from bpm_ai_inference.llm.llama_cpp.llama_chat import ChatLlamaCpp
 from bpm_ai_inference.util.hf import hf_home
 
 
@@ -19,7 +20,7 @@ def local_llm():
 
 @pytest.fixture
 def llm():
-    return None #remote_object("ChatLlamaCpp", "0.0.0.0", 6666, model="QuantFactory/Phi-3-mini-4k-instruct-GGUF")
+    return None #ChatLlamaCpp(model="NousResearch/Hermes-2-Pro-Llama-3-8B-GGUF")
 
 
 @pytest.fixture(autouse=True, scope="module")

bpm-ai/tests/test_decide.py

@@ -238,3 +238,16 @@ async def test_decide_classifier_none():
 
     assert result["decision"] is None
     assert result["reasoning"] == "No input values present."
+
+
+async def test_decide_classifier_textfile():
+    classifier = TransformersClassifier()
+
+    result = await decide_classifier(
+        classifier=classifier,
+        input_data={"doc": "files/document.txt"},
+        possible_values=["invoice", "letter"],
+        output_type="string"
+    )
+
+    assert result["decision"] is "invoice"