Merge pull request #525 from roboflow/florence2-aliases

Add florence2 aliases and bugfix TransformersModel

.github/workflows/test.nvidia_t4.yml

@@ -58,7 +58,7 @@ jobs:
       - name: 🧪 Regression Paligemma - GPU
         id: paligemma_tests
         run: |
-          PORT=9101 API_KEY=${{ secrets.API_KEY }} python3 -m pytest tests/inference/integration_tests/test_paligemma.py
+          PORT=9101 melee_API_KEY=${{ secrets.MELEE_API_KEY }} python3 -m pytest tests/inference/integration_tests/test_paligemma.py
       - name: 🚨 Show server logs on error
         run: docker logs inference-test
         if: ${{ steps.paligemma_tests.outcome != 'success' }}
@@ -67,11 +67,11 @@ jobs:
         if: success() || failure()
       - name: 🔋 Start Test Docker - GPU
         run: |
-          PORT=9101 INFERENCE_SERVER_REPO=roboflow-inference-server-gpu make start_test_docker_gpu_with_roboflow_staging
+          PORT=9101 INFERENCE_SERVER_REPO=roboflow-inference-server-gpu make start_test_docker_gpu
       - name: 🧪 Regression Florence - GPU
         id: florence_tests
         run: |
-          PORT=9101 STAGING_API_KEY_FOR_HACKATON_PROJECT=${{ secrets.STAGING_API_KEY_FOR_HACKATON_PROJECT }} python3 -m pytest tests/inference/integration_tests/test_florence.py
+          PORT=9101 melee_API_KEY=${{ secrets.MELEE_API_KEY }} python3 -m pytest tests/inference/integration_tests/test_florence.py
       - name: 🚨 Show server logs on error
         run: docker logs inference-test
         if: ${{ steps.florence_tests.outcome != 'success' }}

inference/models/aliases.py

@@ -1,7 +1,26 @@
 # We have a duplicate in inference_sdk.http.utils.aliases - please maintain both
 # (to have aliasing work in both libraries)
 
-
+PALIGEMMA_ALIASES = {
+    "paligemma-3b-mix-224": "paligemma-pretrains/1",
+    "paligemma-3b-mix-448": "paligemma-pretrains/20",
+    "paligemma-3b-ft-cococap-224": "paligemma-pretrains/8",
+    "paligemma-3b-ft-screen2words-224": "paligemma-pretrains/9",
+    "paligemma-3b-ft-vqav2-224": "paligemma-pretrains/10",
+    "paligemma-3b-ft-tallyqa-224": "paligemma-pretrains/11",
+    "paligemma-3b-ft-docvqa-224": "paligemma-pretrains/12",
+    "paligemma-3b-ft-ocrvqa-224": "paligemma-pretrains/13",
+    "paligemma-3b-ft-cococap-448": "paligemma-pretrains/14",
+    "paligemma-3b-ft-screen2words-448": "paligemma-pretrains/15",
+    "paligemma-3b-ft-vqav2-448": "paligemma-pretrains/16",
+    "paligemma-3b-ft-tallyqa-448": "paligemma-pretrains/17",
+    "paligemma-3b-ft-docvqa-448": "paligemma-pretrains/18",
+    "paligemma-3b-ft-ocrvqa-448": "paligemma-pretrains/19",
+}
+FLORENCE_ALIASES = {
+    "florence-2-base": "florence-pretrains/1",
+    "florence-2-large": "florence-pretrains/2",
+}
 REGISTERED_ALIASES = {
     "yolov8n-640": "coco/3",
     "yolov8n-1280": "coco/9",
@@ -38,20 +57,8 @@
     "yolov10b-640": "coco/22",
     "yolov10l-640": "coco/23",
     "yolov10x-640": "coco/24",
-    "paligemma-3b-mix-224": "paligemma-pretrains/1",
-    "paligemma-3b-mix-448": "paligemma-pretrains/20",
-    "paligemma-3b-ft-cococap-224": "paligemma-pretrains/8",
-    "paligemma-3b-ft-screen2words-224": "paligemma-pretrains/9",
-    "paligemma-3b-ft-vqav2-224": "paligemma-pretrains/10",
-    "paligemma-3b-ft-tallyqa-224": "paligemma-pretrains/11",
-    "paligemma-3b-ft-docvqa-224": "paligemma-pretrains/12",
-    "paligemma-3b-ft-ocrvqa-224": "paligemma-pretrains/13",
-    "paligemma-3b-ft-cococap-448": "paligemma-pretrains/14",
-    "paligemma-3b-ft-screen2words-448": "paligemma-pretrains/15",
-    "paligemma-3b-ft-vqav2-448": "paligemma-pretrains/16",
-    "paligemma-3b-ft-tallyqa-448": "paligemma-pretrains/17",
-    "paligemma-3b-ft-docvqa-448": "paligemma-pretrains/18",
-    "paligemma-3b-ft-ocrvqa-448": "paligemma-pretrains/19",
+    **PALIGEMMA_ALIASES,
+    **FLORENCE_ALIASES,
 }
 
 

inference/models/paligemma/paligemma.py

@@ -15,3 +15,4 @@ class LoRAPaliGemma(LoRATransformerModel):
 
     generation_includes_input = True
     transformers_class = PaliGemmaForConditionalGeneration
+    load_base_from_roboflow = True

inference/models/transformers/transformers.py

@@ -82,7 +82,7 @@ def initialize_model(self):
             .to(self.dtype)
         )
 
-        self.processor = AutoProcessor.from_pretrained(
+        self.processor = self.processor_class.from_pretrained(
             self.cache_dir, token=self.huggingface_token
         )
 

inference_sdk/http/utils/aliases.py

@@ -50,6 +50,8 @@
     "paligemma-3b-ft-tallyqa-448": "paligemma-pretrains/17",
     "paligemma-3b-ft-docvqa-448": "paligemma-pretrains/18",
     "paligemma-3b-ft-ocrvqa-448": "paligemma-pretrains/19",
+    "florence-2-base": "florence-pretrains/1",
+    "florence-2-large": "florence-pretrains/2",
 }
 
 

tests/inference/integration_tests/conftest.py

@@ -44,6 +44,9 @@ def server_url() -> str:
 def clean_loaded_models_fixture() -> None:
     on_demand_clean_loaded_models()
 
+@pytest.fixture()
+def clean_loaded_models_every_test_fixture() -> None:
+    on_demand_clean_loaded_models()
 
 def on_demand_clean_loaded_models() -> None:
     response = requests.post(f"{base_url}:{port}/model/clear")

tests/inference/integration_tests/test_florence.py

@@ -5,14 +5,54 @@
 
 from tests.inference.integration_tests.regression_test import bool_env
 
-api_key = os.environ.get("STAGING_API_KEY_FOR_HACKATON_PROJECT")
+# Keep up to date with inference.models.aliases.FLORENCE_ALIASES
+# Can't import because adds a lot of requirements to testing environment
+FLORENCE_ALIASES = {
+    "florence-2-base": "florence-pretrains/1",
+    "florence-2-large": "florence-pretrains/2",
+}
 
+api_key = os.environ.get("melee_API_KEY")
 
+
+@pytest.mark.skipif(
+    bool_env(os.getenv("SKIP_FLORENCE_TEST", False))
+    or bool_env(os.getenv("SKIP_LMM_TEST", False)),
+    reason="Skipping Florence test",
+)
+def test_florence_lora_inference(server_url: str, clean_loaded_models_fixture) -> None:
+    # given
+    payload = {
+        "api_key": api_key,
+        "image": {
+            "type": "url",
+            "value": "https://media.roboflow.com/dog.jpeg",
+        },
+        "prompt": "<CAPTION>",
+        "model_id": "melee/40",
+    }
+
+    # when
+    response = requests.post(
+        f"{server_url}/infer/lmm",
+        json=payload,
+    )
+
+    # then
+    response.raise_for_status()
+    data = response.json()
+    assert len(data["response"]) > 0, "Expected non empty generatiom"
+
+
+@pytest.mark.parametrize("model_id", FLORENCE_ALIASES.keys())
 @pytest.mark.skipif(
-    bool_env(os.getenv("SKIP_FLORENCE_TEST", False)) or bool_env(os.getenv("SKIP_LMM_TEST", False)),
+    bool_env(os.getenv("SKIP_FLORENCE_TEST", False))
+    or bool_env(os.getenv("SKIP_LMM_TEST", False)),
     reason="Skipping Florence test",
 )
-def test_florence_inference(server_url: str, clean_loaded_models_fixture) -> None:
+def test_florence_inference(
+    model_id: str, server_url: str, clean_loaded_models_every_test_fixture
+) -> None:
     # given
     payload = {
         "api_key": api_key,
@@ -21,7 +61,7 @@ def test_florence_inference(server_url: str, clean_loaded_models_fixture) -> Non
             "value": "https://media.roboflow.com/dog.jpeg",
         },
         "prompt": "<CAPTION>",
-        "model_id": "beer-can-hackathon/127"
+        "model_id": model_id,
     }
 
     # when

tests/inference/integration_tests/test_paligemma.py

@@ -5,15 +5,25 @@
 
 from tests.inference.integration_tests.regression_test import bool_env
 
-api_key = os.environ.get("API_KEY")
+# Keep up to date with inference.models.aliases.PALIGEMMA_ALIASES
+# Can't import because adds a lot of requirements to testing environment
+PALIGEMMA_ALIASES = {
+    "paligemma-3b-mix-224": "paligemma-pretrains/1",
+    "paligemma-3b-mix-448": "paligemma-pretrains/20",
+}
 
+api_key = os.environ.get("melee_API_KEY")
 
 
 @pytest.mark.skipif(
-    bool_env(os.getenv("SKIP_PALIGEMMA_TEST", False)) or bool_env(os.getenv("SKIP_LMM_TEST", False)),
+    bool_env(os.getenv("SKIP_PALIGEMMA_TEST", False))
+    or bool_env(os.getenv("SKIP_LMM_TEST", False)),
     reason="Skipping Paligemma test",
 )
-def test_paligemma_inference(server_url: str, clean_loaded_models_fixture) -> None:
+@pytest.mark.parametrize("model_id", PALIGEMMA_ALIASES.keys())
+def test_paligemma_inference(
+    model_id: str, server_url: str, clean_loaded_models_every_test_fixture
+) -> None:
     # given
     payload = {
         "api_key": api_key,
@@ -22,7 +32,36 @@ def test_paligemma_inference(server_url: str, clean_loaded_models_fixture) -> No
             "value": "https://media.roboflow.com/dog.jpeg",
         },
         "prompt": "Describe the image",
-        "model_id": "paligemma-3b-mix-224"
+        "model_id": model_id,
+    }
+
+    # when
+    response = requests.post(
+        f"{server_url}/infer/lmm",
+        json=payload,
+    )
+
+    # then
+    response.raise_for_status()
+    data = response.json()
+    assert len(data["response"]) > 0, "Expected non empty generatiom"
+
+
+@pytest.mark.skipif(
+    bool_env(os.getenv("SKIP_PALIGEMMA_TEST", False))
+    or bool_env(os.getenv("SKIP_LMM_TEST", False)),
+    reason="Skipping Paligemma test",
+)
+def test_paligemma_lora_inference(server_url: str, clean_loaded_models_fixture) -> None:
+    # given
+    payload = {
+        "api_key": api_key,
+        "image": {
+            "type": "url",
+            "value": "https://media.roboflow.com/dog.jpeg",
+        },
+        "prompt": "Describe the image",
+        "model_id": "melee/41",
     }
 
     # when