Merge pull request #6 from harmonydata/catalogue-match-instruments

Catalogue match instruments

.gitignore

@@ -134,4 +134,10 @@ data/
 
 # Cache
 instruments_cache.json
-vectors_cache.json
+vectors_cache.json
+
+# Catalogue data
+huggingface_sentence_transformers_paraphrase_multilingual_MiniLM_L12_v2_embeddings_all_float16.pkl.bz2
+all_questions_ever_seen.json
+all_instruments_preprocessed.json
+instrument_idx_to_question_idxs.json

.gitmodules

@@ -1,7 +1,7 @@
 [submodule "harmony"]
 	path = harmony
 	url = https://github.com/harmonydata/harmony.git
-	branch = main
+	branch = catalogue-match-instruments
 [submodule "mhc_embeddings"]
 	path = mhc_embeddings
 	url = https://github.com/harmonydata/mhc_embeddings.git

README.md

@@ -191,6 +191,9 @@ environment. Make sure to give the service account the required `Vertex AI` role
 
 `TIKA_SERVER_ENDPOINT` - This is the endpoint where `Tika` is served from.
 
+`AZURE_STORAGE_URL` - The Azure Blob storage URL. This is required for downloading the
+catalogue data.
+
 You can ideally set these environment variables to show Harmony where to look for dependencies and data, but it will
 work without it (it will download the sentence transformer from HuggingFace Hub, etc).
 

docker_compose.yaml

@@ -16,6 +16,7 @@ services:
        AZURE_OPENAI_API_KEY:
        AZURE_OPENAI_ENDPOINT:
        TIKA_SERVER_ENDPOINT: http://tika:9998
+       AZURE_STORAGE_URL:
      volumes:
         - harmonyapimount:/data
 

harmony_api/core/settings.py

@@ -45,6 +45,7 @@ class Settings(BaseSettings):
     OPENAI_API_KEY: str | None = os.getenv("OPENAI_API_KEY")
     AZURE_OPENAI_API_KEY: str | None = os.getenv("AZURE_OPENAI_API_KEY")
     AZURE_OPENAI_ENDPOINT: str | None = os.getenv("AZURE_OPENAI_ENDPOINT")
+    AZURE_STORAGE_URL: str | None = os.getenv("AZURE_STORAGE_URL")
     GOOGLE_APPLICATION_CREDENTIALS: dict = GOOGLE_APPLICATION_CREDENTIALS
 
 

harmony_api/dependencies.py

@@ -0,0 +1,24 @@
+from harmony.schemas.requests.text import MatchBody
+from harmony_api import http_exceptions, helpers, constants
+
+
+def model_from_match_body_is_available(match_body: MatchBody) -> bool:
+    """
+    Check model availability.
+    """
+
+    model = match_body.parameters
+    __check_model(model.dict())
+
+    return True
+
+
+def __check_model(model_dict: dict):
+    if model_dict not in constants.ALL_HARMONY_API_MODELS:
+        raise http_exceptions.CouldNotProcessRequestHTTPException(
+            "Could not process request because the model does not exist."
+        )
+    if not helpers.check_model_availability(model_dict):
+        raise http_exceptions.CouldNotProcessRequestHTTPException(
+            "Could not process request because the model is not available."
+        )