Add multimodal model support

src/marqo/s2_inference/clip_utils.py

@@ -148,7 +148,9 @@ def download_image_from_url(image_path: str, image_download_headers: dict, timeo
 
     try:
         encoded_url = encode_url(image_path)
+        print(f"encoded url: {encoded_url}")
     except UnicodeEncodeError as e:
+        print(f"error: {e}")
         raise ImageDownloadError(f"Marqo encountered an error when downloading the image url {image_path}. "
                                  f"The url could not be encoded properly. Original error: {e}")
     buffer = BytesIO()

src/marqo/s2_inference/model_registry.py

@@ -1932,6 +1932,45 @@ def _get_onnx_clip_properties() -> Dict:
     }
     return ONNX_CLIP_MODEL_PROPERTIES
 
+def _get_languagebind_properties() -> Dict:
+    LANGUAGEBIND_MODEL_PROPERTIES = {
+        'LanguageBind_Video_V1.5_FT': {
+            "name": "LanguageBind_Video_V1.5_FT",
+            "dimensions": 768,
+            "type": "languagebind",
+            "loader": "languagebind",
+            "supported_modalities": ["video", "text"],
+        },
+        'LanguageBind_Audio_FT': {
+            "name": "LanguageBind_Audio_FT",
+            "dimensions": 768,
+            "type": "languagebind",
+            "loader": "languagebind",
+            "supported_modalities": ["audio", "text"],
+        },
+        'LanguageBind_Image': {
+            "name": "LanguageBind_Image",
+            "dimensions": 768,
+            "type": "languagebind",
+            "loader": "languagebind",
+            "supported_modalities": ["image", "text"],
+        },
+        'LanguageBind_Depth': {
+            "name": "LanguageBind_Depth",
+            "dimensions": 768,
+            "type": "languagebind",
+            "loader": "languagebind",
+            "supported_modalities": ["depth", "text"],
+        },
+        'LanguageBind_Thermal': {
+            "name": "LanguageBind_Thermal",
+            "dimensions": 768,
+            "type": "languagebind",
+            "loader": "languagebind",
+            "supported_modalities": ["thermal", "text"],
+        },
+    }
+    return LANGUAGEBIND_MODEL_PROPERTIES
 
 def _get_fp16_clip_properties() -> Dict:
     FP16_CLIP_MODEL_PROPERTIES = {
@@ -2008,7 +2047,9 @@ def _get_model_load_mappings() -> Dict:
             "fp16_clip": FP16_CLIP,
             'random':Random,
             'hf':HF_MODEL,
-            "no_model": NO_MODEL}
+            "no_model": NO_MODEL,
+            #"languagebind": LANGUAGEBIND,
+            }
 
 def load_model_properties() -> Dict:
     # also truncate the name if not already
@@ -2026,6 +2067,7 @@ def load_model_properties() -> Dict:
     multilingual_clip_model_properties = get_multilingual_clip_properties()
     fp16_clip_model_properties = _get_fp16_clip_properties()
     no_model_properties = _get_no_model_properties()
+    #languagebind_model_properties = _get_languagebind_properties()
 
     # combine the above dicts
     model_properties = dict(clip_model_properties.items())
@@ -2039,6 +2081,7 @@ def load_model_properties() -> Dict:
     model_properties.update(multilingual_clip_model_properties)
     model_properties.update(fp16_clip_model_properties)
     model_properties.update(no_model_properties)
+    #model_properties.update(languagebind_model_properties)
 
 
     all_properties = dict()