BUG: Auto patch trust remote code for embedding model

xinference/client/tests/test_client.py

@@ -92,15 +92,17 @@ def test_client_for_embedding(setup):
     client = ActorClient(endpoint)
     assert len(client.list_models()) == 0
 
-    model_uid = client.launch_model(model_name="gte-base", model_type="embedding")
+    model_uid = client.launch_model(
+        model_name="jina-embeddings-v2-small-en", model_type="embedding"
+    )
     assert len(client.list_models()) == 1
 
     model = client.get_model(model_uid=model_uid)
     assert isinstance(model, EmbeddingModelHandle)
 
     completion = model.create_embedding("write a poem.")
     completion = json.loads(completion)
-    assert len(completion["data"][0]["embedding"]) == 768
+    assert len(completion["data"][0]["embedding"]) == 512
 
     client.terminate_model(model_uid=model_uid)
     assert len(client.list_models()) == 0

xinference/model/embedding/core.py

@@ -220,6 +220,9 @@ def load(self):
             ]
 
             raise ImportError(f"{error_message}\n\n{''.join(installation_guide)}")
+        from ..utils import patch_trust_remote_code
+
+        patch_trust_remote_code()
         self._model = SentenceTransformer(self._model_path, device=self._device)
 
     def create_embedding(self, sentences: Union[str, List[str]], **kwargs):

xinference/model/utils.py

@@ -211,3 +211,28 @@ def copy_from_src_to_dst(
             )
             if attempt + 1 == max_attempt:
                 raise
+
+
+def patch_trust_remote_code():
+    """sentence-transformers calls transformers without the trust_remote_code=True, some embedding
+    models will fail to load, e.g. jina-embeddings-v2-base-en
+
+    :return:
+    """
+    try:
+        from transformers.dynamic_module_utils import resolve_trust_remote_code
+    except ImportError:
+        logger.error("Patch transformers trust_remote_code failed.")
+    else:
+
+        def _patched_resolve_trust_remote_code(*args, **kwargs):
+            logger.info("Patched resolve_trust_remote_code: %s %s", args, kwargs)
+            return True
+
+        if (
+            resolve_trust_remote_code.__code__
+            != _patched_resolve_trust_remote_code.__code__
+        ):
+            resolve_trust_remote_code.__code__ = (
+                _patched_resolve_trust_remote_code.__code__
+            )