chorechore: add model download pattern from hugging face

server/clip_server/model/pretrained_models.py

@@ -2,9 +2,11 @@
 import hashlib
 import shutil
 import urllib
+import requests
 
 
 _OPENCLIP_S3_BUCKET = 'https://clip-as-service.s3.us-east-2.amazonaws.com/models/torch'
+_OPENCLIP_HUGGINGFACE_BUCKET = 'https://huggingface.co/jinaai/'
 _OPENCLIP_MODELS = {
  'RN50::openai': ('RN50.pt', '9140964eaaf9f68c95aa8df6ca13777c'),
  'RN50::yfcc15m': ('RN50-yfcc15m.pt', 'e9c564f91ae7dc754d9043fdcd2a9f22'),
@@ -143,9 +145,29 @@ def get_model_url_md5(name: str):
  if len(model_pretrained) == 0: # not on s3
  return None, None
  else:
+ hg_download_url = _OPENCLIP_HUGGINGFACE_BUCKET + name.split('::')[0] + '/resolve/main/' + model_pretrained[0] + '?download=true'
+ try: 
+ response = requests.head(hg_download_url) 
+ if response.status_code == 200: 
+ print(f"Hugging face download address valid.")
+ return (hg_download_url, model_pretrained[1])
+ else: 
+ print(f"Hugging face download address valid.")
+ return (hg_download_url, model_pretrained[1])
+ except requests.exceptions.RequestException as e:
+ print(str(e))
+ print(f"Model not found on hugging face, trying to download from s3.")
  return (_OPENCLIP_S3_BUCKET + '/' + model_pretrained[0], model_pretrained[1])
 
 
+def get_model_url_md5_hg(name: str):
+ model_pretrained = _OPENCLIP_MODELS[name]
+ if len(model_pretrained) == 0: # not on model list
+ return None, None
+ else:
+ return (_OPENCLIP_HUGGINGFACE_BUCKET + name.split('::')[0] + '/resolve/main/' + model_pretrained[0] + '?download=true', model_pretrained[1])
+
+
 def download_model(
  url: str,
  target_folder: str = os.path.expanduser("~/.cache/clip"),