From 6213fcb0e4d94c2db0d597d020dc726b2c5ea0d8 Mon Sep 17 00:00:00 2001 From: Li Wan Date: Wed, 1 Feb 2023 15:39:26 +1100 Subject: [PATCH 1/6] change open_clip to version 2.10.2 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 300ee233b..530856796 100644 --- a/requirements.txt +++ b/requirements.txt @@ -15,7 +15,7 @@ Pillow==9.3.0 numpy==1.23.4 validators==0.20.0 sentence-transformers==2.2.2 -open_clip_torch==2.9.2 +open_clip_torch==2.10.1 clip-marqo==1.0.2 protobuf==3.20.1 onnx==1.12.0 From 0429c271e83891b51fe38bc397ee5af1c5bc319b Mon Sep 17 00:00:00 2001 From: Li Wan Date: Wed, 1 Feb 2023 16:16:28 +1100 Subject: [PATCH 2/6] add open clip models and tests --- src/marqo/s2_inference/model_registry.py | 47 +++++++++++++++++++++++- tests/s2_inference/test_encoding.py | 18 ++++++--- 2 files changed, 59 insertions(+), 6 deletions(-) diff --git a/src/marqo/s2_inference/model_registry.py b/src/marqo/s2_inference/model_registry.py index 6eec35831..77d65f778 100644 --- a/src/marqo/s2_inference/model_registry.py +++ b/src/marqo/s2_inference/model_registry.py @@ -284,7 +284,52 @@ def _get_open_clip_properties() -> Dict: 'dimensions': 1024, 'note': 'clip model from open_clip implementation', 'type': 'open_clip', - 'pretrained': 'laion2b_s12b_b42k'}} + 'pretrained': 'laion2b_s12b_b42k'}, + + 'open_clip/convnext_base/laion400m_s13b_b51k': + {'name': 'open_clip/convnext_base/laion400m_s13b_b51k', + 'dimensions': 512, + 'note': 'clip model from open_clip implementation', + 'type': 'open_clip', + 'pretrained': 'laion400m_s13b_b51k'}, + + 'open_clip/convnext_base_w/laion2b_s13b_b82k': { + 'name': 'open_clip/convnext_base_w/laion2b_s13b_b82k', + 'dimensions': 640, + 'note': 'clip model from open_clip implementation', + 'type': 'open_clip', + 'pretrained': 'laion2b_s13b_b82k'}, + + 'open_clip/convnext_base_w/laion2b_s13b_b82k_augreg': { + 'name': 'open_clip/convnext_base_w/laion2b_s13b_b82k_augreg', + 'dimensions': 640, + 'note': 'clip model from open_clip implementation', + 'type': 'open_clip', + 'pretrained': 'laion2b_s13b_b82k_augreg'}, + + 'open_clip/convnext_base_w/laion_aesthetic_s13b_b82k': { + 'name': 'open_clip/convnext_base_w/laion_aesthetic_s13b_b82k', + 'dimensions': 640, + 'note': 'clip model from open_clip implementation', + 'type': 'open_clip', + 'pretrained': 'laion_aesthetic_s13b_b82k'}, + + 'open_clip/convnext_base_w_320/laion_aesthetic_s13b_b82k': { + 'name': 'open_clip/convnext_base_w_320/laion_aesthetic_s13b_b82k', + 'dimensions': 640, + 'note': 'clip model from open_clip implementation', + 'type': 'open_clip', + 'pretrained': 'laion_aesthetic_s13b_b82k'}, + + 'open_clip/convnext_base_w_320/laion_aesthetic_s13b_b82k_augreg': { + 'name': 'open_clip/convnext_base_w_320/laion_aesthetic_s13b_b82k_augreg', + 'dimensions': 640, + 'note': 'clip model from open_clip implementation', + 'type': 'open_clip', + 'pretrained': 'laion_aesthetic_s13b_b82k_augreg'}, +} + + return OPEN_CLIP_MODEL_PROPERTIES diff --git a/tests/s2_inference/test_encoding.py b/tests/s2_inference/test_encoding.py index 7473dd8ba..1c4a52a13 100644 --- a/tests/s2_inference/test_encoding.py +++ b/tests/s2_inference/test_encoding.py @@ -24,7 +24,9 @@ def tearDown(self) -> None: clear_loaded_models() def test_vectorize(self): - names = ["onnx16/open_clip/ViT-B-32/laion400m_e32", 'onnx32/open_clip/ViT-B-32-quickgelu/laion400m_e32', + names = ["open_clip/convnext_base/laion400m_s13b_b51k","open_clip/convnext_base_w/laion2b_s13b_b82k", + "open_clip/convnext_base_w_320/laion_aesthetic_s13b_b82k_augreg", + "onnx16/open_clip/ViT-B-32/laion400m_e32", 'onnx32/open_clip/ViT-B-32-quickgelu/laion400m_e32', "all-MiniLM-L6-v1", "all_datasets_v4_MiniLM-L6", "hf/all-MiniLM-L6-v1", "hf/all_datasets_v4_MiniLM-L6", "onnx/all-MiniLM-L6-v1", "onnx/all_datasets_v4_MiniLM-L6"] @@ -50,7 +52,8 @@ def test_vectorize(self): def test_load_clip_text_model(self): names = [ "onnx16/open_clip/ViT-B-32/laion400m_e32", 'onnx32/open_clip/ViT-B-32-quickgelu/laion400m_e32', - 'RN50', "ViT-B/16"] + 'RN50', "ViT-B/16", "open_clip/convnext_base/laion400m_s13b_b51k","open_clip/convnext_base_w/laion2b_s13b_b82k", + "open_clip/convnext_base_w_320/laion_aesthetic_s13b_b82k_augreg",] device = 'cpu' eps = 1e-9 @@ -128,7 +131,9 @@ def test_compare_onnx_sbert_text_models(self): def test_model_outputs(self): - names = ["onnx16/open_clip/ViT-B-32/laion400m_e32", 'onnx32/open_clip/ViT-B-32-quickgelu/laion400m_e32', + names = ["open_clip/convnext_base/laion400m_s13b_b51k","open_clip/convnext_base_w/laion2b_s13b_b82k", + "open_clip/convnext_base_w_320/laion_aesthetic_s13b_b82k_augreg", + "onnx16/open_clip/ViT-B-32/laion400m_e32", 'onnx32/open_clip/ViT-B-32-quickgelu/laion400m_e32', 'open_clip/ViT-B-32/laion400m_e32', "all-MiniLM-L6-v1", "all_datasets_v4_MiniLM-L6", "hf/all-MiniLM-L6-v1", "hf/all_datasets_v4_MiniLM-L6", "onnx/all-MiniLM-L6-v1", "onnx/all_datasets_v4_MiniLM-L6"] @@ -199,7 +204,9 @@ def test_model_un_normalization(self): def test_open_clip_vectorize(self): - names = ['open_clip/ViT-B-32/laion400m_e32', 'open_clip/RN50/openai'] + names = ["open_clip/convnext_base/laion400m_s13b_b51k","open_clip/convnext_base_w/laion2b_s13b_b82k", + "open_clip/convnext_base_w_320/laion_aesthetic_s13b_b82k_augreg", + 'open_clip/ViT-B-32/laion400m_e32', 'open_clip/RN50/openai'] sentences = ['hello', 'this is a test sentence. so is this.', ['hello', 'this is a test sentence. so is this.']] device = 'cpu' @@ -222,7 +229,8 @@ def test_open_clip_vectorize(self): def test_open_clip_embedding_size(self): - names = ['open_clip/ViT-B-32/laion400m_e32', 'open_clip/RN50/openai'] + names = ["open_clip/convnext_base/laion400m_s13b_b51k","open_clip/convnext_base_w/laion2b_s13b_b82k", + "open_clip/convnext_base_w_320/laion_aesthetic_s13b_b82k_augreg",'open_clip/ViT-B-32/laion400m_e32', 'open_clip/RN50/openai'] device = "cpu" From 380d6851c86d2adab0578aca2d8c1b29fb5dd2cc Mon Sep 17 00:00:00 2001 From: Li Wan Date: Wed, 1 Feb 2023 16:49:19 +1100 Subject: [PATCH 3/6] remove open_clip/convnext_base/laion400m_s13b_b51k as it is not ready --- src/marqo/s2_inference/model_registry.py | 14 ++++++++------ tests/s2_inference/test_encoding.py | 10 +++++----- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/src/marqo/s2_inference/model_registry.py b/src/marqo/s2_inference/model_registry.py index 77d65f778..5406c5354 100644 --- a/src/marqo/s2_inference/model_registry.py +++ b/src/marqo/s2_inference/model_registry.py @@ -286,12 +286,14 @@ def _get_open_clip_properties() -> Dict: 'type': 'open_clip', 'pretrained': 'laion2b_s12b_b42k'}, - 'open_clip/convnext_base/laion400m_s13b_b51k': - {'name': 'open_clip/convnext_base/laion400m_s13b_b51k', - 'dimensions': 512, - 'note': 'clip model from open_clip implementation', - 'type': 'open_clip', - 'pretrained': 'laion400m_s13b_b51k'}, + # This model is not ready in open_clip yet. + # I will add this later + # 'open_clip/convnext_base/laion400m_s13b_b51k': + # {'name': 'open_clip/convnext_base/laion400m_s13b_b51k', + # 'dimensions': 512, + # 'note': 'clip model from open_clip implementation', + # 'type': 'open_clip', + # 'pretrained': 'laion400m_s13b_b51k'}, 'open_clip/convnext_base_w/laion2b_s13b_b82k': { 'name': 'open_clip/convnext_base_w/laion2b_s13b_b82k', diff --git a/tests/s2_inference/test_encoding.py b/tests/s2_inference/test_encoding.py index 1c4a52a13..0b42256e1 100644 --- a/tests/s2_inference/test_encoding.py +++ b/tests/s2_inference/test_encoding.py @@ -24,7 +24,7 @@ def tearDown(self) -> None: clear_loaded_models() def test_vectorize(self): - names = ["open_clip/convnext_base/laion400m_s13b_b51k","open_clip/convnext_base_w/laion2b_s13b_b82k", + names = ["open_clip/convnext_base_w/laion2b_s13b_b82k", "open_clip/convnext_base_w_320/laion_aesthetic_s13b_b82k_augreg", "onnx16/open_clip/ViT-B-32/laion400m_e32", 'onnx32/open_clip/ViT-B-32-quickgelu/laion400m_e32', "all-MiniLM-L6-v1", "all_datasets_v4_MiniLM-L6", "hf/all-MiniLM-L6-v1", "hf/all_datasets_v4_MiniLM-L6", @@ -52,7 +52,7 @@ def test_vectorize(self): def test_load_clip_text_model(self): names = [ "onnx16/open_clip/ViT-B-32/laion400m_e32", 'onnx32/open_clip/ViT-B-32-quickgelu/laion400m_e32', - 'RN50', "ViT-B/16", "open_clip/convnext_base/laion400m_s13b_b51k","open_clip/convnext_base_w/laion2b_s13b_b82k", + 'RN50', "ViT-B/16", "open_clip/convnext_base_w/laion2b_s13b_b82k", "open_clip/convnext_base_w_320/laion_aesthetic_s13b_b82k_augreg",] device = 'cpu' @@ -131,7 +131,7 @@ def test_compare_onnx_sbert_text_models(self): def test_model_outputs(self): - names = ["open_clip/convnext_base/laion400m_s13b_b51k","open_clip/convnext_base_w/laion2b_s13b_b82k", + names = ["open_clip/convnext_base_w/laion2b_s13b_b82k", "open_clip/convnext_base_w_320/laion_aesthetic_s13b_b82k_augreg", "onnx16/open_clip/ViT-B-32/laion400m_e32", 'onnx32/open_clip/ViT-B-32-quickgelu/laion400m_e32', 'open_clip/ViT-B-32/laion400m_e32', "all-MiniLM-L6-v1", @@ -204,7 +204,7 @@ def test_model_un_normalization(self): def test_open_clip_vectorize(self): - names = ["open_clip/convnext_base/laion400m_s13b_b51k","open_clip/convnext_base_w/laion2b_s13b_b82k", + names = ["open_clip/convnext_base_w/laion2b_s13b_b82k", "open_clip/convnext_base_w_320/laion_aesthetic_s13b_b82k_augreg", 'open_clip/ViT-B-32/laion400m_e32', 'open_clip/RN50/openai'] @@ -229,7 +229,7 @@ def test_open_clip_vectorize(self): def test_open_clip_embedding_size(self): - names = ["open_clip/convnext_base/laion400m_s13b_b51k","open_clip/convnext_base_w/laion2b_s13b_b82k", + names = ["open_clip/convnext_base_w/laion2b_s13b_b82k", "open_clip/convnext_base_w_320/laion_aesthetic_s13b_b82k_augreg",'open_clip/ViT-B-32/laion400m_e32', 'open_clip/RN50/openai'] device = "cpu" From ae7b993c70fe5214dd2d29b267c9be62e6618375 Mon Sep 17 00:00:00 2001 From: Li Wan Date: Wed, 1 Feb 2023 16:57:32 +1100 Subject: [PATCH 4/6] remove open_clip/convnext_base/laion400m_s13b_b51k as it is not ready --- src/marqo/s2_inference/model_registry.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/marqo/s2_inference/model_registry.py b/src/marqo/s2_inference/model_registry.py index 5406c5354..17109eb69 100644 --- a/src/marqo/s2_inference/model_registry.py +++ b/src/marqo/s2_inference/model_registry.py @@ -286,8 +286,8 @@ def _get_open_clip_properties() -> Dict: 'type': 'open_clip', 'pretrained': 'laion2b_s12b_b42k'}, - # This model is not ready in open_clip yet. - # I will add this later + # There is a typo in the current release of open_clip. + # We will add this model once in the next open_clip release. # 'open_clip/convnext_base/laion400m_s13b_b51k': # {'name': 'open_clip/convnext_base/laion400m_s13b_b51k', # 'dimensions': 512, From 8aab37788feaa0c7c0c2479e677a4e1ffd7ab543 Mon Sep 17 00:00:00 2001 From: Li Wan Date: Wed, 1 Feb 2023 17:00:17 +1100 Subject: [PATCH 5/6] remove open_clip/convnext_base/laion400m_s13b_b51k as it is not ready --- src/marqo/s2_inference/model_registry.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/marqo/s2_inference/model_registry.py b/src/marqo/s2_inference/model_registry.py index 17109eb69..cf3d0c70d 100644 --- a/src/marqo/s2_inference/model_registry.py +++ b/src/marqo/s2_inference/model_registry.py @@ -286,6 +286,7 @@ def _get_open_clip_properties() -> Dict: 'type': 'open_clip', 'pretrained': 'laion2b_s12b_b42k'}, + # TODO Uncomment this model in the next open_clip release # There is a typo in the current release of open_clip. # We will add this model once in the next open_clip release. # 'open_clip/convnext_base/laion400m_s13b_b51k': From b78f98580481cb6338a4a43f3ecac3db588ba646 Mon Sep 17 00:00:00 2001 From: Li Wan Date: Wed, 1 Feb 2023 17:39:31 +1100 Subject: [PATCH 6/6] remove space --- src/marqo/s2_inference/model_registry.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/marqo/s2_inference/model_registry.py b/src/marqo/s2_inference/model_registry.py index cf3d0c70d..6db493482 100644 --- a/src/marqo/s2_inference/model_registry.py +++ b/src/marqo/s2_inference/model_registry.py @@ -332,8 +332,6 @@ def _get_open_clip_properties() -> Dict: 'pretrained': 'laion_aesthetic_s13b_b82k_augreg'}, } - - return OPEN_CLIP_MODEL_PROPERTIES