From 67a3e17852a1e88e2a97b955f54275b5d8edafde Mon Sep 17 00:00:00 2001 From: Neal DeBuhr Date: Tue, 26 Nov 2024 00:53:41 +0000 Subject: [PATCH 01/10] Add CXR Foundation and Derm Foundation model libraries (Google Health AI) --- packages/tasks/src/model-libraries.ts | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/packages/tasks/src/model-libraries.ts b/packages/tasks/src/model-libraries.ts index 9118a1eda..b00630f3e 100644 --- a/packages/tasks/src/model-libraries.ts +++ b/packages/tasks/src/model-libraries.ts @@ -150,6 +150,13 @@ export const MODEL_LIBRARIES_UI_ELEMENTS = { filter: false, countDownloads: `path:"adapter_config.json"`, }, + "cxr-foundation": { + prettyLabel: "CXR Foundation", + repoName: "CXR-Foundation", + repoUrl: "https://github.com/google-health/cxr-foundation", + filter: false, + countDownloads: `path:"precomputed_embeddings/embeddings.npz" OR path:"pax-elixr-b-text/saved_model.pb"`, + }, deepforest: { prettyLabel: "DeepForest", repoName: "deepforest", @@ -173,6 +180,13 @@ export const MODEL_LIBRARIES_UI_ELEMENTS = { snippets: snippets.depth_pro, filter: false, }, + "derm-foundation": { + prettyLabel: "Derm Foundation", + repoName: "Derm-Foundation", + repoUrl: "https://github.com/google-health/derm-foundation", + filter: false, + countDownloads: `path:"scin_dataset_precomputed_embeddings.npz" OR path:"saved_model.pb"`, + }, diffree: { prettyLabel: "Diffree", repoName: "Diffree", From d02c1525558a29030af8c249677769213b50ca18 Mon Sep 17 00:00:00 2001 From: Neal DeBuhr Date: Tue, 26 Nov 2024 11:09:10 -0500 Subject: [PATCH 02/10] Implement the PR review suggestion for Derm Foundation repo name casing Co-authored-by: vb --- packages/tasks/src/model-libraries.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/tasks/src/model-libraries.ts b/packages/tasks/src/model-libraries.ts index b00630f3e..a401cfcde 100644 --- a/packages/tasks/src/model-libraries.ts +++ b/packages/tasks/src/model-libraries.ts @@ -182,7 +182,7 @@ export const MODEL_LIBRARIES_UI_ELEMENTS = { }, "derm-foundation": { prettyLabel: "Derm Foundation", - repoName: "Derm-Foundation", + repoName: "derm-foundation", repoUrl: "https://github.com/google-health/derm-foundation", filter: false, countDownloads: `path:"scin_dataset_precomputed_embeddings.npz" OR path:"saved_model.pb"`, From a9dc7dcd80ea3af8eca6b859114b009b6c389178 Mon Sep 17 00:00:00 2001 From: Neal DeBuhr Date: Tue, 26 Nov 2024 11:09:41 -0500 Subject: [PATCH 03/10] Implement the PR review suggestion for CXR Foundation repo name casing Co-authored-by: vb --- packages/tasks/src/model-libraries.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/tasks/src/model-libraries.ts b/packages/tasks/src/model-libraries.ts index a401cfcde..d8d31a2d4 100644 --- a/packages/tasks/src/model-libraries.ts +++ b/packages/tasks/src/model-libraries.ts @@ -152,7 +152,7 @@ export const MODEL_LIBRARIES_UI_ELEMENTS = { }, "cxr-foundation": { prettyLabel: "CXR Foundation", - repoName: "CXR-Foundation", + repoName: "cxr-foundation", repoUrl: "https://github.com/google-health/cxr-foundation", filter: false, countDownloads: `path:"precomputed_embeddings/embeddings.npz" OR path:"pax-elixr-b-text/saved_model.pb"`, From 1560b82cd4ede56084bc8fe422d573e59a759d36 Mon Sep 17 00:00:00 2001 From: Neal DeBuhr Date: Tue, 24 Dec 2024 03:47:58 +0000 Subject: [PATCH 04/10] Add model library usage snippets to the CXR Foundation and Derm Foundation models (Google Health AI), to improve their Hugging Face Hub pages and ease model adoption --- .../tasks/src/model-libraries-snippets.ts | 58 +++++++++++++++++++ packages/tasks/src/model-libraries.ts | 2 + 2 files changed, 60 insertions(+) diff --git a/packages/tasks/src/model-libraries-snippets.ts b/packages/tasks/src/model-libraries-snippets.ts index bb6ac12c3..d19a13e59 100644 --- a/packages/tasks/src/model-libraries-snippets.ts +++ b/packages/tasks/src/model-libraries-snippets.ts @@ -95,6 +95,11 @@ export const bm25s = (model: ModelData): string[] => [ retriever = BM25HF.load_from_hub("${model.id}")`, ]; +export const cxr_foundation = (model: ModelData): string[] => [ + `# The quick start notebook shows Hugging Face usage: https://github.com/Google-Health/cxr-foundation/blob/master/notebooks/quick_start_with_hugging_face.ipynb +# Other notebooks are also available: https://github.com/Google-Health/cxr-foundation/tree/master/notebooks`, +] + export const depth_anything_v2 = (model: ModelData): string[] => { let encoder: string; let features: string; @@ -168,6 +173,59 @@ focallength_px = prediction["focallength_px"]`; return [installSnippet, inferenceSnippet]; }; +export const derm_foundation = (model: ModelData): string[] => [ + `from PIL import Image +from io import BytesIO +from IPython.display import Image as IPImage, display +from huggingface_hub import from_pretrained_keras +import tensorflow as tf +import matplotlib.pyplot as plt +import requests + +# Load test image from SCIN Dataset +# https://github.com/google-research-datasets/scin +IMAGE_URL = "https://storage.googleapis.com/dx-scin-public-data/dataset/images/3445096909671059178.png" +response = requests.get(IMAGE_URL, stream=True) +# Raise an exception if the request fails +response.raise_for_status() +# Load the image into a PIL Image object +image = Image.open(response.raw) + +buf = BytesIO() +image.convert("RGB").save(buf, "PNG") +image_bytes = buf.getvalue() +# Format input +input_tensor = tf.train.Example( + features=tf.train.Features( + feature={ + "image/encoded": tf.train.Feature( + bytes_list=tf.train.BytesList(value=[image_bytes]) + ) + } + ) +).SerializeToString() + +# Load the model directly from Hugging Face Hub +loaded_model = from_pretrained_keras("google/derm-foundation") + +# Call inference +infer = loaded_model.signatures["serving_default"] +output = infer(inputs=tf.constant([input_tensor])) + +# Extract the embedding vector +embedding_vector = output["embedding"].numpy().flatten() +print("Size of embedding vector:", len(embedding_vector)) + +# Plot the embedding vector +plt.figure(figsize=(12, 4)) +plt.plot(embedding_vector) +plt.title("Embedding Vector") +plt.xlabel("Index") +plt.ylabel("Value") +plt.grid(True) +plt.show()`, +] + const diffusersDefaultPrompt = "Astronaut in a jungle, cold color palette, muted colors, detailed, 8k"; const diffusers_default = (model: ModelData) => [ diff --git a/packages/tasks/src/model-libraries.ts b/packages/tasks/src/model-libraries.ts index d8d31a2d4..83ebed127 100644 --- a/packages/tasks/src/model-libraries.ts +++ b/packages/tasks/src/model-libraries.ts @@ -154,6 +154,7 @@ export const MODEL_LIBRARIES_UI_ELEMENTS = { prettyLabel: "CXR Foundation", repoName: "cxr-foundation", repoUrl: "https://github.com/google-health/cxr-foundation", + snippets: snippets.cxr_foundation, filter: false, countDownloads: `path:"precomputed_embeddings/embeddings.npz" OR path:"pax-elixr-b-text/saved_model.pb"`, }, @@ -184,6 +185,7 @@ export const MODEL_LIBRARIES_UI_ELEMENTS = { prettyLabel: "Derm Foundation", repoName: "derm-foundation", repoUrl: "https://github.com/google-health/derm-foundation", + snippets: snippets.derm_foundation, filter: false, countDownloads: `path:"scin_dataset_precomputed_embeddings.npz" OR path:"saved_model.pb"`, }, From f31df495e82f40ce3150d0d4997e51e707d38a42 Mon Sep 17 00:00:00 2001 From: Neal DeBuhr Date: Thu, 2 Jan 2025 22:23:39 +0000 Subject: [PATCH 05/10] Remove the CXR Foundation model library snippet --- packages/tasks/src/model-libraries-snippets.ts | 5 ----- packages/tasks/src/model-libraries.ts | 1 - 2 files changed, 6 deletions(-) diff --git a/packages/tasks/src/model-libraries-snippets.ts b/packages/tasks/src/model-libraries-snippets.ts index d41c55536..ae4658eed 100644 --- a/packages/tasks/src/model-libraries-snippets.ts +++ b/packages/tasks/src/model-libraries-snippets.ts @@ -95,11 +95,6 @@ export const bm25s = (model: ModelData): string[] => [ retriever = BM25HF.load_from_hub("${model.id}")`, ]; -export const cxr_foundation = (model: ModelData): string[] => [ - `# The quick start notebook shows Hugging Face usage: https://github.com/Google-Health/cxr-foundation/blob/master/notebooks/quick_start_with_hugging_face.ipynb -# Other notebooks are also available: https://github.com/Google-Health/cxr-foundation/tree/master/notebooks`, -] - export const depth_anything_v2 = (model: ModelData): string[] => { let encoder: string; let features: string; diff --git a/packages/tasks/src/model-libraries.ts b/packages/tasks/src/model-libraries.ts index fd9aa38d7..34bb9ddbb 100644 --- a/packages/tasks/src/model-libraries.ts +++ b/packages/tasks/src/model-libraries.ts @@ -170,7 +170,6 @@ export const MODEL_LIBRARIES_UI_ELEMENTS = { prettyLabel: "CXR Foundation", repoName: "cxr-foundation", repoUrl: "https://github.com/google-health/cxr-foundation", - snippets: snippets.cxr_foundation, filter: false, countDownloads: `path:"precomputed_embeddings/embeddings.npz" OR path:"pax-elixr-b-text/saved_model.pb"`, }, From 719c7af9d60e646b0687270727ce0b4ec7281e40 Mon Sep 17 00:00:00 2001 From: Neal DeBuhr Date: Thu, 2 Jan 2025 22:31:42 +0000 Subject: [PATCH 06/10] Simplify the Derm Foundation model library usage snippet - focus on image preprocessing and embeddings computation, not post-processing/visualization --- packages/tasks/src/model-libraries-snippets.ts | 17 +---------------- 1 file changed, 1 insertion(+), 16 deletions(-) diff --git a/packages/tasks/src/model-libraries-snippets.ts b/packages/tasks/src/model-libraries-snippets.ts index ae4658eed..3dfdf4e02 100644 --- a/packages/tasks/src/model-libraries-snippets.ts +++ b/packages/tasks/src/model-libraries-snippets.ts @@ -171,10 +171,8 @@ focallength_px = prediction["focallength_px"]`; export const derm_foundation = (model: ModelData): string[] => [ `from PIL import Image from io import BytesIO -from IPython.display import Image as IPImage, display from huggingface_hub import from_pretrained_keras import tensorflow as tf -import matplotlib.pyplot as plt import requests # Load test image from SCIN Dataset @@ -205,20 +203,7 @@ loaded_model = from_pretrained_keras("google/derm-foundation") # Call inference infer = loaded_model.signatures["serving_default"] -output = infer(inputs=tf.constant([input_tensor])) - -# Extract the embedding vector -embedding_vector = output["embedding"].numpy().flatten() -print("Size of embedding vector:", len(embedding_vector)) - -# Plot the embedding vector -plt.figure(figsize=(12, 4)) -plt.plot(embedding_vector) -plt.title("Embedding Vector") -plt.xlabel("Index") -plt.ylabel("Value") -plt.grid(True) -plt.show()`, +output = infer(inputs=tf.constant([input_tensor]))`, ] const diffusersDefaultPrompt = "Astronaut in a jungle, cold color palette, muted colors, detailed, 8k"; From c4d3af3e3aad7bc109f1687b512db3dff4a763c4 Mon Sep 17 00:00:00 2001 From: Neal DeBuhr Date: Thu, 2 Jan 2025 23:19:49 +0000 Subject: [PATCH 07/10] Add a new usage snippet for the CXR Foundation model library, which utilizes CxrClient make_hugging_face_client --- .../tasks/src/model-libraries-snippets.ts | 20 +++++++++++++++++++ packages/tasks/src/model-libraries.ts | 1 + 2 files changed, 21 insertions(+) diff --git a/packages/tasks/src/model-libraries-snippets.ts b/packages/tasks/src/model-libraries-snippets.ts index 3dfdf4e02..3029291ae 100644 --- a/packages/tasks/src/model-libraries-snippets.ts +++ b/packages/tasks/src/model-libraries-snippets.ts @@ -95,6 +95,26 @@ export const bm25s = (model: ModelData): string[] => [ retriever = BM25HF.load_from_hub("${model.id}")`, ]; +export const cxr_foundation = (model: ModelData): string[] => [ + `# Install library +!git clone https://github.com/Google-Health/cxr-foundation.git +import tensorflow as tf, sys +sys.path.append('cxr-foundation/python/') + +# Install dependencies +major_version = tf.__version__.rsplit(".", 1)[0] +!pip install tensorflow-text=={major_version} pypng && pip install --no-deps pydicom hcls_imaging_ml_toolkit retrying + +# Run inference +from PIL import Image +from clientside.clients import make_hugging_face_client + +cxr_client = make_hugging_face_client('cxr_model') +!wget -nc -q https://upload.wikimedia.org/wikipedia/commons/c/c8/Chest_Xray_PA_3-8-2010.png + +print(cxr_client.get_image_embeddings_from_images([Image.open("Chest_Xray_PA_3-8-2010.png")]))`, +]; + export const depth_anything_v2 = (model: ModelData): string[] => { let encoder: string; let features: string; diff --git a/packages/tasks/src/model-libraries.ts b/packages/tasks/src/model-libraries.ts index 34bb9ddbb..fd9aa38d7 100644 --- a/packages/tasks/src/model-libraries.ts +++ b/packages/tasks/src/model-libraries.ts @@ -170,6 +170,7 @@ export const MODEL_LIBRARIES_UI_ELEMENTS = { prettyLabel: "CXR Foundation", repoName: "cxr-foundation", repoUrl: "https://github.com/google-health/cxr-foundation", + snippets: snippets.cxr_foundation, filter: false, countDownloads: `path:"precomputed_embeddings/embeddings.npz" OR path:"pax-elixr-b-text/saved_model.pb"`, }, From fa66830a5a38867b43484e065366df2425819f5d Mon Sep 17 00:00:00 2001 From: Neal DeBuhr Date: Tue, 7 Jan 2025 18:37:59 +0000 Subject: [PATCH 08/10] Trim down the Derm Foundation usage snippet comments --- packages/tasks/src/model-libraries-snippets.ts | 3 --- 1 file changed, 3 deletions(-) diff --git a/packages/tasks/src/model-libraries-snippets.ts b/packages/tasks/src/model-libraries-snippets.ts index 3029291ae..17bccd8ee 100644 --- a/packages/tasks/src/model-libraries-snippets.ts +++ b/packages/tasks/src/model-libraries-snippets.ts @@ -195,7 +195,6 @@ from huggingface_hub import from_pretrained_keras import tensorflow as tf import requests -# Load test image from SCIN Dataset # https://github.com/google-research-datasets/scin IMAGE_URL = "https://storage.googleapis.com/dx-scin-public-data/dataset/images/3445096909671059178.png" response = requests.get(IMAGE_URL, stream=True) @@ -218,10 +217,8 @@ input_tensor = tf.train.Example( ) ).SerializeToString() -# Load the model directly from Hugging Face Hub loaded_model = from_pretrained_keras("google/derm-foundation") -# Call inference infer = loaded_model.signatures["serving_default"] output = infer(inputs=tf.constant([input_tensor]))`, ] From 60a89d5f6df7ac1d4f5431e7a927191ff080fbfe Mon Sep 17 00:00:00 2001 From: Lucain Date: Wed, 15 Jan 2025 16:30:47 +0100 Subject: [PATCH 09/10] Apply suggestions from code review --- packages/tasks/src/model-libraries-snippets.ts | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/packages/tasks/src/model-libraries-snippets.ts b/packages/tasks/src/model-libraries-snippets.ts index 17bccd8ee..7019b8728 100644 --- a/packages/tasks/src/model-libraries-snippets.ts +++ b/packages/tasks/src/model-libraries-snippets.ts @@ -195,11 +195,7 @@ from huggingface_hub import from_pretrained_keras import tensorflow as tf import requests -# https://github.com/google-research-datasets/scin -IMAGE_URL = "https://storage.googleapis.com/dx-scin-public-data/dataset/images/3445096909671059178.png" -response = requests.get(IMAGE_URL, stream=True) -# Raise an exception if the request fails -response.raise_for_status() +response = requests.get("https://storage.googleapis.com/dx-scin-public-data/dataset/images/3445096909671059178.png") # Load the image into a PIL Image object image = Image.open(response.raw) From 35dc73a327870efc4ad64a9c3878d9818b312ce3 Mon Sep 17 00:00:00 2001 From: Neal DeBuhr Date: Tue, 21 Jan 2025 18:16:22 +0000 Subject: [PATCH 10/10] Update Derm and CXR snippets to the latest state of feedback/discussions --- .../tasks/src/model-libraries-snippets.ts | 42 ++++++++----------- 1 file changed, 18 insertions(+), 24 deletions(-) diff --git a/packages/tasks/src/model-libraries-snippets.ts b/packages/tasks/src/model-libraries-snippets.ts index 8b362fd7c..1f6328ae8 100644 --- a/packages/tasks/src/model-libraries-snippets.ts +++ b/packages/tasks/src/model-libraries-snippets.ts @@ -96,23 +96,26 @@ retriever = BM25HF.load_from_hub("${model.id}")`, ]; export const cxr_foundation = (model: ModelData): string[] => [ - `# Install library -!git clone https://github.com/Google-Health/cxr-foundation.git -import tensorflow as tf, sys + `!git clone https://github.com/Google-Health/cxr-foundation.git +import tensorflow as tf, sys, requests sys.path.append('cxr-foundation/python/') # Install dependencies major_version = tf.__version__.rsplit(".", 1)[0] !pip install tensorflow-text=={major_version} pypng && pip install --no-deps pydicom hcls_imaging_ml_toolkit retrying -# Run inference +# Load image (Stillwaterising, CC0, via Wikimedia Commons) from PIL import Image -from clientside.clients import make_hugging_face_client +from io import BytesIO +image_url = "https://upload.wikimedia.org/wikipedia/commons/c/c8/Chest_Xray_PA_3-8-2010.png" +response = requests.get(image_url, headers={'User-Agent': 'Demo'}, stream=True) +response.raw.decode_content = True # Ensure correct decoding +img = Image.open(BytesIO(response.content)).convert('L') # Convert to grayscale +# Run inference +from clientside.clients import make_hugging_face_client cxr_client = make_hugging_face_client('cxr_model') -!wget -nc -q https://upload.wikimedia.org/wikipedia/commons/c/c8/Chest_Xray_PA_3-8-2010.png - -print(cxr_client.get_image_embeddings_from_images([Image.open("Chest_Xray_PA_3-8-2010.png")]))`, +print(cxr_client.get_image_embeddings_from_images([img]))`, ]; export const depth_anything_v2 = (model: ModelData): string[] => { @@ -189,34 +192,25 @@ focallength_px = prediction["focallength_px"]`; }; export const derm_foundation = (model: ModelData): string[] => [ - `from PIL import Image -from io import BytesIO -from huggingface_hub import from_pretrained_keras -import tensorflow as tf -import requests + `from huggingface_hub import from_pretrained_keras +import tensorflow as tf, requests -response = requests.get("https://storage.googleapis.com/dx-scin-public-data/dataset/images/3445096909671059178.png") -# Load the image into a PIL Image object -image = Image.open(response.raw) - -buf = BytesIO() -image.convert("RGB").save(buf, "PNG") -image_bytes = buf.getvalue() -# Format input +# Load and format input +IMAGE_URL = "https://storage.googleapis.com/dx-scin-public-data/dataset/images/3445096909671059178.png" input_tensor = tf.train.Example( features=tf.train.Features( feature={ "image/encoded": tf.train.Feature( - bytes_list=tf.train.BytesList(value=[image_bytes]) + bytes_list=tf.train.BytesList(value=[requests.get(IMAGE_URL, stream=True).content]) ) } ) ).SerializeToString() +# Load model and run inference loaded_model = from_pretrained_keras("google/derm-foundation") - infer = loaded_model.signatures["serving_default"] -output = infer(inputs=tf.constant([input_tensor]))`, +print(infer(inputs=tf.constant([input_tensor])))`, ] const diffusersDefaultPrompt = "Astronaut in a jungle, cold color palette, muted colors, detailed, 8k";