ml6team · RobbeSneyders · Oct 12, 2023 · Oct 12, 2023 · Oct 12, 2023 · Oct 12, 2023
diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml
@@ -4,6 +4,7 @@ on:
   push:
     branches:
       - main
+  workflow_dispatch:
 
 jobs:
   docker:
@@ -17,13 +18,18 @@ jobs:
 
       - name: Set buildx alias
         run: docker buildx install
+
+      - name: Install docker pushrm
+        run: |
+          sudo wget https://github.com/christian-korneck/docker-pushrm/releases/download/v1.9.0/docker-pushrm_linux_amd64 -O /usr/libexec/docker/cli-plugins/docker-pushrm
+          sudo chmod +x /usr/libexec/docker/cli-plugins/docker-pushrm
+          docker pushrm --help
 
       - name: Login to GitHub Container Registry
         uses: docker/login-action@v2
         with:
-          registry: ghcr.io
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_ACCESS_TOKEN }}
 
       - name: Build components
         run: ./scripts/build_components.sh --cache -t $GITHUB_SHA -t dev
diff --git a/.github/workflows/prep-release.yaml b/.github/workflows/prep-release.yaml
@@ -24,12 +24,16 @@ jobs:
       - name: Set buildx alias
         run: docker buildx install
 
+      - name: Install docker pushrm
+        run: |
+          sudo wget https://github.com/christian-korneck/docker-pushrm/releases/download/v1.9.0/docker-pushrm_linux_amd64 -O /usr/libexec/docker/cli-plugins/docker-pushrm
+          sudo chmod +x /usr/libexec/docker/cli-plugins/docker-pushrm
+
       - name: Login to GitHub Container Registry
         uses: docker/login-action@v2
         with:
-          registry: ghcr.io
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_ACCESS_TOKEN }}
 
       - name: Build components
         run: ./scripts/build_components.sh -t $GITHUB_REF_NAME

diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml
@@ -24,12 +24,16 @@ jobs:
     - name: Set buildx alias
       run: docker buildx install
 
+    - name: Install docker pushrm
+      run: |
+        sudo wget https://github.com/christian-korneck/docker-pushrm/releases/download/v1.9.0/docker-pushrm_linux_amd64 -O /usr/libexec/docker/cli-plugins/docker-pushrm
+        sudo chmod +x /usr/libexec/docker/cli-plugins/docker-pushrm
+
     - name: Login to GitHub Container Registry
       uses: docker/login-action@v2
       with:
-        registry: ghcr.io
-        username: ${{ github.actor }}
-        password: ${{ secrets.GITHUB_TOKEN }}
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_ACCESS_TOKEN }}
 
     - name: Tag components
       run: ./scripts/tag_components.sh -o $GITHUB_REF_NAME -n latest

diff --git a/components/caption_images/fondant_component.yaml b/components/caption_images/fondant_component.yaml
@@ -1,6 +1,6 @@
 name: Caption images
 description: This component captions images using a BLIP model from the Hugging Face hub
-image: ghcr.io/ml6team/caption_images:dev
+image: fndnt/caption_images:dev
 
 consumes:
   images:

diff --git a/components/download_images/fondant_component.yaml b/components/download_images/fondant_component.yaml
@@ -8,7 +8,7 @@ description: |
   [resizer](https://github.com/rom1504/img2dataset/blob/main/img2dataset/resizer.py) function 
   from the img2dataset library.
 
-image: ghcr.io/ml6team/download_images:dev
+image: fndnt/download_images:dev
 
 consumes:
   images:

diff --git a/components/embed_images/fondant_component.yaml b/components/embed_images/fondant_component.yaml
@@ -1,6 +1,6 @@
 name: Embed images
 description: Component that generates CLIP embeddings from images
-image: ghcr.io/ml6team/embed_images:dev
+image: fndnt/embed_images:dev
 
 consumes:
   images:

diff --git a/components/embedding_based_laion_retrieval/fondant_component.yaml b/components/embedding_based_laion_retrieval/fondant_component.yaml
@@ -2,7 +2,7 @@ name: Embedding based LAION retrieval
 description: |
   This component retrieves image URLs from LAION-5B based on a set of CLIP embeddings. It can be 
   used to find images similar to the embedded images / captions.
-image: ghcr.io/ml6team/embedding_based_laion_retrieval:dev
+image: fndnt/embedding_based_laion_retrieval:dev
 
 consumes:
   embeddings:

diff --git a/components/filter_comments/fondant_component.yaml b/components/filter_comments/fondant_component.yaml
@@ -1,6 +1,6 @@
 name: Filter comments
 description: Component that filters code based on the code to comment ratio
-image: ghcr.io/ml6team/filter_comments:dev
+image: fndnt/filter_comments:dev
 
 consumes:
   code:

diff --git a/components/filter_image_resolution/fondant_component.yaml b/components/filter_image_resolution/fondant_component.yaml
@@ -1,6 +1,6 @@
 name: Filter image resolution
 description: Component that filters images based on minimum size and max aspect ratio
-image: ghcr.io/ml6team/filter_image_resolution:dev
+image: fndnt/filter_image_resolution:dev
 
 consumes:
   images:

diff --git a/components/filter_line_length/fondant_component.yaml b/components/filter_line_length/fondant_component.yaml
@@ -1,6 +1,6 @@
 name: Filter line length
 description: Component that filters code based on line length
-image: ghcr.io/ml6team/filter_line_length:dev
+image: fndnt/filter_line_length:dev
 
 consumes:
   code:

diff --git a/components/image_cropping/fondant_component.yaml b/components/image_cropping/fondant_component.yaml
@@ -1,5 +1,5 @@
 name: Image cropping
-image: ghcr.io/ml6team/image_cropping:dev
+image: fndnt/image_cropping:dev
 description: |
   This component crops out image borders. This is typically useful when working with graphical 
   images that have single-color borders (e.g. logos, icons, etc.).

diff --git a/components/image_resolution_extraction/fondant_component.yaml b/components/image_resolution_extraction/fondant_component.yaml
@@ -1,6 +1,6 @@
 name: Image resolution extraction
 description: Component that extracts image resolution data from the images
-image: ghcr.io/ml6team/image_resolution_extraction:dev
+image: fndnt/image_resolution_extraction:dev
 
 consumes:
   images:

diff --git a/components/language_filter/fondant_component.yaml b/components/language_filter/fondant_component.yaml
@@ -1,6 +1,6 @@
 name: Filter languages
 description: A component that filters text based on the provided language.
-image: ghcr.io/ml6team/filter_language:latest
+image: fndnt/filter_language:latest
 
 consumes:
   text:

diff --git a/components/load_from_files/fondant_component.yaml b/components/load_from_files/fondant_component.yaml
@@ -2,7 +2,7 @@ name: Load from files
 description: |
   This component loads data from files in a local or remote (AWS S3, Azure Blob storage, GCS) 
   location. It supports the following formats: .zip, gzip, tar and tar.gz.
-image: ghcr.io/ml6team/load_from_files:dev
+image: fndnt/load_from_files:dev
 
 produces:
   file:

diff --git a/components/load_from_hf_hub/fondant_component.yaml b/components/load_from_hf_hub/fondant_component.yaml
@@ -1,6 +1,6 @@
 name: Load from hub
 description: Component that loads a dataset from the hub
-image: ghcr.io/ml6team/load_from_hf_hub:dev
+image: fndnt/load_from_hf_hub:dev
 
 produces:
   dummy_variable:  #TODO: fill in here

diff --git a/components/load_from_parquet/fondant_component.yaml b/components/load_from_parquet/fondant_component.yaml
@@ -1,6 +1,6 @@
 name: Load from parquet
 description: Component that loads a dataset from a parquet uri
-image: ghcr.io/ml6team/load_from_parquet:dev
+image: fndnt/load_from_parquet:dev
 
 produces:
   dummy_variable:  #TODO: fill in here

diff --git a/components/minhash_generator/fondant_component.yaml b/components/minhash_generator/fondant_component.yaml
@@ -1,6 +1,6 @@
 name: MinHash generator
 description: A component that generates minhashes of text.
-image: ghcr.io/ml6team/minhash_generator:latest
+image: fndnt/minhash_generator:latest
 
 consumes:
   text:

diff --git a/components/pii_redaction/fondant_component.yaml b/components/pii_redaction/fondant_component.yaml
@@ -20,7 +20,7 @@ description: |
   PII is replaced by random data which is stored in the `replacements.json` file.
   A component that detects and redacts Personal Identifiable Information (PII) from 
   code.
-image: ghcr.io/ml6team/pii_redaction:dev
+image: fndnt/pii_redaction:dev
 
 consumes:
   code:

diff --git a/components/prompt_based_laion_retrieval/fondant_component.yaml b/components/prompt_based_laion_retrieval/fondant_component.yaml
@@ -5,7 +5,7 @@ description: |
   the prompt sentences and the captions in the LAION dataset. 
 
   This component doesn’t return the actual images, only URLs.
-image: ghcr.io/ml6team/prompt_based_laion_retrieval:dev
+image: fndnt/prompt_based_laion_retrieval:dev
 
 consumes:
   prompts:

diff --git a/components/segment_images/fondant_component.yaml b/components/segment_images/fondant_component.yaml
@@ -1,6 +1,6 @@
 name: Segment images
 description: Component that creates segmentation masks for images using a model from the Hugging Face hub
-image: ghcr.io/ml6team/segment_images:dev
+image: fndnt/segment_images:dev
 
 consumes:
   images:

diff --git a/components/text_length_filter/fondant_component.yaml b/components/text_length_filter/fondant_component.yaml
@@ -1,6 +1,6 @@
 name: Filter text length
 description: A component that filters out text based on their length
-image: ghcr.io/ml6team/filter_text_length:latest
+image: fndnt/filter_text_length:latest
 
 consumes:
   text:

diff --git a/components/text_normalization/fondant_component.yaml b/components/text_normalization/fondant_component.yaml
@@ -1,5 +1,5 @@
 name: Normalize text
-image: ghcr.io/ml6team/text_normalization:latest
+image: fndnt/text_normalization:latest
 description: |
   This component implements several text normalization techniques to clean and preprocess textual 
   data:

diff --git a/components/write_to_hf_hub/fondant_component.yaml b/components/write_to_hf_hub/fondant_component.yaml
@@ -1,6 +1,6 @@
 name: Write to hub
 description: Component that writes a dataset to the hub
-image: ghcr.io/ml6team/write_to_hf_hub:dev
+image: fndnt/write_to_hf_hub:dev
 
 consumes:
   dummy_variable:  #TODO: fill in here

diff --git a/docs/components/generic_component.md b/docs/components/generic_component.md
@@ -33,7 +33,7 @@ The component specification can be modified as follows
 ```yaml
 name: Load from hub
 description: Component that loads a dataset from the hub
-image: ghcr.io/ml6team/load_from_hf_hub:latest
+image: fndnt/load_from_hf_hub:latest
 
 consumes:
   images:
@@ -100,7 +100,7 @@ If we want to write this dataset to a Hugging Face Hub location, we can use the
 ```yaml
 name: Write to hub
 description: Component that writes a dataset to the hub
-image: ghcr.io/ml6team/write_to_hf_hub:latest
+image: fndnt/write_to_hf_hub:latest
 
 consumes:
   images:

diff --git a/docs/guides/build_a_simple_pipeline.md b/docs/guides/build_a_simple_pipeline.md
@@ -57,7 +57,7 @@ Create a folder `component/load_from_hub` and create a `fondant_component.yaml`
 ```yaml
 name: Load from hub
 description: Component that loads a dataset from the hub
-image: ghcr.io/ml6team/load_from_hf_hub:dev
+image: fndnt/load_from_hf_hub:dev
 
 produces:
   images:

diff --git a/examples/pipelines/commoncrawl/components/extract_images_from_warc/fondant_component.yaml b/examples/pipelines/commoncrawl/components/extract_images_from_warc/fondant_component.yaml
@@ -1,6 +1,6 @@
 name: Extract image licenses from warc
 description: A component that extracts images and their licenses from warc files
-image: ghcr.io/ml6team/extract_images_from_warc:d4619b5
+image: fndnt/extract_images_from_warc:dev
 
 consumes:
   warc:

diff --git a/examples/pipelines/commoncrawl/components/read_warc_paths/fondant_component.yaml b/examples/pipelines/commoncrawl/components/read_warc_paths/fondant_component.yaml
@@ -1,6 +1,6 @@
 name: Common crawl download component
 description: A component that downloads parts of the common crawl
-image: ghcr.io/ml6team/read_warc_paths:57404ff
+image: fndnt/read_warc_paths:dev
 
 produces:
   warc:

diff --git a/...s/pipelines/controlnet-interior-design/components/generate_prompts/fondant_component.yaml b/...s/pipelines/controlnet-interior-design/components/generate_prompts/fondant_component.yaml
@@ -1,6 +1,6 @@
 name: Generate prompts
 description: Component that generates a set of seed prompts
-image: ghcr.io/ml6team/generate_prompts:dev
+image: fndnt/generate_prompts:dev
 
 produces:
   prompts:

diff --git a/...ines/controlnet-interior-design/components/write_to_hub_controlnet/fondant_component.yaml b/...ines/controlnet-interior-design/components/write_to_hub_controlnet/fondant_component.yaml
@@ -1,6 +1,6 @@
 name: Write to hub
 description: Component that writes a dataset to the hub
-image: ghcr.io/ml6team/write_to_hf_hub:latest
+image: fndnt/write_to_hf_hub:latest
 
 consumes:
   images:

diff --git a/examples/pipelines/datacomp/components/add_clip_score/fondant_component.yaml b/examples/pipelines/datacomp/components/add_clip_score/fondant_component.yaml
@@ -1,6 +1,6 @@
 name: Add CLIP score
 description: Component that adds the CLIP score
-image: ghcr.io/ml6team/add_clip_score:dev
+image: fndnt/add_clip_score:dev
 
 consumes:
   embeddings:

diff --git a/examples/pipelines/datacomp/components/clean_captions/fondant_component.yaml b/examples/pipelines/datacomp/components/clean_captions/fondant_component.yaml
@@ -1,6 +1,6 @@
 name: Clean captions
 description: Component that filters out bad captions (Empty captions, Captions with weird characters, Captions that are dates)
-image: ghcr.io/ml6team/clean_captions:50f3a97878ac81670ebe624039ff0fcec0542e4f
+image: fndnt/clean_captions:50f3a97878ac81670ebe624039ff0fcec0542e4f
 
 consumes:
   text:

diff --git a/examples/pipelines/datacomp/components/cluster_image_embeddings/fondant_component.yaml b/examples/pipelines/datacomp/components/cluster_image_embeddings/fondant_component.yaml
@@ -1,6 +1,6 @@
 name: Cluster embeddings
 description: Component that applies k-means clustering on subsampled image embeddings
-image: ghcr.io/ml6team/cluster_image_embeddings:latest
+image: fndnt/cluster_image_embeddings:latest
 
 consumes:
   image:

diff --git a/examples/pipelines/datacomp/components/detect_text/fondant_component.yaml b/examples/pipelines/datacomp/components/detect_text/fondant_component.yaml
@@ -1,6 +1,6 @@
 name: Detect text
 description: Component that detects text in images using an mmocr model
-image: ghcr.io/ml6team/detect_text:dev
+image: fndnt/detect_text:dev
 
 consumes:
   images:

diff --git a/examples/pipelines/datacomp/components/filter_clip_score/fondant_component.yaml b/examples/pipelines/datacomp/components/filter_clip_score/fondant_component.yaml
@@ -1,6 +1,6 @@
 name: Filter CLIP score
 description: Component that filters out bad captions (Empty captions, Captions with weird characters, Captions that are dates)
-image: ghcr.io/ml6team/filter_clip_score:dev
+image: fndnt/filter_clip_score:dev
 
 consumes:
   imagetext:

diff --git a/examples/pipelines/datacomp/components/filter_text_complexity/fondant_component.yaml b/examples/pipelines/datacomp/components/filter_text_complexity/fondant_component.yaml
@@ -1,6 +1,6 @@
 name: Filter text complexity
 description: Component that filters text based on their dependency parse complexity and number of actions
-image: ghcr.io/ml6team/filter_text_complexity:dev
+image: fndnt/filter_text_complexity:dev
 
 consumes:
   text:

diff --git a/examples/pipelines/datacomp/components/load_from_hf_hub/fondant_component.yaml b/examples/pipelines/datacomp/components/load_from_hf_hub/fondant_component.yaml
@@ -1,6 +1,6 @@
 name: Load from hub
 description: Component that loads a dataset from the hub
-image: ghcr.io/ml6team/load_from_hf_hub:dev
+image: fndnt/load_from_hf_hub:dev
 
 produces:
   images:

diff --git a/examples/pipelines/datacomp/components/mask_images/fondant_component.yaml b/examples/pipelines/datacomp/components/mask_images/fondant_component.yaml
@@ -1,6 +1,6 @@
 name: Mask images
 description: Component that masks images based on bounding boxes
-image: ghcr.io/ml6team/mask_images:dev
+image: fndnt/mask_images:dev
 
 consumes:
   images:

diff --git a/examples/pipelines/filter-cc-25m/components/load_from_hf_hub/fondant_component.yaml b/examples/pipelines/filter-cc-25m/components/load_from_hf_hub/fondant_component.yaml
@@ -1,6 +1,6 @@
 name: Load from hub
 description: Component that loads a dataset from the hub
-image: ghcr.io/ml6team/load_from_hf_hub:latest
+image: fndnt/load_from_hf_hub:latest
 
 produces:
   images:

diff --git a/...es/pipelines/finetune_stable_diffusion/components/load_from_hf_hub/fondant_component.yaml b/...es/pipelines/finetune_stable_diffusion/components/load_from_hf_hub/fondant_component.yaml
@@ -1,6 +1,6 @@
 name: Load from hub
 description: Component that loads a dataset from the hub
-image: ghcr.io/ml6team/load_from_hf_hub:latest
+image: fndnt/load_from_hf_hub:latest
 
 produces:
   images:

diff --git a/...les/pipelines/finetune_stable_diffusion/components/write_to_hf_hub/fondant_component.yaml b/...les/pipelines/finetune_stable_diffusion/components/write_to_hf_hub/fondant_component.yaml
@@ -1,6 +1,6 @@
 name: Write to hub
 description: Component that writes a dataset to the hub
-image: ghcr.io/ml6team/write_to_hf_hub:latest
+image: fndnt/write_to_hf_hub:latest
 
 consumes:
   images:

diff --git a/examples/pipelines/starcoder/components/load_from_hub/fondant_component.yaml b/examples/pipelines/starcoder/components/load_from_hub/fondant_component.yaml
@@ -1,6 +1,6 @@
 name: Load code dataset from hub
 description: Component that loads the stack dataset from the hub
-image: ghcr.io/ml6team/load_from_hf_hub:latest
+image: fndnt/load_from_hf_hub:latest
 
 produces:
   code: