Fix hub generation with new components location

.pre-commit-config.yaml

@@ -68,5 +68,5 @@ repos:
         name: Generate component READMEs
         language: python
         entry: python scripts/component_readme/generate_readme.py
-        files: ^components/[^/]*/fondant_component.yaml
+        files: ^src/fondant/components/[^/]*/fondant_component.yaml
         additional_dependencies: ["fondant@git+https://github.com/ml6team/fondant@main", "Jinja2==3.1.2"]

docs/components/hub.md

@@ -11,138 +11,133 @@ Below you can find the reusable components offered by Fondant.
 <a id="load_from_csv"></a>
 ??? "Load from csv"
 
-    --8<-- "components/load_from_csv/README.md:1"
+    --8<-- "src/fondant/components/load_from_csv/README.md:1"
 
 <a id="load_from_files"></a>
 ??? "Load from files"
 
-    --8<-- "components/load_from_files/README.md:1"
+    --8<-- "src/fondant/components/load_from_files/README.md:1"
 
 <a id="load_from_hf_hub"></a>
 ??? "Load from Hugging Face hub"
 
-    --8<-- "components/load_from_hf_hub/README.md:1"
+    --8<-- "src/fondant/components/load_from_hf_hub/README.md:1"
 
 <a id="load_from_parquet"></a>
 ??? "Load from parquet"
 
-    --8<-- "components/load_from_parquet/README.md:1"
+    --8<-- "src/fondant/components/load_from_parquet/README.md:1"
 
-<a id="load_with_llamahub"></a>
-??? "Load with LlamaHub"
+<a id="load_from_pdf"></a>
+??? "Load from pdf"
 
-    --8<-- "components/load_with_llamahub/README.md:1"
+    --8<-- "src/fondant/components/load_from_pdf/README.md:1"
 
 **Data retrieval**
 
 <a id="download_images"></a>
 ??? "Download images"
 
-    --8<-- "components/download_images/README.md:1"
-
-<a id="retrieve_from_weaviate"></a>
-??? "retrieve_from_weaviate"
-
-    --8<-- "components/retrieve_from_weaviate/README.md:1"
+    --8<-- "src/fondant/components/download_images/README.md:1"
 
 <a id="retrieve_laion_by_embedding"></a>
 ??? "Retrieve LAION by embedding"
 
-    --8<-- "components/retrieve_laion_by_embedding/README.md:1"
+    --8<-- "src/fondant/components/retrieve_laion_by_embedding/README.md:1"
 
 <a id="retrieve_laion_by_prompt"></a>
 ??? "Retrieve LAION by prompt"
 
-    --8<-- "components/retrieve_laion_by_prompt/README.md:1"
+    --8<-- "src/fondant/components/retrieve_laion_by_prompt/README.md:1"
 
 **Data writing**
 
+<a id="index_aws_opensearch"></a>
+??? "Index AWS OpenSearch"
+
+    --8<-- "src/fondant/components/index_aws_opensearch/README.md:1"
+
 <a id="index_qdrant"></a>
 ??? "Index Qdrant"
 
-    --8<-- "components/index_qdrant/README.md:1"
+    --8<-- "src/fondant/components/index_qdrant/README.md:1"
 
 <a id="index_weaviate"></a>
 ??? "Index Weaviate"
 
-    --8<-- "components/index_weaviate/README.md:1"
+    --8<-- "src/fondant/components/index_weaviate/README.md:1"
+
+<a id="write_to_file"></a>
+??? "Write to file"
+
+    --8<-- "src/fondant/components/write_to_file/README.md:1"
 
 <a id="write_to_hf_hub"></a>
 ??? "Write to Hugging Face hub"
 
-    --8<-- "components/write_to_hf_hub/README.md:1"
+    --8<-- "src/fondant/components/write_to_hf_hub/README.md:1"
 
 **Image processing**
 
 <a id="caption_images"></a>
 ??? "Caption images"
 
-    --8<-- "components/caption_images/README.md:1"
+    --8<-- "src/fondant/components/caption_images/README.md:1"
 
 <a id="crop_images"></a>
 ??? "Crop images"
 
-    --8<-- "components/crop_images/README.md:1"
+    --8<-- "src/fondant/components/crop_images/README.md:1"
 
 <a id="embed_images"></a>
 ??? "Embed images"
 
-    --8<-- "components/embed_images/README.md:1"
+    --8<-- "src/fondant/components/embed_images/README.md:1"
 
 <a id="extract_image_resolution"></a>
 ??? "Extract image resolution"
 
-    --8<-- "components/extract_image_resolution/README.md:1"
+    --8<-- "src/fondant/components/extract_image_resolution/README.md:1"
 
 <a id="filter_image_resolution"></a>
 ??? "Filter image resolution"
 
-    --8<-- "components/filter_image_resolution/README.md:1"
+    --8<-- "src/fondant/components/filter_image_resolution/README.md:1"
 
 <a id="resize_images"></a>
 ??? "Resize images"
 
-    --8<-- "components/resize_images/README.md:1"
+    --8<-- "src/fondant/components/resize_images/README.md:1"
 
 <a id="segment_images"></a>
 ??? "Segment images"
 
-    --8<-- "components/segment_images/README.md:1"
+    --8<-- "src/fondant/components/segment_images/README.md:1"
 
 **Text processing**
 
 <a id="chunk_text"></a>
 ??? "Chunk text"
 
-    --8<-- "components/chunk_text/README.md:1"
+    --8<-- "src/fondant/components/chunk_text/README.md:1"
 
 <a id="embed_text"></a>
 ??? "Embed text"
 
-    --8<-- "components/embed_text/README.md:1"
-
-<a id="evaluate_ragas"></a>
-??? "Evalute ragas"
-
-    --8<-- "components/evaluate_ragas/README.md:1"
+    --8<-- "src/fondant/components/embed_text/README.md:1"
 
 <a id="filter_language"></a>
 ??? "Filter language"
 
-    --8<-- "components/filter_language/README.md:1"
+    --8<-- "src/fondant/components/filter_language/README.md:1"
 
 <a id="filter_text_length"></a>
 ??? "Filter text length"
 
-    --8<-- "components/filter_text_length/README.md:1"
+    --8<-- "src/fondant/components/filter_text_length/README.md:1"
 
 <a id="generate_minhash"></a>
 ??? "Generate minhash"
 
-    --8<-- "components/generate_minhash/README.md:1"
-
-<a id="normalize_text"></a>
-??? "Normalize text"
-
-    --8<-- "components/normalize_text/README.md:1"
+    --8<-- "src/fondant/components/generate_minhash/README.md:1"
 

scripts/component_readme/generate_hub.py

@@ -8,7 +8,7 @@
 
 HUB_FILE = "docs/components/hub.md"
 HUB_TEMPLATE_FILE = "hub_template.md"
-COMPONENTS_DIR = "components"
+COMPONENTS_DIR = "src/fondant/components"
 COMPONENT_SPEC_FILE = "fondant_component.yaml"
 COMPONENT_TYPE_TAGS = [
     "Data loading",

scripts/component_readme/hub_template.md

@@ -13,7 +13,7 @@ Below you can find the reusable components offered by Fondant.
 <a id="{{ component['dir'] }}"></a>
 ??? "{{ component['name'] }}"
 
-    --8<-- "components/{{ component['dir'] }}/README.md:1"
+    --8<-- "src/fondant/components/{{ component['dir'] }}/README.md:1"
 
 {% endfor %}
 {% endfor %}