From 172ed7bfac66dae0c4fd99a8fd9306139c4e68cc Mon Sep 17 00:00:00 2001 From: Robbe Sneyders Date: Mon, 12 Feb 2024 13:50:21 +0100 Subject: [PATCH] Fix hub generation with new components location --- .pre-commit-config.yaml | 2 +- docs/components/hub.md | 75 +++++++++++------------- scripts/component_readme/generate_hub.py | 2 +- scripts/component_readme/hub_template.md | 2 +- 4 files changed, 38 insertions(+), 43 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f53ae280e..582a7c4ee 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -68,5 +68,5 @@ repos: name: Generate component READMEs language: python entry: python scripts/component_readme/generate_readme.py - files: ^components/[^/]*/fondant_component.yaml + files: ^src/fondant/components/[^/]*/fondant_component.yaml additional_dependencies: ["fondant@git+https://github.com/ml6team/fondant@main", "Jinja2==3.1.2"] \ No newline at end of file diff --git a/docs/components/hub.md b/docs/components/hub.md index 36a85d573..85c03c884 100644 --- a/docs/components/hub.md +++ b/docs/components/hub.md @@ -11,138 +11,133 @@ Below you can find the reusable components offered by Fondant. ??? "Load from csv" - --8<-- "components/load_from_csv/README.md:1" + --8<-- "src/fondant/components/load_from_csv/README.md:1" ??? "Load from files" - --8<-- "components/load_from_files/README.md:1" + --8<-- "src/fondant/components/load_from_files/README.md:1" ??? "Load from Hugging Face hub" - --8<-- "components/load_from_hf_hub/README.md:1" + --8<-- "src/fondant/components/load_from_hf_hub/README.md:1" ??? "Load from parquet" - --8<-- "components/load_from_parquet/README.md:1" + --8<-- "src/fondant/components/load_from_parquet/README.md:1" - -??? "Load with LlamaHub" + +??? "Load from pdf" - --8<-- "components/load_with_llamahub/README.md:1" + --8<-- "src/fondant/components/load_from_pdf/README.md:1" **Data retrieval** ??? "Download images" - --8<-- "components/download_images/README.md:1" - - -??? "retrieve_from_weaviate" - - --8<-- "components/retrieve_from_weaviate/README.md:1" + --8<-- "src/fondant/components/download_images/README.md:1" ??? "Retrieve LAION by embedding" - --8<-- "components/retrieve_laion_by_embedding/README.md:1" + --8<-- "src/fondant/components/retrieve_laion_by_embedding/README.md:1" ??? "Retrieve LAION by prompt" - --8<-- "components/retrieve_laion_by_prompt/README.md:1" + --8<-- "src/fondant/components/retrieve_laion_by_prompt/README.md:1" **Data writing** + +??? "Index AWS OpenSearch" + + --8<-- "src/fondant/components/index_aws_opensearch/README.md:1" + ??? "Index Qdrant" - --8<-- "components/index_qdrant/README.md:1" + --8<-- "src/fondant/components/index_qdrant/README.md:1" ??? "Index Weaviate" - --8<-- "components/index_weaviate/README.md:1" + --8<-- "src/fondant/components/index_weaviate/README.md:1" + + +??? "Write to file" + + --8<-- "src/fondant/components/write_to_file/README.md:1" ??? "Write to Hugging Face hub" - --8<-- "components/write_to_hf_hub/README.md:1" + --8<-- "src/fondant/components/write_to_hf_hub/README.md:1" **Image processing** ??? "Caption images" - --8<-- "components/caption_images/README.md:1" + --8<-- "src/fondant/components/caption_images/README.md:1" ??? "Crop images" - --8<-- "components/crop_images/README.md:1" + --8<-- "src/fondant/components/crop_images/README.md:1" ??? "Embed images" - --8<-- "components/embed_images/README.md:1" + --8<-- "src/fondant/components/embed_images/README.md:1" ??? "Extract image resolution" - --8<-- "components/extract_image_resolution/README.md:1" + --8<-- "src/fondant/components/extract_image_resolution/README.md:1" ??? "Filter image resolution" - --8<-- "components/filter_image_resolution/README.md:1" + --8<-- "src/fondant/components/filter_image_resolution/README.md:1" ??? "Resize images" - --8<-- "components/resize_images/README.md:1" + --8<-- "src/fondant/components/resize_images/README.md:1" ??? "Segment images" - --8<-- "components/segment_images/README.md:1" + --8<-- "src/fondant/components/segment_images/README.md:1" **Text processing** ??? "Chunk text" - --8<-- "components/chunk_text/README.md:1" + --8<-- "src/fondant/components/chunk_text/README.md:1" ??? "Embed text" - --8<-- "components/embed_text/README.md:1" - - -??? "Evalute ragas" - - --8<-- "components/evaluate_ragas/README.md:1" + --8<-- "src/fondant/components/embed_text/README.md:1" ??? "Filter language" - --8<-- "components/filter_language/README.md:1" + --8<-- "src/fondant/components/filter_language/README.md:1" ??? "Filter text length" - --8<-- "components/filter_text_length/README.md:1" + --8<-- "src/fondant/components/filter_text_length/README.md:1" ??? "Generate minhash" - --8<-- "components/generate_minhash/README.md:1" - - -??? "Normalize text" - - --8<-- "components/normalize_text/README.md:1" + --8<-- "src/fondant/components/generate_minhash/README.md:1" diff --git a/scripts/component_readme/generate_hub.py b/scripts/component_readme/generate_hub.py index fccb358af..d71e2c3d8 100644 --- a/scripts/component_readme/generate_hub.py +++ b/scripts/component_readme/generate_hub.py @@ -8,7 +8,7 @@ HUB_FILE = "docs/components/hub.md" HUB_TEMPLATE_FILE = "hub_template.md" -COMPONENTS_DIR = "components" +COMPONENTS_DIR = "src/fondant/components" COMPONENT_SPEC_FILE = "fondant_component.yaml" COMPONENT_TYPE_TAGS = [ "Data loading", diff --git a/scripts/component_readme/hub_template.md b/scripts/component_readme/hub_template.md index 3faaf5dc5..249162611 100644 --- a/scripts/component_readme/hub_template.md +++ b/scripts/component_readme/hub_template.md @@ -13,7 +13,7 @@ Below you can find the reusable components offered by Fondant. ??? "{{ component['name'] }}" - --8<-- "components/{{ component['dir'] }}/README.md:1" + --8<-- "src/fondant/components/{{ component['dir'] }}/README.md:1" {% endfor %} {% endfor %} \ No newline at end of file