better meilisearch handling

huggingface · Jul 2, 2024 · a105d1f · a105d1f
1 parent ed8cee7
commit a105d1f
Show file tree

Hide file tree

Showing 3 changed files with 16 additions and 6 deletions.
diff --git a/.github/workflows/build_embeddings.yml b/.github/workflows/build_embeddings.yml
@@ -75,12 +75,18 @@ jobs:
   cleanup-job:
     needs: matrix-job
     runs-on: ubuntu-latest
+    if: always() # This ensures that the cleanup job runs regardless of the result of matrix-job
     steps:
       - name: Checkout doc-builder
         uses: actions/checkout@v2
-        
+
       - name: Install doc-builder
         run: pip install .[dev]
 
-      - name: Clean meilisearch
-        run: doc-builder meilisearch-clean --meilisearch_key ${{ secrets.MEILISEARCH_KEY }}
+      - name: Success Cleanup
+        if: success() # Runs if all matrix jobs succeeded
+        run: doc-builder meilisearch-clean --meilisearch_key ${{ secrets.MEILISEARCH_KEY }} --swap
+
+      - name: Failure Cleanup
+        if: failure() # Runs if any matrix job failed
+        run: doc-builder meilisearch-clean --meilisearch_key ${{ secrets.MEILISEARCH_KEY }}
diff --git a/src/doc_builder/build_embeddings.py b/src/doc_builder/build_embeddings.py
@@ -473,12 +473,13 @@ def build_embeddings(
         add_embeddings_to_db(client, MEILI_INDEX_TEMP, chunk_embeddings)
 
 
-def clean_meilisearch(meilisearch_key: str):
+def clean_meilisearch(meilisearch_key: str, swap: bool):
     """
     Swap & delete temp index.
     """
     client = meilisearch.Client("https://edge.meilisearch.com", meilisearch_key)
-    swap_indexes(client, MEILI_INDEX, MEILI_INDEX_TEMP)
+    if swap:
+        swap_indexes(client, MEILI_INDEX, MEILI_INDEX_TEMP)
     delete_embedding_db(client, MEILI_INDEX_TEMP)
     create_embedding_db(client, MEILI_INDEX_TEMP)
     update_db_settings(client, MEILI_INDEX_TEMP)

diff --git a/src/doc_builder/commands/embeddings.py b/src/doc_builder/commands/embeddings.py
@@ -127,7 +127,10 @@ def embeddings_command_parser(subparsers=None):
             "Doc Builder meilisearch clean command. Swap & delete the temp index."
         )
     parser_meilisearch_clean.add_argument("--meilisearch_key", type=str, help="Meilisearch key.", required=True)
+    parser_meilisearch_clean.add_argument(
+        "--swap", action="store_true", help="Whether to swap temp index with prod index."
+    )
     if subparsers is not None:
-        parser_meilisearch_clean.set_defaults(func=lambda args: clean_meilisearch(args.meilisearch_key))
+        parser_meilisearch_clean.set_defaults(func=lambda args: clean_meilisearch(args.meilisearch_key, args.swap))
 
     return parser