Merge branch 'main' into jinja2-set-vars-incorrect

Author: srini047

.github/utils/create_unstable_docs_docusaurus.py

@@ -0,0 +1,97 @@
+"""
+This script creates an unstable documentation version at the time of branch-off for a new Haystack release.
+
+Between branch-off and the actual release, two unstable doc versions coexist.
+If we branch off for 2.20, we have:
+1. the target unstable version, 2.20-unstable (lives in docs-website/versioned_docs/version-2.20-unstable)
+2. the next unstable version, 2.21-unstable (lives in docs-website/docs)
+
+This script takes care of all the necessary updates to the documentation website.
+"""
+
+import argparse
+import json
+import os
+import re
+import shutil
+import sys
+
+VERSION_VALIDATOR = re.compile(r"^[0-9]+\.[0-9]+$")
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "-v", "--new-version", help="The new unstable version that is being created (e.g. 2.20).", required=True
+    )
+    args = parser.parse_args()
+
+    if VERSION_VALIDATOR.match(args.new_version) is None:
+        sys.exit("Version must be formatted like so <major>.<minor>")
+
+    target_version = f"{args.new_version}"  # e.g., "2.20" - the target release version
+    major, minor = args.new_version.split(".")
+
+    target_unstable = f"{target_version}-unstable"  # e.g., "2.20-unstable"
+    next_unstable = f"{major}.{int(minor) + 1}-unstable"  # e.g., "2.21-unstable" - next cycle
+    previous_stable = f"{major}.{int(minor) - 1}"  # e.g., "2.19" - previous stable release
+
+    versions = [
+        folder.replace("version-", "")
+        for folder in os.listdir("docs-website/versioned_docs")
+        if os.path.isdir(os.path.join("docs-website/versioned_docs", folder))
+    ]
+
+    # Check if the versions we're about to create already exist in versioned_docs
+    if target_version in versions:
+        sys.exit(f"{target_version} already exists (already released). Aborting.")
+    if target_unstable in versions:
+        print(f"{target_unstable} already exists. Nothing to do.")
+        sys.exit(0)
+
+    # Create new unstable from the currently existing one.
+    # The new unstable will be made stable at a later time by another workflow
+    print(f"Creating new unstable version {target_unstable} from main")
+
+    ### Docusaurus updates
+
+    # copy docs to versioned_docs/version-target_unstable
+    shutil.copytree("docs-website/docs", f"docs-website/versioned_docs/version-{target_unstable}")
+
+    # copy reference to reference_versioned_docs/version-target_unstable
+    shutil.copytree("docs-website/reference", f"docs-website/reference_versioned_docs/version-{target_unstable}")
+
+    # copy versioned_sidebars/version-previous_stable-sidebars.json
+    # to versioned_sidebars/version-target_unstable-sidebars.json
+    shutil.copy(
+        f"docs-website/versioned_sidebars/version-{previous_stable}-sidebars.json",
+        f"docs-website/versioned_sidebars/version-{target_unstable}-sidebars.json",
+    )
+
+    # copy reference_versioned_sidebars/version-previous_stable-sidebars.json
+    # to reference_versioned_sidebars/version-target_unstable-sidebars.json
+    shutil.copy(
+        f"docs-website/reference_versioned_sidebars/version-{previous_stable}-sidebars.json",
+        f"docs-website/reference_versioned_sidebars/version-{target_unstable}-sidebars.json",
+    )
+
+    # add unstable version to versions.json
+    with open("docs-website/versions.json", "r") as f:
+        versions_list = json.load(f)
+    versions_list.insert(0, target_unstable)
+    with open("docs-website/versions.json", "w") as f:
+        json.dump(versions_list, f)
+
+    # add unstable version to reference_versions.json
+    with open("docs-website/reference_versions.json", "r") as f:
+        reference_versions_list = json.load(f)
+    reference_versions_list.insert(0, target_unstable)
+    with open("docs-website/reference_versions.json", "w") as f:
+        json.dump(reference_versions_list, f)
+
+    # in docusaurus.config.js, replace the target unstable version with the next unstable version
+    with open("docs-website/docusaurus.config.js", "r") as f:
+        config = f.read()
+    config = config.replace(f"label: '{target_unstable}'", f"label: '{next_unstable}'")
+    with open("docs-website/docusaurus.config.js", "w") as f:
+        f.write(config)

.github/utils/promote_unstable_docs_docusaurus.py

@@ -0,0 +1,91 @@
+"""
+This script promotes an unstable documentation version to a stable version at the time of a new Haystack release.
+
+To understand how unstable doc versions are created, see create_unstable_docs_docusaurus.py.
+"""
+
+import argparse
+import json
+import os
+import re
+import shutil
+import sys
+
+VERSION_VALIDATOR = re.compile(r"^[0-9]+\.[0-9]+$")
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("-v", "--version", help="The version to promote to stable (e.g. 2.20).", required=True)
+    args = parser.parse_args()
+
+    if VERSION_VALIDATOR.match(args.version) is None:
+        sys.exit("Version must be formatted like so <major>.<minor>")
+
+    target_version = f"{args.version}"  # e.g., "2.20" - the target release version
+    major, minor = args.version.split(".")
+
+    target_unstable = f"{target_version}-unstable"  # e.g., "2.20-unstable"
+    previous_stable = f"{major}.{int(minor) - 1}"  # e.g., "2.19" - previous stable release
+
+    versions = [
+        folder.replace("version-", "")
+        for folder in os.listdir("docs-website/versioned_docs")
+        if os.path.isdir(os.path.join("docs-website/versioned_docs", folder))
+    ]
+
+    if target_version in versions:
+        sys.exit(f"{target_version} already exists (already released). Aborting.")
+    if not target_unstable in versions:
+        sys.exit(f"Can't find version {target_unstable} to promote to {target_version}")
+
+    print(f"Promoting unstable version {target_unstable} to stable version {target_version}")
+
+    ### Docusaurus updates
+
+    # rename versioned_docs/version-target_unstable to versioned_docs/version-target_version
+    shutil.move(
+        f"docs-website/versioned_docs/version-{target_unstable}",
+        f"docs-website/versioned_docs/version-{target_version}",
+    )
+
+    # rename reference_versioned_docs/version-target_unstable to reference_versioned_docs/version-target_version
+    shutil.move(
+        f"docs-website/reference_versioned_docs/version-{target_unstable}",
+        f"docs-website/reference_versioned_docs/version-{target_version}",
+    )
+
+    # copy versioned_sidebars/version-target_unstable-sidebars.json
+    # to versioned_sidebars/version-target_version-sidebars.json
+    shutil.move(
+        f"docs-website/versioned_sidebars/version-{target_unstable}-sidebars.json",
+        f"docs-website/versioned_sidebars/version-{target_version}-sidebars.json",
+    )
+
+    # rename reference_versioned_sidebars/version-target_unstable-sidebars.json
+    # to reference_versioned_sidebars/version-target_version-sidebars.json
+    shutil.move(
+        f"docs-website/reference_versioned_sidebars/version-{target_unstable}-sidebars.json",
+        f"docs-website/reference_versioned_sidebars/version-{target_version}-sidebars.json",
+    )
+
+    # replace unstable version with stable version in versions.json
+    with open("docs-website/versions.json", "r") as f:
+        versions_list = json.load(f)
+    versions_list[versions_list.index(target_unstable)] = target_version
+    with open("docs-website/versions.json", "w") as f:
+        json.dump(versions_list, f)
+
+    # replace unstable version with stable version in reference_versions.json
+    with open("docs-website/reference_versions.json", "r") as f:
+        reference_versions_list = json.load(f)
+    reference_versions_list[reference_versions_list.index(target_unstable)] = target_version
+    with open("docs-website/reference_versions.json", "w") as f:
+        json.dump(reference_versions_list, f)
+
+    # in docusaurus.config.js, replace previous stable version with the target version
+    with open("docs-website/docusaurus.config.js", "r") as f:
+        config = f.read()
+    config = config.replace(f"lastVersion: '{previous_stable}'", f"lastVersion: '{target_version}'")  # "2.19" -> "2.20"
+    with open("docs-website/docusaurus.config.js", "w") as f:
+        f.write(config)

.github/workflows/minor_version_release.yml

@@ -2,7 +2,6 @@ name: Minor Version Release
 
 on:
   workflow_dispatch:
-
 env:
   PYTHON_VERSION: "3.9"
 
@@ -77,3 +76,23 @@ jobs:
         run: |
           git checkout main
           python ./.github/utils/create_unstable_docs.py --new-version ${{ steps.versions.outputs.current_release_minor }}
+
+      - name: Generate unstable docs for Docusaurus
+        run: |
+          git checkout main
+          python ./.github/utils/create_unstable_docs_docusaurus.py --new-version ${{ steps.versions.outputs.current_release_minor }}
+
+      - name: Create Pull Request with Docusaurus docs updates
+        uses: peter-evans/create-pull-request@v7
+        with:
+          token: ${{ secrets.HAYSTACK_BOT_TOKEN }}
+          commit-message: "Create unstable docs for Haystack ${{ steps.versions.outputs.current_release_minor }}"
+          branch: create-unstable-docs-${{ steps.versions.outputs.current_release_minor }}
+          base: main
+          title: "docs: create unstable docs for Haystack ${{ steps.versions.outputs.current_release_minor }}"
+          add-paths: |
+            docs-website
+          body: |
+            This PR creates the unstable docs for Haystack ${{ steps.versions.outputs.current_release_minor }}.
+            It is expected to run during the branch-off process.
+            You can inspect the docs preview (two unstable versions will be available) and merge it.

.github/workflows/promote_unstable_docs.yml

@@ -36,3 +36,23 @@ jobs:
           RDME_API_KEY: ${{ secrets.README_API_KEY }}
         run: |
           python ./.github/utils/promote_unstable_docs.py --version ${{ steps.version.outputs.version }}
+
+      - name: Promote unstable docs for Docusaurus
+        run: |
+          git checkout main
+          python ./.github/utils/promote_unstable_docs_docusaurus.py --version ${{ steps.version.outputs.version }}
+
+      - name: Create Pull Request with Docusaurus docs updates
+        uses: peter-evans/create-pull-request@v7
+        with:
+          token: ${{ secrets.HAYSTACK_BOT_TOKEN }}
+          commit-message: "Promote unstable docs for Haystack ${{ steps.version.outputs.version }}"
+          branch: promote-unstable-docs-${{ steps.version.outputs.version }}
+          base: main
+          title: "docs: promote unstable docs for Haystack ${{ steps.version.outputs.version }}"
+          add-paths: |
+            docs-website
+          body: |
+            This PR promotes the unstable docs for Haystack ${{ steps.version.outputs.version }} to stable.
+            It is expected to run at the time of the release.
+            You can inspect the docs preview and merge it. There should now be only one unstable version representing the next (main) branch.

docs-website/docs/concepts/agents/state.mdx

@@ -2,12 +2,12 @@
 title: "State"
 id: state
 slug: "/state"
-description: "`State` is a container for storing shared information during Agent and Tool execution. It provides a structured way to maintain conversation history, share data between tools, and store intermediate results throughout an agent's workflow."
+description: "`State` is a container for storing shared information during Agent and Tool execution. It provides a structured way to store messages during execution, share data between tools, and store intermediate results throughout an agent's workflow."
 ---
 
 # State
 
-`State` is a container for storing shared information during Agent and Tool execution. It provides a structured way to maintain conversation history, share data between tools, and store intermediate results throughout an agent's workflow.
+`State` is a container for storing shared information during Agent and Tool execution. It provides a structured way to store messages during execution, share data between tools, and store intermediate results throughout an agent's workflow.
 
 ## Overview
 
@@ -30,7 +30,7 @@ State supports standard Python types:
 
 ### Automatic Message Handling
 
-State automatically includes a `messages` field to store conversation history. You don't need to define this in your schema.
+State automatically includes a `messages` field to store messages during execution. You don't need to define this in your schema.
 
 ```python
 ## State automatically adds messages field
@@ -40,11 +40,11 @@ state = State(schema={"user_id": {"type": str}})
 print("messages" in state.schema)  # True
 print(state.schema["messages"]["type"])  # list[ChatMessage]
 
-## Access conversation history
+## Access messages
 messages = state.get("messages", [])
 ```
 
-The `messages` field uses `list[ChatMessage]` type and `merge_lists` handler by default, which means new messages are appended to the conversation history.
+The `messages` field uses `list[ChatMessage]` type and `merge_lists` handler by default, which means new messages are appended during execution.
 
 ## Usage
 
@@ -171,7 +171,7 @@ You can also override handlers for individual operations:
 ```python
 def concatenate_strings(current, new):
     return f"{current}-{new}" if current else new
-    
+
 schema = {"user_name": {"type": str}}
 state = State(schema=schema)
 
@@ -505,7 +505,7 @@ result = agent.run(
 factorial_result = result["factorial_result"]
 calc_result = result["calc_result"]
 
-## Access conversation messages
+## Access messages from execution
 for message in result["messages"]:
     print(f"{message.role}: {message.text}")
-```
+```

docs-website/docs/concepts/components.mdx

@@ -17,14 +17,14 @@ Components are connected to each other using a [pipeline](pipelines.mdx), and th
 
 You can integrate components in a pipeline to perform a specific task. But you can also use some of them stand-alone, outside of a pipeline. For example, you can run `DocumentWriter` on its own, to write documents into a Document Store. To check how to use a component and if it's usable outside of a pipeline, check the _Usage_ section on the component's documentation page.
 
-Each component has a `run()` method. When you connect components in a pipeline, and you run the pipeline by calling `Pipeline.run()`, it invokes the `run()` method for each component sequentially. 
+Each component has a `run()` method. When you connect components in a pipeline, and you run the pipeline by calling `Pipeline.run()`, it invokes the `run()` method for each component sequentially.
 
 ## Input and Output
 
-To connect components in a pipeline, you need to know the names of the inputs and outputs they accept. The output of one component must be compatible with the input the subsequent component accepts. For example, to connect Retriever and Ranker in a pipeline, you must know that the Retriever outputs `documents` and the Ranker accepts `documents` as input. 
+To connect components in a pipeline, you need to know the names of the inputs and outputs they accept. The output of one component must be compatible with the input the subsequent component accepts. For example, to connect Retriever and Ranker in a pipeline, you must know that the Retriever outputs `documents` and the Ranker accepts `documents` as input.
 
 The mandatory inputs and outputs are listed in a table at the top of each component's documentation page so that you can quickly check them:
-<ClickableImage src="/img/3a53f3e-inputs_and_outputs.png" alt="" />
+<ClickableImage src="/img/3a53f3e-inputs_and_outputs.png" alt="DocumentWriter component specification table showing Name, Folder Path, Position in Pipeline, Inputs (documents list), and Outputs (documents_written integer)" />
 
 You can also look them up in the code in the component`run()` method. Here's an example of the inputs and outputs of `TransformerSimilarityRanker`:
 
@@ -56,6 +56,6 @@ result = doc_embedder.run([doc]) # And finally, run it
 print(result['documents'][0].embedding)
 ```
 
-If you're using a component that has the `warm_up()` method in a pipeline, you don't have to do anything additionally. The pipeline takes care of warming it up before running. 
+If you're using a component that has the `warm_up()` method in a pipeline, you don't have to do anything additionally. The pipeline takes care of warming it up before running.
 
-The `warm_up()` method is a nice way to keep the `init()` methods lightweight and the validation fast. (Validation in the pipeline happens when connecting the components but before warming them up and running.)
+The `warm_up()` method is a nice way to keep the `init()` methods lightweight and the validation fast. (Validation in the pipeline happens when connecting the components but before warming them up and running.)

docs-website/docs/concepts/components/custom-components.mdx

@@ -81,9 +81,9 @@ An example of an extended component is Haystack's [FaithfulnessEvaluator](https:
 
 ## Additional References
 
-:cook: Cookbooks:
+🧑‍🍳 Cookbooks:
 
 - [Build quizzes and adventures with Character Codex and llamafile](https://haystack.deepset.ai/cookbook/charactercodex_llamafile/)
 - [Run tasks concurrently within a custom component](https://haystack.deepset.ai/cookbook/concurrent_tasks/)
 - [Chat With Your SQL Database](https://haystack.deepset.ai/cookbook/chat_with_sql_3_ways/)
-- [Hacker News Summaries with Custom Components](https://haystack.deepset.ai/cookbook/hackernews-custom-component-rag/)
+- [Hacker News Summaries with Custom Components](https://haystack.deepset.ai/cookbook/hackernews-custom-component-rag/)