From 60bcd75b67d5bbea09cb46ce3676bf83bd12a169 Mon Sep 17 00:00:00 2001 From: Maciej Majek Date: Mon, 7 Oct 2024 23:41:06 +0200 Subject: [PATCH 01/10] docs: update create_robots_whoami documentation to clarify the role of the whoami package and provide additional steps for generated files inspection refactor(rai_cli.py): change argument names and paths in parse_whoami_package function to improve clarity and organization of generated files refactor(rai_whoami_node.py): update paths to robot constitution, identity, and URDF description files to reflect new directory structure for generated files --- docs/create_robots_whoami.md | 23 +++++++--- src/rai/rai/cli/rai_cli.py | 46 ++++++++++---------- src/rai_whoami/rai_whoami/rai_whoami_node.py | 8 ++-- 3 files changed, 45 insertions(+), 32 deletions(-) diff --git a/docs/create_robots_whoami.md b/docs/create_robots_whoami.md index 08a2946f..a0efc5b3 100644 --- a/docs/create_robots_whoami.md +++ b/docs/create_robots_whoami.md @@ -6,6 +6,8 @@ To configure RAI for your robot, provide contents for your robot's so called `wh Your robot's `whoami` package serves as a configuration package for the `rai_whoami` node. +> **TIP**: The Human-Machine Interface (HMI), both text and voice versions, relies heavily on the whoami package. It uses the robot's identity, constitution, and documentation to provide context-aware responses and ensure the robot behaves according to its defined characteristics. + ## Example (Franka Emika Panda arm) 1. Setup the repository using 1st and 2nd step from [Setup](../README.md#setup) @@ -16,10 +18,10 @@ Your robot's `whoami` package serves as a configuration package for the `rai_who poetry run create_rai_ws --name panda --destination-directory src/examples ``` -3. Fill in the `src/examples/panda_whoami/description` folder with data:\ - 2.1 Save [this image](https://robodk.com/robot/img/Franka-Emika-Panda-robot.png) into `src/examples/panda_whoami/description/images`\ - 2.2 Save [this document](https://github.com/user-attachments/files/16417196/Franka.Emika.Panda.robot.-.RoboDK.pdf) in `src/examples/panda_whoami/description/documentation` - 2.3 Save [this urdf](https://github.com/frankaemika/franka_ros/blob/develop/franka_description/robots/panda/panda.urdf.xacro) in `src/examples/panda_whoami/description/urdf` +3. Fill in the `src/examples/panda_whoami/description` folder with data: + 3.1 Save [this image](https://robodk.com/robot/img/Franka-Emika-Panda-robot.png) into `src/examples/panda_whoami/description/images` + 3.2 Save [this document](https://github.com/user-attachments/files/16417196/Franka.Emika.Panda.robot.-.RoboDK.pdf) in `src/examples/panda_whoami/description/documentation` + 3.3 Save [this urdf](https://github.com/frankaemika/franka_ros/blob/develop/franka_description/robots/panda/panda.urdf.xacro) in `src/examples/panda_whoami/description/urdf` 4. Run the `parse_whoami_package`. This will process the documentation, building it into a vector database, which is used by RAI agent to reason about its identity. @@ -27,14 +29,23 @@ Your robot's `whoami` package serves as a configuration package for the `rai_who > [config.toml](../config.toml) (`ollama` works locally, see [docs/vendors.md](./vendors.md#ollama)). ```shell -poetry run parse_whoami_package src/examples/panda_whoami/description +poetry run parse_whoami_package src/examples/panda_whoami ``` +5. Optional: Examine the generated files + +After running the `parse_whoami_package` command, you can inspect the generated files in the `src/examples/panda_whoami/description/generated` directory. These files contain important information about your robot: + +- `robot_identity.txt`: Contains a detailed description of the robot's identity, capabilities, and characteristics. +- `robot_description.urdf.txt`: Provides a summary of the robot's URDF (Unified Robot Description Format), describing its physical structure. +- `robot_constitution.txt`: Outlines the ethical guidelines and operational rules for the robot. +- `faiss_index`: A directory containing the vector store of the robot's documentation, used for efficient information retrieval. + ## Testing You can test your new `panda_whoami` package by calling `rai_whoami` services: -2. Building the `rai_whoami` package and running the `rai_whoami_node` for your `Panda` robot: +1. Building the `rai_whoami` package and running the `rai_whoami_node` for your `Panda` robot: ```shell colcon build --symlink-install diff --git a/src/rai/rai/cli/rai_cli.py b/src/rai/rai/cli/rai_cli.py index e302ca0c..c7e00623 100644 --- a/src/rai/rai/cli/rai_cli.py +++ b/src/rai/rai/cli/rai_cli.py @@ -38,24 +38,20 @@ def parse_whoami_package(): description="Parse robot whoami package. Script builds a vector store, creates a robot identity and a URDF description." ) parser.add_argument( - "documentation_root", type=str, help="Path to the root of the documentation" - ) - parser.add_argument( - "--output", - type=str, - required=False, - default=None, - help="Path to the output directory", + "whoami_package_root", type=str, help="Path to the root of the whoami package" ) + args = parser.parse_args() - save_dir = args.output if args.output is not None else args.documentation_root + save_dir = Path(args.whoami_package_root) / "description" / "generated" + description_path = Path(args.whoami_package_root) / "description" + save_dir.mkdir(parents=True, exist_ok=True) llm = get_llm_model(model_type="simple_model") embeddings_model = get_embeddings_model() def calculate_urdf_tokens(): combined_urdf = "" - xacro_files = glob.glob(args.documentation_root + "/urdf/*.xacro") + xacro_files = glob.glob(str(description_path / "urdf" / "*.xacro")) for xacro_file in xacro_files: combined_urdf += f"# {xacro_file}\n" combined_urdf += open(xacro_file, "r").read() @@ -65,18 +61,23 @@ def calculate_urdf_tokens(): def build_urdf_description(): logger.info("Building the URDF description...") combined_urdf = "" - xacro_files = glob.glob(args.documentation_root + "/urdf/*.xacro") + xacro_files = glob.glob(str(description_path / "urdf" / "*.xacro")) for xacro_file in xacro_files: combined_urdf += f"# {xacro_file}\n" combined_urdf += open(xacro_file, "r").read() combined_urdf += "\n\n" prompt = "You will be given a URDF file. Your task is to create a short and detailed description of links and joints. " - parsed_urdf = llm.invoke( - [SystemMessage(content=prompt), HumanMessage(content=str(combined_urdf))] - ).content - - with open(save_dir + "/robot_description.urdf.txt", "w") as f: + parsed_urdf = "" + if len(combined_urdf): + parsed_urdf = llm.invoke( + [ + SystemMessage(content=prompt), + HumanMessage(content=str(combined_urdf)), + ] + ).content + + with open(str(save_dir / "robot_description.urdf.txt"), "w") as f: f.write(parsed_urdf) logger.info("Done") @@ -84,7 +85,7 @@ def build_docs_vector_store(): logger.info("Building the robot docs vector store...") faiss_index = FAISS.from_documents(docs, embeddings_model) faiss_index.add_documents(docs) - faiss_index.save_local(save_dir) + faiss_index.save_local(str(save_dir)) def build_robot_identity(): logger.info("Building the robot identity...") @@ -98,7 +99,7 @@ def build_robot_identity(): "Your reply should start with I am a ..." ) - images = glob.glob(args.documentation_root + "/images/*") + images = glob.glob(str(description_path / "images" / "*")) messages = [SystemMessage(content=prompt)] + [ HumanMultimodalMessage( @@ -109,12 +110,12 @@ def build_robot_identity(): output = llm.invoke(messages) assert isinstance(output.content, str), "Malformed output" - with open(save_dir + "/robot_identity.txt", "w") as f: + with open(str(save_dir / "robot_identity.txt"), "w") as f: f.write(output.content) logger.info("Done") docs = ingest_documentation( - documentation_root=args.documentation_root + "/documentation" + documentation_root=str(description_path / "documentation") ) documentation = str([doc.page_content for doc in docs]) n_tokens = len(documentation) // 4.0 @@ -189,7 +190,8 @@ def create_rai_ws(): (package_path / "documentation").mkdir(exist_ok=True) (package_path / "images").mkdir(exist_ok=True) - (package_path / "robot_constitution.txt").touch() + (package_path / "generated").mkdir(exist_ok=True) + (package_path / "generated" / "robot_constitution.txt").touch() default_constitution_path = ( "src/rai/rai/cli/resources/default_robot_constitution.txt" @@ -197,7 +199,7 @@ def create_rai_ws(): with open(default_constitution_path, "r") as file: default_constitution = file.read() - with open(f"{package_path}/robot_constitution.txt", "w") as file: + with open(f"{package_path}/generated/robot_constitution.txt", "w") as file: file.write(default_constitution) # Modify setup.py file diff --git a/src/rai_whoami/rai_whoami/rai_whoami_node.py b/src/rai_whoami/rai_whoami/rai_whoami_node.py index ca5de21a..07d7c558 100644 --- a/src/rai_whoami/rai_whoami/rai_whoami_node.py +++ b/src/rai_whoami/rai_whoami/rai_whoami_node.py @@ -67,7 +67,7 @@ def __init__(self): ) # type: ignore self.robot_constitution_path = os.path.join( get_package_share_directory(self.robot_description_package), - "description/robot_constitution.txt", + "description/generated/robot_constitution.txt", ) with open(self.robot_constitution_path, "r") as file: @@ -83,7 +83,7 @@ def __init__(self): def _load_documentation(self) -> FAISS: faiss_index = FAISS.load_local( get_package_share_directory(self.robot_description_package) - + "/description", + + "/description/generated", get_embeddings_model(), allow_dangerous_deserialization=True, ) @@ -95,7 +95,7 @@ def get_urdf_callback( """Return URDF description""" urdf_path = ( get_package_share_directory(self.robot_description_package) - + "/description/robot_description.urdf.txt" + + "/description/generated/robot_description.urdf.txt" ) with open(urdf_path, "r") as f: urdf = f.read() @@ -175,7 +175,7 @@ def get_identity_callback( """Return robot identity""" identity_path = ( get_package_share_directory(self.robot_description_package) - + "/description/robot_identity.txt" + + "/description/generated/robot_identity.txt" ) with open(identity_path, "r") as f: identity = f.read() From a6b115a3e9f243cbeae6e460aa4b45fd69f0d955 Mon Sep 17 00:00:00 2001 From: Maciej Majek Date: Mon, 7 Oct 2024 23:41:27 +0200 Subject: [PATCH 02/10] docs(README.md): update section title from "Further documentation" to "Developer resources" for clarity docs(README.md): enhance description in Developer Resources section to provide more context on the Developer Guide --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 8c0d99bf..958ebeca 100644 --- a/README.md +++ b/README.md @@ -47,7 +47,7 @@ The RAI framework aims to: - [Features](#features) - [Setup](#setup) - [Usage examples (demos)](#planned-demos) -- [Further documentation](#further-documentation) +- [Developer resources](#developer-resources) - [ROSCon 2024 Talk](#roscon-2024) ## Features @@ -180,7 +180,7 @@ Please take a look at [Q&A](https://github.com/RobotecAI/rai/discussions/categor ### Developer Resources -See our [Developer Guide](docs/developer_guide.md). +See our [Developer Guide](docs/developer_guide.md) for a deeper dive into RAI, including instructions on creating a configuration specifically for your robot. ### Contributing From 1c0d8c00001c19ece15f482d4194e57a18436164 Mon Sep 17 00:00:00 2001 From: Maciej Majek Date: Mon, 7 Oct 2024 23:43:14 +0200 Subject: [PATCH 03/10] docs(create_robots_whoami.md): update tips and notes formatting for better visibility and emphasis in documentation --- docs/create_robots_whoami.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/create_robots_whoami.md b/docs/create_robots_whoami.md index a0efc5b3..e0074798 100644 --- a/docs/create_robots_whoami.md +++ b/docs/create_robots_whoami.md @@ -6,7 +6,8 @@ To configure RAI for your robot, provide contents for your robot's so called `wh Your robot's `whoami` package serves as a configuration package for the `rai_whoami` node. -> **TIP**: The Human-Machine Interface (HMI), both text and voice versions, relies heavily on the whoami package. It uses the robot's identity, constitution, and documentation to provide context-aware responses and ensure the robot behaves according to its defined characteristics. +> [!TIP] +> The Human-Machine Interface (HMI), both text and voice versions, relies heavily on the whoami package. It uses the robot's identity, constitution, and documentation to provide context-aware responses and ensure the robot behaves according to its defined characteristics. ## Example (Franka Emika Panda arm) @@ -25,7 +26,8 @@ Your robot's `whoami` package serves as a configuration package for the `rai_who 4. Run the `parse_whoami_package`. This will process the documentation, building it into a vector database, which is used by RAI agent to reason about its identity. -> **NOTE**: Parsing bigger documents might lead to costs. Embedding model can be configured in +> [!IMPORTANT] +> Parsing bigger documents might lead to costs. Embedding model can be configured in > [config.toml](../config.toml) (`ollama` works locally, see [docs/vendors.md](./vendors.md#ollama)). ```shell From 05b943c6bb7e6bf6626a656a0e1cddee7f3692c7 Mon Sep 17 00:00:00 2001 From: Maciej Majek Date: Mon, 7 Oct 2024 23:45:16 +0200 Subject: [PATCH 04/10] docs(create_robots_whoami): improve formatting of instructions for clarity by adding line breaks between steps --- docs/create_robots_whoami.md | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/docs/create_robots_whoami.md b/docs/create_robots_whoami.md index e0074798..bca87f02 100644 --- a/docs/create_robots_whoami.md +++ b/docs/create_robots_whoami.md @@ -20,9 +20,12 @@ Your robot's `whoami` package serves as a configuration package for the `rai_who ``` 3. Fill in the `src/examples/panda_whoami/description` folder with data: - 3.1 Save [this image](https://robodk.com/robot/img/Franka-Emika-Panda-robot.png) into `src/examples/panda_whoami/description/images` - 3.2 Save [this document](https://github.com/user-attachments/files/16417196/Franka.Emika.Panda.robot.-.RoboDK.pdf) in `src/examples/panda_whoami/description/documentation` - 3.3 Save [this urdf](https://github.com/frankaemika/franka_ros/blob/develop/franka_description/robots/panda/panda.urdf.xacro) in `src/examples/panda_whoami/description/urdf` + + 3.1. Save [this image](https://robodk.com/robot/img/Franka-Emika-Panda-robot.png) into `src/examples/panda_whoami/description/images` + + 3.2. Save [this document](https://github.com/user-attachments/files/16417196/Franka.Emika.Panda.robot.-.RoboDK.pdf) in `src/examples/panda_whoami/description/documentation` + + 3.3. Save [this urdf](https://github.com/frankaemika/franka_ros/blob/develop/franka_description/robots/panda/panda.urdf.xacro) in `src/examples/panda_whoami/description/urdf` 4. Run the `parse_whoami_package`. This will process the documentation, building it into a vector database, which is used by RAI agent to reason about its identity. From 7265c7e0b27facc38b930afb5444ce4746519371 Mon Sep 17 00:00:00 2001 From: Maciej Majek Date: Mon, 7 Oct 2024 23:52:00 +0200 Subject: [PATCH 05/10] chore: pre-commit --- docs/create_robots_whoami.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/create_robots_whoami.md b/docs/create_robots_whoami.md index bca87f02..c55d573b 100644 --- a/docs/create_robots_whoami.md +++ b/docs/create_robots_whoami.md @@ -20,11 +20,11 @@ Your robot's `whoami` package serves as a configuration package for the `rai_who ``` 3. Fill in the `src/examples/panda_whoami/description` folder with data: - + 3.1. Save [this image](https://robodk.com/robot/img/Franka-Emika-Panda-robot.png) into `src/examples/panda_whoami/description/images` - + 3.2. Save [this document](https://github.com/user-attachments/files/16417196/Franka.Emika.Panda.robot.-.RoboDK.pdf) in `src/examples/panda_whoami/description/documentation` - + 3.3. Save [this urdf](https://github.com/frankaemika/franka_ros/blob/develop/franka_description/robots/panda/panda.urdf.xacro) in `src/examples/panda_whoami/description/urdf` 4. Run the `parse_whoami_package`. This will process the documentation, building it into a vector database, which is used by RAI agent to reason about its identity. From f30f3c92d66c12141596efd0943ad0ba80208214 Mon Sep 17 00:00:00 2001 From: Maciej Majek Date: Mon, 7 Oct 2024 23:53:46 +0200 Subject: [PATCH 06/10] docs(developer_guide): add instructions for testing the robot using the Text HMI to enhance developer experience docs(developer_guide): update section numbering to reflect the addition of the new testing instructions --- docs/developer_guide.md | 30 ++++++++++++++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) diff --git a/docs/developer_guide.md b/docs/developer_guide.md index 7a38aee2..7a744ec7 100644 --- a/docs/developer_guide.md +++ b/docs/developer_guide.md @@ -30,7 +30,33 @@ print(agent("Please publish 'Hello RAI' to /chatter topic only once")) # make su Follow instructions to [configure RAI identity for your robot](create_robots_whoami.md). -### 2. Implement new tools specific to your robot +### 2. Test Your Robot Using the Text HMI + +To run the fully initialized Streamlit HMI, use the following command, replacing `my_robot_whoami` with the name of the package you created: + +```bash +streamlit run src/rai_hmi/rai_hmi/text_hmi.py my_robot_whoami +``` + +To verify if Streamlit successfully loaded the configuration: + +1. Expand the "System status" menu in the Streamlit interface. + +2. Check for the following indicators: + - "robot_database": true + - "system_prompt": true + +If both indicators are present and set to true, your configuration has been loaded correctly. You can now interact with your robot by: + +1. Asking about its identity and purpose + +2. Inquiring about its capabilities + +3. Requesting information on the ROS topics it can access + +This interaction will help you verify that the robot's 'whoami' package is functioning as intended and providing accurate information about your robot's configuration. + +### 3. Implement new tools specific to your robot RAI has general capabilities to interact through ROS interfaces such as actions and topics. However, you can extend RAI with tools dedicated to what your robot needs to do. @@ -77,7 +103,7 @@ def state_retriever(): ``` -### 3. Run the agent with new tools +### 4. Run the agent with new tools Once you have implemented your tools, you can run the agent with these new tools as follows: From e7dfe17ad3bade4ce11113c8c3db35b1abd0711c Mon Sep 17 00:00:00 2001 From: Maciej Majek Date: Mon, 7 Oct 2024 23:55:29 +0200 Subject: [PATCH 07/10] fix(test_rai_cli.py): update path to description files to reflect new directory structure in tests --- tests/core/test_rai_cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/core/test_rai_cli.py b/tests/core/test_rai_cli.py index be49b070..19f16f97 100644 --- a/tests/core/test_rai_cli.py +++ b/tests/core/test_rai_cli.py @@ -44,5 +44,5 @@ def test_create_rai_ws(rai_ws: Path): assert os.path.exists(whoami_directory), "Description folder is missing" - description_files = os.listdir(whoami_directory / "description") + description_files = os.listdir(whoami_directory / "description" / "generated") assert "robot_constitution.txt" in description_files From 06bf923e6dfdcb4e3d74a410d1853564038902d4 Mon Sep 17 00:00:00 2001 From: Maciej Majek Date: Mon, 7 Oct 2024 23:57:49 +0200 Subject: [PATCH 08/10] docs(developer_guide): update example code to reflect automatic vendor initialization in ROS2Agent for better clarity and usability --- docs/developer_guide.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/developer_guide.md b/docs/developer_guide.md index 7a744ec7..a36adfa0 100644 --- a/docs/developer_guide.md +++ b/docs/developer_guide.md @@ -18,7 +18,7 @@ It is easily extendable, allowing developers to adapt and integrate new function ```python from rai import ROS2Agent -agent = ROS2Agent(vendor='openai') # openai or bedrock +agent = ROS2Agent() # vendor will be automatically initialized based on the config.toml print(agent("What topics, services, and actions are available?")) print(agent("Please describe the interfaces of two of the existing topics.")) print(agent("Please publish 'Hello RAI' to /chatter topic only once")) # make sure to listen first ros2 topic echo /chatter From d80105b5a685c48d4ece8e134a98d13009369b38 Mon Sep 17 00:00:00 2001 From: Maciej Majek Date: Tue, 8 Oct 2024 09:39:04 +0200 Subject: [PATCH 09/10] fix: update faiss index path --- src/rai_hmi/rai_hmi/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/rai_hmi/rai_hmi/base.py b/src/rai_hmi/rai_hmi/base.py index c2193eb8..54ba534c 100644 --- a/src/rai_hmi/rai_hmi/base.py +++ b/src/rai_hmi/rai_hmi/base.py @@ -199,7 +199,7 @@ def _load_documentation(self) -> Optional[FAISS]: try: faiss_index = FAISS.load_local( get_package_share_directory(self.robot_description_package) - + "/description", + + "/description/generated", get_embeddings_model(), allow_dangerous_deserialization=True, ) From 27071bc553819dfaf56218d9b7f825d4c247afa4 Mon Sep 17 00:00:00 2001 From: Maciej Majek Date: Tue, 8 Oct 2024 09:41:55 +0200 Subject: [PATCH 10/10] fix: bad path in main README --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 958ebeca..d5d1d85b 100644 --- a/README.md +++ b/README.md @@ -156,7 +156,7 @@ Follow this guide: [husarion-rosbot-xl-demo](docs/demos.md) ## What's next? -Once you know your way around RAI, try the following challenges, with the aid the [developer guide](developer_guide.md): +Once you know your way around RAI, try the following challenges, with the aid the [developer guide](docs/developer_guide.md): - Run RAI on your own robot and talk to it, asking questions about what is in its documentation (and others!). - Implement additional tools and use them in your interaction.