diff --git a/docs/sphinx_doc/source/tutorial/faq.md b/docs/sphinx_doc/source/tutorial/faq.md index c408c69180..f362620ccd 100644 --- a/docs/sphinx_doc/source/tutorial/faq.md +++ b/docs/sphinx_doc/source/tutorial/faq.md @@ -148,14 +148,16 @@ for exp in exp_list: **A:** You need to specify model path and checkpoint path. The following code snippet gives an example with transformers. +Here is an example of loading from fsdp trainer checkpoints: + ```python import os from transformers import AutoTokenizer, AutoModelForCausalLM -from trinity.common.models.utils import load_state_dict_from_verl_checkpoint +from trinity.common.models.utils import load_fsdp_state_dict_from_verl_checkpoint # Assume we need the checkpoint at step 780; # model_path, checkpoint_root_dir, project, and name are already defined model = AutoModelForCausalLM.from_pretrained(model_path) ckp_path = os.path.join(checkpoint_root_dir, project, name, "global_step_780", "actor") -model.load_state_dict(load_state_dict_from_verl_checkpoint(ckp_path)) +model.load_state_dict(load_fsdp_state_dict_from_verl_checkpoint(ckp_path)) ``` diff --git a/tests/template/data/sft_with_tools/sft_with_tools.json b/tests/template/data/sft_with_tools/sft_with_tools.json deleted file mode 100644 index 49d7ab3da5..0000000000 --- a/tests/template/data/sft_with_tools/sft_with_tools.json +++ /dev/null @@ -1,371 +0,0 @@ -[ - { - "messages": [ - { - "role": "system", - "content": "You are a helpful assistant with access to various tools. Use them when needed to help users." - }, - { - "role": "user", - "content": "What's the weather like in Beijing today?" - }, - { - "role": "assistant", - "content": "Let me get the weather for you.", - "tool_calls": [ - { - "id": "call_abc123", - "type": "function", - "function": { - "name": "get_weather", - "arguments": "{\"location\": \"Beijing\", \"unit\": \"celsius\"}" - } - } - ] - }, - { - "role": "tool", - "content": "{\"temperature\": 22, \"condition\": \"sunny\", \"humidity\": 45}", - "tool_call_id": "call_abc123" - }, - { - "role": "assistant", - "content": "The weather in Beijing today is sunny with a temperature of 22°C and humidity at 45%. It's a pleasant day!" - } - ], - "tools": [ - { - "type": "function", - "function": { - "name": "get_weather", - "description": "Get the current weather in a given location", - "parameters": { - "type": "object", - "properties": { - "location": { - "type": "string", - "description": "The city and state, e.g. San Francisco, CA" - }, - "unit": { - "type": "string", - "enum": ["celsius", "fahrenheit"], - "description": "The temperature unit" - } - }, - "required": ["location"] - } - } - } - ] - }, - { - "messages": [ - { - "role": "system", - "content": "You are a helpful math assistant. Use the calculator tool when needed." - }, - { - "role": "user", - "content": "Can you help me calculate 1584 * 327?" - }, - { - "role": "assistant", - "content": "let me calculate 1584 * 327 using the tools provided.", - "tool_calls": [ - { - "id": "call_def456", - "type": "function", - "function": { - "name": "calculator", - "arguments": "{\"operation\": \"multiply\", \"a\": 1584, \"b\": 327}" - } - } - ] - }, - { - "role": "tool", - "content": "{\"result\": 517968}", - "tool_call_id": "call_def456" - }, - { - "role": "assistant", - "content": "The result of 1584 × 327 is 517,968." - } - ], - "tools": [ - { - "type": "function", - "function": { - "name": "calculator", - "description": "Perform basic mathematical operations", - "parameters": { - "type": "object", - "properties": { - "operation": { - "type": "string", - "enum": ["add", "subtract", "multiply", "divide"], - "description": "The mathematical operation to perform" - }, - "a": { - "type": "number", - "description": "First operand" - }, - "b": { - "type": "number", - "description": "Second operand" - } - }, - "required": ["operation", "a", "b"] - } - } - } - ] - }, - { - "messages": [ - { - "role": "system", - "content": "You are a helpful assistant that can search for information and send emails." - }, - { - "role": "user", - "content": "Search for the latest news about AI and then send a summary to john@example.com" - }, - { - "role": "assistant", - "tool_calls": [ - { - "id": "call_ghi789", - "type": "function", - "function": { - "name": "web_search", - "arguments": "{\"query\": \"latest AI news 2024\", \"num_results\": 5}" - } - } - ] - }, - { - "role": "tool", - "content": "{\"results\": [{\"title\": \"GPT-5 Announced\", \"snippet\": \"OpenAI announces next generation model...\"}, {\"title\": \"AI Regulation Updates\", \"snippet\": \"EU passes new AI act...\"}]}", - "tool_call_id": "call_ghi789" - }, - { - "role": "assistant", - "tool_calls": [ - { - "id": "call_jkl012", - "type": "function", - "function": { - "name": "send_email", - "arguments": "{\"to\": \"john@example.com\", \"subject\": \"Latest AI News Summary\", \"body\": \"Here are the latest AI developments:\\n1. GPT-5 Announced - OpenAI announces next generation model\\n2. AI Regulation Updates - EU passes new AI act\"}" - } - } - ] - }, - { - "role": "tool", - "content": "{\"status\": \"sent\", \"message_id\": \"msg_12345\"}", - "tool_call_id": "call_jkl012" - }, - { - "role": "assistant", - "content": "I've found the latest AI news and sent a summary to john@example.com. The email includes information about GPT-5 announcement and EU AI regulation updates." - } - ], - "tools": [ - { - "type": "function", - "function": { - "name": "web_search", - "description": "Search the web for information", - "parameters": { - "type": "object", - "properties": { - "query": { - "type": "string", - "description": "The search query" - }, - "num_results": { - "type": "integer", - "description": "Number of results to return", - "default": 10 - } - }, - "required": ["query"] - } - } - }, - { - "type": "function", - "function": { - "name": "send_email", - "description": "Send an email to a recipient", - "parameters": { - "type": "object", - "properties": { - "to": { - "type": "string", - "description": "Email address of the recipient" - }, - "subject": { - "type": "string", - "description": "Email subject" - }, - "body": { - "type": "string", - "description": "Email body content" - } - }, - "required": ["to", "subject", "body"] - } - } - } - ] - }, - { - "messages": [ - { - "role": "system", - "content": "You are a helpful assistant with access to multiple tools." - }, - { - "role": "user", - "content": "I need to convert 1000 USD to EUR, then calculate 15% tax on that amount, and finally send the result to my email alice@example.com" - }, - { - "role": "assistant", - "content": null, - "tool_calls": [ - { - "id": "call_001", - "type": "function", - "function": { - "name": "currency_converter", - "arguments": "{\"amount\": 1000, \"from_currency\": \"USD\", \"to_currency\": \"EUR\"}" - } - } - ] - }, - { - "role": "tool", - "content": "{\"converted_amount\": 920.50, \"exchange_rate\": 0.9205}", - "tool_call_id": "call_001" - }, - { - "role": "assistant", - "content": null, - "tool_calls": [ - { - "id": "call_002", - "type": "function", - "function": { - "name": "calculate_percentage", - "arguments": "{\"base_amount\": 920.50, \"percentage\": 15}" - } - } - ] - }, - { - "role": "tool", - "content": "{\"result\": 138.075, \"total_with_percentage\": 1058.575}", - "tool_call_id": "call_002" - }, - { - "role": "assistant", - "content": null, - "tool_calls": [ - { - "id": "call_003", - "type": "function", - "function": { - "name": "send_email", - "arguments": "{\"to\": \"alice@example.com\", \"subject\": \"Currency Conversion and Tax Calculation\", \"body\": \"Here are your requested calculations:\\n\\n1. Currency Conversion: 1000 USD = 920.50 EUR (rate: 0.9205)\\n2. 15% tax on 920.50 EUR = 138.08 EUR\\n3. Total amount with tax: 1058.58 EUR\"}" - } - } - ] - }, - { - "role": "tool", - "content": "{\"status\": \"sent\", \"message_id\": \"msg_789xyz\"}", - "tool_call_id": "call_003" - }, - { - "role": "assistant", - "content": "I've completed all the calculations and sent the results to alice@example.com:\n\n1. Converted 1000 USD to 920.50 EUR (exchange rate: 0.9205)\n2. Calculated 15% tax: 138.08 EUR\n3. Total with tax: 1058.58 EUR\n\nThe email has been successfully sent (message ID: msg_789xyz)." - } - ], - "tools": [ - { - "type": "function", - "function": { - "name": "currency_converter", - "description": "Convert amount from one currency to another", - "parameters": { - "type": "object", - "properties": { - "amount": { - "type": "number", - "description": "The amount to convert" - }, - "from_currency": { - "type": "string", - "description": "Source currency code" - }, - "to_currency": { - "type": "string", - "description": "Target currency code" - } - }, - "required": ["amount", "from_currency", "to_currency"] - } - } - }, - { - "type": "function", - "function": { - "name": "calculate_percentage", - "description": "Calculate percentage of a given amount", - "parameters": { - "type": "object", - "properties": { - "base_amount": { - "type": "number", - "description": "The base amount" - }, - "percentage": { - "type": "number", - "description": "The percentage to calculate" - } - }, - "required": ["base_amount", "percentage"] - } - } - }, - { - "type": "function", - "function": { - "name": "send_email", - "description": "Send an email to a recipient", - "parameters": { - "type": "object", - "properties": { - "to": { - "type": "string", - "description": "Email address of the recipient" - }, - "subject": { - "type": "string", - "description": "Email subject" - }, - "body": { - "type": "string", - "description": "Email body content" - } - }, - "required": ["to", "subject", "body"] - } - } - } - ] - } -] diff --git a/tests/template/data/sft_with_tools/sft_with_tools.jsonl b/tests/template/data/sft_with_tools/sft_with_tools.jsonl new file mode 100644 index 0000000000..bf05746670 --- /dev/null +++ b/tests/template/data/sft_with_tools/sft_with_tools.jsonl @@ -0,0 +1,4 @@ +{"messages": "[{\"role\": \"system\", \"content\": \"You are a helpful assistant with access to various tools. Use them when needed to help users.\"}, {\"role\": \"user\", \"content\": \"What's the weather like in Beijing today?\"}, {\"role\": \"assistant\", \"content\": \"Let me get the weather for you.\", \"tool_calls\": [{\"id\": \"call_abc123\", \"type\": \"function\", \"function\": {\"name\": \"get_weather\", \"arguments\": {\"location\": \"Beijing\", \"unit\": \"celsius\"}}}]}, {\"role\": \"tool\", \"content\": {\"temperature\": 22, \"condition\": \"sunny\", \"humidity\": 45}, \"tool_call_id\": \"call_abc123\"}, {\"role\": \"assistant\", \"content\": \"The weather in Beijing today is sunny with a temperature of 22°C and humidity at 45%. It's a pleasant day!\"}]", "tools": "[{\"type\": \"function\", \"function\": {\"name\": \"get_weather\", \"description\": \"Get the current weather in a given location\", \"parameters\": {\"type\": \"object\", \"properties\": {\"location\": {\"type\": \"string\", \"description\": \"The city and state, e.g. San Francisco, CA\"}, \"unit\": {\"type\": \"string\", \"enum\": [\"celsius\", \"fahrenheit\"], \"description\": \"The temperature unit\"}}, \"required\": [\"location\"]}}}]"} +{"messages": "[{\"role\": \"system\", \"content\": \"You are a helpful math assistant. Use the calculator tool when needed.\"}, {\"role\": \"user\", \"content\": \"Can you help me calculate 1584 * 327?\"}, {\"role\": \"assistant\", \"content\": \"let me calculate 1584 * 327 using the tools provided.\", \"tool_calls\": [{\"id\": \"call_def456\", \"type\": \"function\", \"function\": {\"name\": \"calculator\", \"arguments\": {\"operation\": \"multiply\", \"a\": 1584, \"b\": 327}}}]}, {\"role\": \"tool\", \"content\": {\"result\": 517968}, \"tool_call_id\": \"call_def456\"}, {\"role\": \"assistant\", \"content\": \"The result of 1584 × 327 is 517,968.\"}]", "tools": "[{\"type\": \"function\", \"function\": {\"name\": \"calculator\", \"description\": \"Perform basic mathematical operations\", \"parameters\": {\"type\": \"object\", \"properties\": {\"operation\": {\"type\": \"string\", \"enum\": [\"add\", \"subtract\", \"multiply\", \"divide\"], \"description\": \"The mathematical operation to perform\"}, \"a\": {\"type\": \"number\", \"description\": \"First operand\"}, \"b\": {\"type\": \"number\", \"description\": \"Second operand\"}}, \"required\": [\"operation\", \"a\", \"b\"]}}}]"} +{"messages": "[{\"role\": \"system\", \"content\": \"You are a helpful assistant that can search for information and send emails.\"}, {\"role\": \"user\", \"content\": \"Search for the latest news about AI and then send a summary to john@example.com\"}, {\"role\": \"assistant\", \"tool_calls\": [{\"id\": \"call_ghi789\", \"type\": \"function\", \"function\": {\"name\": \"web_search\", \"arguments\": {\"query\": \"latest AI news 2024\", \"num_results\": 5}}}]}, {\"role\": \"tool\", \"content\": {\"results\": [{\"title\": \"GPT-5 Announced\", \"snippet\": \"OpenAI announces next generation model...\"}, {\"title\": \"AI Regulation Updates\", \"snippet\": \"EU passes new AI act...\"}]}, \"tool_call_id\": \"call_ghi789\"}, {\"role\": \"assistant\", \"tool_calls\": [{\"id\": \"call_jkl012\", \"type\": \"function\", \"function\": {\"name\": \"send_email\", \"arguments\": {\"to\": \"john@example.com\", \"subject\": \"Latest AI News Summary\", \"body\": \"Here are the latest AI developments:\\n1. GPT-5 Announced - OpenAI announces next generation model\\n2. AI Regulation Updates - EU passes new AI act\"}}}]}, {\"role\": \"tool\", \"content\": {\"status\": \"sent\", \"message_id\": \"msg_12345\"}, \"tool_call_id\": \"call_jkl012\"}, {\"role\": \"assistant\", \"content\": \"I've found the latest AI news and sent a summary to john@example.com. The email includes information about GPT-5 announcement and EU AI regulation updates.\"}]", "tools": "[{\"type\": \"function\", \"function\": {\"name\": \"web_search\", \"description\": \"Search the web for information\", \"parameters\": {\"type\": \"object\", \"properties\": {\"query\": {\"type\": \"string\", \"description\": \"The search query\"}, \"num_results\": {\"type\": \"integer\", \"description\": \"Number of results to return\", \"default\": 10}}, \"required\": [\"query\"]}}}, {\"type\": \"function\", \"function\": {\"name\": \"send_email\", \"description\": \"Send an email to a recipient\", \"parameters\": {\"type\": \"object\", \"properties\": {\"to\": {\"type\": \"string\", \"description\": \"Email address of the recipient\"}, \"subject\": {\"type\": \"string\", \"description\": \"Email subject\"}, \"body\": {\"type\": \"string\", \"description\": \"Email body content\"}}, \"required\": [\"to\", \"subject\", \"body\"]}}}]"} +{"messages": "[{\"role\": \"system\", \"content\": \"You are a helpful assistant with access to multiple tools.\"}, {\"role\": \"user\", \"content\": \"I need to convert 1000 USD to EUR, then calculate 15% tax on that amount, and finally send the result to my email alice@example.com\"}, {\"role\": \"assistant\", \"content\": null, \"tool_calls\": [{\"id\": \"call_001\", \"type\": \"function\", \"function\": {\"name\": \"currency_converter\", \"arguments\": {\"amount\": 1000, \"from_currency\": \"USD\", \"to_currency\": \"EUR\"}}}]}, {\"role\": \"tool\", \"content\": {\"converted_amount\": 920.5, \"exchange_rate\": 0.9205}, \"tool_call_id\": \"call_001\"}, {\"role\": \"assistant\", \"content\": null, \"tool_calls\": [{\"id\": \"call_002\", \"type\": \"function\", \"function\": {\"name\": \"calculate_percentage\", \"arguments\": {\"base_amount\": 920.5, \"percentage\": 15}}}]}, {\"role\": \"tool\", \"content\": {\"result\": 138.075, \"total_with_percentage\": 1058.575}, \"tool_call_id\": \"call_002\"}, {\"role\": \"assistant\", \"content\": null, \"tool_calls\": [{\"id\": \"call_003\", \"type\": \"function\", \"function\": {\"name\": \"send_email\", \"arguments\": {\"to\": \"alice@example.com\", \"subject\": \"Currency Conversion and Tax Calculation\", \"body\": \"Here are your requested calculations:\\n\\n1. Currency Conversion: 1000 USD = 920.50 EUR (rate: 0.9205)\\n2. 15% tax on 920.50 EUR = 138.08 EUR\\n3. Total amount with tax: 1058.58 EUR\"}}}]}, {\"role\": \"tool\", \"content\": {\"status\": \"sent\", \"message_id\": \"msg_789xyz\"}, \"tool_call_id\": \"call_003\"}, {\"role\": \"assistant\", \"content\": \"I've completed all the calculations and sent the results to alice@example.com:\\n\\n1. Converted 1000 USD to 920.50 EUR (exchange rate: 0.9205)\\n2. Calculated 15% tax: 138.08 EUR\\n3. Total with tax: 1058.58 EUR\\n\\nThe email has been successfully sent (message ID: msg_789xyz).\"}]", "tools": "[{\"type\": \"function\", \"function\": {\"name\": \"currency_converter\", \"description\": \"Convert amount from one currency to another\", \"parameters\": {\"type\": \"object\", \"properties\": {\"amount\": {\"type\": \"number\", \"description\": \"The amount to convert\"}, \"from_currency\": {\"type\": \"string\", \"description\": \"Source currency code\"}, \"to_currency\": {\"type\": \"string\", \"description\": \"Target currency code\"}}, \"required\": [\"amount\", \"from_currency\", \"to_currency\"]}}}, {\"type\": \"function\", \"function\": {\"name\": \"calculate_percentage\", \"description\": \"Calculate percentage of a given amount\", \"parameters\": {\"type\": \"object\", \"properties\": {\"base_amount\": {\"type\": \"number\", \"description\": \"The base amount\"}, \"percentage\": {\"type\": \"number\", \"description\": \"The percentage to calculate\"}}, \"required\": [\"base_amount\", \"percentage\"]}}}, {\"type\": \"function\", \"function\": {\"name\": \"send_email\", \"description\": \"Send an email to a recipient\", \"parameters\": {\"type\": \"object\", \"properties\": {\"to\": {\"type\": \"string\", \"description\": \"Email address of the recipient\"}, \"subject\": {\"type\": \"string\", \"description\": \"Email subject\"}, \"body\": {\"type\": \"string\", \"description\": \"Email body content\"}}, \"required\": [\"to\", \"subject\", \"body\"]}}}]"} diff --git a/trinity/buffer/schema/formatter.py b/trinity/buffer/schema/formatter.py index bf74589266..cdf1447f00 100644 --- a/trinity/buffer/schema/formatter.py +++ b/trinity/buffer/schema/formatter.py @@ -1,3 +1,4 @@ +import json from abc import ABC, abstractmethod from typing import Dict, List, Optional @@ -7,6 +8,7 @@ from trinity.common.models.utils import get_action_mask_method from trinity.common.rewards import REWARD_FUNCTIONS from trinity.common.workflows import WORKFLOWS, Task +from trinity.utils.log import get_logger from trinity.utils.registry import Registry FORMATTER = Registry("formatter") @@ -98,6 +100,7 @@ class SFTFormatter(ExperienceFormatter): """ def __init__(self, tokenizer, format_config: FormatConfig): + self.logger = get_logger("sft_dataset_formatter", in_ray_actor=True) self.tokenizer = tokenizer self.prompt_type = format_config.prompt_type self.enable_concatenated_multi_turn = format_config.enable_concatenated_multi_turn @@ -120,18 +123,43 @@ def __init__(self, tokenizer, format_config: FormatConfig): def _messages_to_experience( self, - messages: List[Dict], - tools: Optional[List[Dict]] = None, + messages: List[Dict] | str, # or could be str from json dumps + tools: Optional[List[Dict] | str] = None, # or could also be str from json dumps ) -> Experience: """Convert messages and tools into an Experience object. Args: - messages (List[Dict]): The list of message dictionaries. - tools (Optional[List[Dict]], optional): The list of tool dictionaries. Defaults to None. + messages (List[Dict]|str): The list of message dictionaries or a JSON string. + tools (Optional[List[Dict]|str], optional): The list of tool dictionaries or a JSON string. Defaults to None. Returns: Experience: The resulting Experience object. """ + if isinstance(messages, str): + try: + messages = json.loads(messages) + except json.JSONDecodeError: + self.logger.error( + "[SFT Data Error] Failed to decode 'messages' JSON. please check your data format." + ) + raise ValueError("Invalid JSON format for messages") + # Warning if tools is accidentally provided as list of dicts (with Huggingface datasets this may cause schema issues) + if tools is not None and isinstance(tools, list): + self.logger.warning( + "[SFT Data Warning] 'tools' is provided as a list of dictionaries. " + "When loading with Huggingface Datasets, schema auto-alignment may set unmatched fields to null, " + "potentially causing undesired behavior. " + "It is recommended to pre-process 'tools' objects with json.dumps before saving/loading, " + "and to restore them with json.loads in this function." + ) + if isinstance(tools, str): + try: + tools = json.loads(tools) + except json.JSONDecodeError: + self.logger.error( + "[SFT Data Error] Failed to decode 'tools' JSON. Please check your data format." + ) + raise ValueError("Invalid JSON format for tools") tokens = self.tokenizer.apply_chat_template( messages, tools=tools,