|
| 1 | +"""LLM Input Mocker implementation.""" |
| 2 | + |
| 3 | +import importlib.util |
| 4 | +import inspect |
| 5 | +import json |
| 6 | +from datetime import datetime |
| 7 | +from typing import Any, Dict |
| 8 | + |
| 9 | +from pydantic import TypeAdapter |
| 10 | + |
| 11 | +from uipath import UiPath |
| 12 | +from uipath._cli._evals._models._evaluation_set import EvaluationItem |
| 13 | +from uipath._services.llm_gateway_service import _cleanup_schema |
| 14 | +from uipath.tracing._traced import traced |
| 15 | + |
| 16 | +from .mocker import UiPathInputMockingError |
| 17 | + |
| 18 | + |
| 19 | +def get_input_mocking_prompt( |
| 20 | + input_schema: str, |
| 21 | + test_run_proctor_instructions: str, |
| 22 | + input_generation_instructions: str, |
| 23 | + expected_behavior: str, |
| 24 | + expected_output: str, |
| 25 | +) -> str: |
| 26 | + """Generate the LLM input mocking prompt.""" |
| 27 | + current_datetime = datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S") |
| 28 | + |
| 29 | + return f"""You are simulating input for automated testing purposes of an Agent as part of a simulation run. |
| 30 | +You will need to generate realistic input to a LLM agent which will call various tools to achieve a goal. This must be in the exact format of the INPUT_SCHEMA. |
| 31 | +You may need to follow specific INPUT_GENERATION_INSTRUCTIONS. If no relevant instructions are provided pertaining to input generation, use the other provided information and your own judgement to generate input. |
| 32 | +If the INPUT_GENERATION_INSTRUCTIONS are provided, you MUST follow them exactly. For example if the instructions say to generate a value for a field to be before a certain calendar date, you must generate a value that is before that date. |
| 33 | +The SIMULATION_INSTRUCTIONS will provide context around how the tools are being simulated. |
| 34 | +
|
| 35 | +
|
| 36 | +The current date and time is: {current_datetime} |
| 37 | +
|
| 38 | +#INPUT_SCHEMA: You MUST OUTPUT THIS EXACT JSON SCHEMA |
| 39 | +{input_schema} |
| 40 | +#END_INPUT_SCHEMA |
| 41 | +
|
| 42 | +#INPUT_GENERATION_INSTRUCTIONS |
| 43 | +{input_generation_instructions} |
| 44 | +#END_INPUT_GENERATION_INSTRUCTIONS |
| 45 | +
|
| 46 | +#SIMULATION_INSTRUCTIONS |
| 47 | +{test_run_proctor_instructions} |
| 48 | +#END_SIMULATION_INSTRUCTIONS |
| 49 | +
|
| 50 | +#EXPECTED_BEHAVIOR |
| 51 | +{expected_behavior} |
| 52 | +#END_EXPECTED_BEHAVIOR |
| 53 | +
|
| 54 | +#EXPECTED_OUTPUT |
| 55 | +{expected_output} |
| 56 | +#END_EXPECTED_OUTPUT |
| 57 | +
|
| 58 | +Based on the above information, provide a realistic input to the LLM agent. Your response should: |
| 59 | +1. Match the expected input format according to the INPUT_SCHEMA exactly |
| 60 | +2. Be consistent with the style and level of detail in the example inputs |
| 61 | +3. Consider the context of the the agent being tested |
| 62 | +4. Be realistic and representative of what a real user might say or ask |
| 63 | +
|
| 64 | +OUTPUT: ONLY the simulated agent input in the exact format of the INPUT_SCHEMA in valid JSON. Do not include any explanations, quotation marks, or markdown.""" |
| 65 | + |
| 66 | + |
| 67 | +def extract_input_schema_from_entrypoint(entrypoint_path: str) -> Dict[str, Any]: |
| 68 | + """Extract JSON schema from the entrypoint file's main function.""" |
| 69 | + spec = importlib.util.spec_from_file_location("entrypoint_module", entrypoint_path) |
| 70 | + if not spec or not spec.loader: |
| 71 | + raise UiPathInputMockingError( |
| 72 | + f"Failed to load module spec from entrypoint: {entrypoint_path}" |
| 73 | + ) |
| 74 | + |
| 75 | + module = importlib.util.module_from_spec(spec) |
| 76 | + spec.loader.exec_module(module) |
| 77 | + |
| 78 | + # Check for main, run, or execute (same as ScriptExecutor) |
| 79 | + for func_name in ["main", "run", "execute"]: |
| 80 | + func = getattr(module, func_name, None) |
| 81 | + if func is not None: |
| 82 | + sig = inspect.signature(func) |
| 83 | + params = list(sig.parameters.values()) |
| 84 | + |
| 85 | + if not params: |
| 86 | + continue |
| 87 | + |
| 88 | + first_param = params[0] |
| 89 | + if first_param.annotation == inspect.Parameter.empty: |
| 90 | + continue |
| 91 | + |
| 92 | + adapter = TypeAdapter(first_param.annotation) |
| 93 | + return adapter.json_schema() |
| 94 | + |
| 95 | + raise UiPathInputMockingError( |
| 96 | + f"No suitable entrypoint (main, run, execute) with typed parameters found in {entrypoint_path}" |
| 97 | + ) |
| 98 | + |
| 99 | + |
| 100 | +@traced(name="__mocker__") |
| 101 | +async def generate_llm_input( |
| 102 | + evaluation_item: EvaluationItem, |
| 103 | + input_schema: Dict[str, Any], |
| 104 | +) -> Dict[str, Any]: |
| 105 | + """Generate synthetic input using an LLM based on the evaluation context.""" |
| 106 | + try: |
| 107 | + llm = UiPath().llm |
| 108 | + |
| 109 | + prompt = get_input_mocking_prompt( |
| 110 | + input_schema=json.dumps(input_schema, indent=2), |
| 111 | + test_run_proctor_instructions=evaluation_item.simulation_instructions or "", |
| 112 | + input_generation_instructions=evaluation_item.input_generation_instructions |
| 113 | + or "", |
| 114 | + expected_behavior=evaluation_item.expected_agent_behavior or "", |
| 115 | + expected_output=json.dumps(evaluation_item.expected_output, indent=2) |
| 116 | + if evaluation_item.expected_output |
| 117 | + else "", |
| 118 | + ) |
| 119 | + |
| 120 | + cleaned_schema = _cleanup_schema(input_schema) |
| 121 | + |
| 122 | + response_format = { |
| 123 | + "type": "json_schema", |
| 124 | + "json_schema": { |
| 125 | + "name": "agent_input", |
| 126 | + "strict": True, |
| 127 | + "schema": cleaned_schema, |
| 128 | + }, |
| 129 | + } |
| 130 | + |
| 131 | + response = await llm.chat_completions( |
| 132 | + [{"role": "user", "content": prompt}], |
| 133 | + temperature=0.3, |
| 134 | + response_format=response_format, |
| 135 | + ) |
| 136 | + |
| 137 | + generated_input_str = response.choices[0].message.content |
| 138 | + if not generated_input_str: |
| 139 | + raise UiPathInputMockingError("LLM returned empty response") |
| 140 | + |
| 141 | + return json.loads(generated_input_str) |
| 142 | + except json.JSONDecodeError as e: |
| 143 | + raise UiPathInputMockingError( |
| 144 | + f"Failed to parse LLM response as JSON: {str(e)}" |
| 145 | + ) from e |
| 146 | + except UiPathInputMockingError: |
| 147 | + raise |
| 148 | + except Exception as e: |
| 149 | + raise UiPathInputMockingError(f"Failed to generate input: {str(e)}") from e |
0 commit comments