diff --git a/examples/gen_test_data/gen_test_data/generate_test_data_flow/flow.dag.yaml b/examples/gen_test_data/gen_test_data/generate_test_data_flow/flow.dag.yaml index 608f22948a8..2ecb428677e 100644 --- a/examples/gen_test_data/gen_test_data/generate_test_data_flow/flow.dag.yaml +++ b/examples/gen_test_data/gen_test_data/generate_test_data_flow/flow.dag.yaml @@ -96,7 +96,6 @@ nodes: type: code path: generate_debug_info.py inputs: - question_type: ${validate_question.output.question_type} text_chunk: ${inputs.text_chunk} validate_suggested_answer_output: ${validate_suggested_answer.output} text_chunk_validation_res: ${validate_text_chunk.output.validation_res} diff --git a/examples/gen_test_data/gen_test_data/generate_test_data_flow/generate_debug_info.py b/examples/gen_test_data/gen_test_data/generate_test_data_flow/generate_debug_info.py index fd703204772..487147153f7 100644 --- a/examples/gen_test_data/gen_test_data/generate_test_data_flow/generate_debug_info.py +++ b/examples/gen_test_data/gen_test_data/generate_test_data_flow/generate_debug_info.py @@ -8,59 +8,40 @@ # Please update the function name/signature per need @tool def my_python_tool( - question_type: str, text_chunk: str, text_chunk_validation_res: ValidationResult = None, validate_question_output: dict = None, validate_suggested_answer_output: dict = None, ) -> dict: - generated_question = validate_question_output["question"] question_validation_res = validate_question_output["validation_res"] generated_suggested_answer = validate_suggested_answer_output["suggested_answer"] suggested_answer_validation_res = validate_suggested_answer_output["validation_res"] is_generation_success = generated_suggested_answer != "" - is_text_chunk_valid = text_chunk_validation_res.pass_validation if text_chunk_validation_res else None - is_seed_question_valid = question_validation_res.pass_validation if question_validation_res else None + is_text_chunk_valid = text_chunk_validation_res["pass_validation"] if text_chunk_validation_res else None + is_seed_question_valid = question_validation_res["pass_validation"] if question_validation_res else None is_suggested_answer_valid = ( - suggested_answer_validation_res.pass_validation if suggested_answer_validation_res else None + suggested_answer_validation_res["pass_validation"] if suggested_answer_validation_res else None ) failed_step = "" - failed_reason = "" if not is_generation_success: if is_text_chunk_valid is False: failed_step = ValidateObj.TEXT_CHUNK - failed_reason = text_chunk_validation_res.reason elif is_seed_question_valid is False: failed_step = ValidateObj.QUESTION - failed_reason = question_validation_res.reason elif is_suggested_answer_valid is False: failed_step = ValidateObj.SUGGESTED_ANSWER - failed_reason = suggested_answer_validation_res.reason return { # TODO: support more question types like multi-context etc. # "question_type": question_type, "text_chunk": text_chunk, - "validation_summary": { - "success": is_generation_success, - "failed_step": failed_step - }, + "validation_summary": {"success": is_generation_success, "failed_step": failed_step}, "validation_details": { - ValidateObj.TEXT_CHUNK: { - "score": text_chunk_validation_res.score if text_chunk_validation_res else None, - "success": is_text_chunk_valid, - "reason": text_chunk_validation_res.reason if text_chunk_validation_res else None, - }, - ValidateObj.QUESTION: { - "success": is_seed_question_valid, - "reason": question_validation_res.reason if question_validation_res else None, - }, - ValidateObj.SUGGESTED_ANSWER: { - "success": is_suggested_answer_valid, - "reason": suggested_answer_validation_res.reason if suggested_answer_validation_res else None, - }, + ValidateObj.TEXT_CHUNK: text_chunk_validation_res, + ValidateObj.QUESTION: question_validation_res, + ValidateObj.SUGGESTED_ANSWER: suggested_answer_validation_res, }, } diff --git a/examples/gen_test_data/gen_test_data/generate_test_data_flow/validate_question.py b/examples/gen_test_data/gen_test_data/generate_test_data_flow/validate_question.py index 3442f23581a..20fcfd18c35 100644 --- a/examples/gen_test_data/gen_test_data/generate_test_data_flow/validate_question.py +++ b/examples/gen_test_data/gen_test_data/generate_test_data_flow/validate_question.py @@ -45,4 +45,4 @@ def validate_question( question = generated_question question_type = QuestionType.SIMPLE - return {"question": question, "question_type": question_type, "validation_res": validation_res} + return {"question": question, "question_type": question_type, "validation_res": validation_res._asdict()} diff --git a/examples/gen_test_data/gen_test_data/generate_test_data_flow/validate_suggested_answer.py b/examples/gen_test_data/gen_test_data/generate_test_data_flow/validate_suggested_answer.py index 491f2327618..636687957f5 100644 --- a/examples/gen_test_data/gen_test_data/generate_test_data_flow/validate_suggested_answer.py +++ b/examples/gen_test_data/gen_test_data/generate_test_data_flow/validate_suggested_answer.py @@ -16,7 +16,7 @@ def validate_suggested_answer( suggested_answer: str, validate_suggested_answer_prompt: str, temperature: float = 0.2, - response_format: str = "text" + response_format: str = "text", ): """ 1. Validates the given suggested answer. @@ -33,7 +33,7 @@ def validate_suggested_answer( validate_suggested_answer_prompt, suggested_answer, temperature, - response_format=response_format + response_format=response_format, ) is_valid_gt = validation_res.pass_validation failed_reason = "" @@ -42,4 +42,4 @@ def validate_suggested_answer( print(failed_reason) suggested_answer = "" - return {"suggested_answer": suggested_answer, "validation_res": validation_res} + return {"suggested_answer": suggested_answer, "validation_res": validation_res._asdict()} diff --git a/examples/gen_test_data/gen_test_data/generate_test_data_flow/validate_text_chunk.py b/examples/gen_test_data/gen_test_data/generate_test_data_flow/validate_text_chunk.py index b724a7686e4..a45b57ec3e6 100644 --- a/examples/gen_test_data/gen_test_data/generate_test_data_flow/validate_text_chunk.py +++ b/examples/gen_test_data/gen_test_data/generate_test_data_flow/validate_text_chunk.py @@ -32,6 +32,6 @@ def validate_text_chunk( ) if not text_chunk_score_res.pass_validation: print(ErrorMsg.INVALID_TEXT_CHUNK.format(context)) - return {"context": "", "validation_res": text_chunk_score_res} + return {"context": "", "validation_res": text_chunk_score_res._asdict()} - return {"context": context, "validation_res": text_chunk_score_res} + return {"context": context, "validation_res": text_chunk_score_res._asdict()}