Support data asset input

examples/gen_test_data/config.ini.example

@@ -11,6 +11,7 @@ document_chunk_size = 512
 document_chunk_overlap = 100
 ; However, if you wish to bypass the document split process, simply provide the 'document_nodes_file', which is a JSONL file.
 ; When both `documents_folder` and `document_nodes_file` are configured, will use 'document_nodes_file' and ignore 'documents_folder'.
+; For cloud mode, both local files and data assets can be used.
 ; document_nodes_file = "<your-node-file-path>"
 
 ; Test data gen flow configs

examples/gen_test_data/gen_test_data/common.py

@@ -185,3 +185,11 @@ def convert_to_abs_path(file_path: str) -> str:
         return abs
     else:
         return file_path
+
+
+def local_path_exists(path):
+    return Path(path).exists()
+
+
+def non_padding_path(path):
+    return not (path.startswith("<") and path.endswith(">"))

examples/gen_test_data/gen_test_data/run.py

@@ -16,7 +16,7 @@
 
 from common import clean_data, count_non_blank_lines, \
     split_document, copy_flow_folder_and_set_node_inputs, \
-    print_progress, convert_to_abs_path  # noqa: E402
+    print_progress, convert_to_abs_path, non_padding_path, local_path_exists  # noqa: E402
 from constants import TEXT_CHUNK, DETAILS_FILE_NAME  # noqa: E402
 
 logger = get_logger("data.gen")
@@ -276,10 +276,11 @@ def get_ml_client(subscription_id: str, resource_group: str, workspace_name: str
         documents_folder = convert_to_abs_path(args.documents_folder)
         flow_folder = convert_to_abs_path(args.flow_folder)
         output_folder = convert_to_abs_path(args.output_folder)
+        validate_path_func = non_padding_path if args.cloud else local_path_exists
 
-        if document_nodes_file and Path(document_nodes_file).is_file():
+        if document_nodes_file and validate_path_func(document_nodes_file):
             should_skip_split_documents = True
-        elif not documents_folder or not Path(documents_folder).is_dir():
+        elif not documents_folder or not validate_path_func(documents_folder):
             parser.error(
                 "Either 'documents_folder' or 'document_nodes_file' should be specified correctly.\n"
                 f"documents_folder: '{documents_folder}'\ndocument_nodes_file: '{document_nodes_file}'"
@@ -295,7 +296,8 @@ def get_ml_client(subscription_id: str, resource_group: str, workspace_name: str
                 "Skip step 1 'Split documents to document nodes' as received document nodes from "
                 f"input file path '{document_nodes_file}'."
             )
-            logger.info(f"Collected {count_non_blank_lines(document_nodes_file)} document nodes.")
+            if Path(document_nodes_file).is_file():
+                logger.info(f"Collected {count_non_blank_lines(document_nodes_file)} document nodes.")
 
         copy_flow_folder_and_set_node_inputs(copied_flow_folder, args.flow_folder, args.node_inputs_override)