diff --git a/src/promptflow/promptflow/_sdk/_submitter/experiment_orchestrator.py b/src/promptflow/promptflow/_sdk/_submitter/experiment_orchestrator.py index bd2c71c3761..f748bfd5899 100644 --- a/src/promptflow/promptflow/_sdk/_submitter/experiment_orchestrator.py +++ b/src/promptflow/promptflow/_sdk/_submitter/experiment_orchestrator.py @@ -54,7 +54,7 @@ from promptflow._sdk._utils import overwrite_null_std_logger from promptflow._sdk.entities import Run from promptflow._sdk.entities._experiment import Experiment, ExperimentTemplate -from promptflow._sdk.entities._flow import ProtectedFlow +from promptflow._sdk.entities._utils import get_flow_definition from promptflow._sdk.operations import RunOperations from promptflow._sdk.operations._local_storage_operations import LocalStorageOperations from promptflow._utils.inputs_mapping_utils import apply_inputs_mapping @@ -101,8 +101,7 @@ def test( start_nodes = [ node for node in template.nodes - if node.type == ExperimentNodeType.FLOW - and ProtectedFlow._get_flow_definition(node.path) == ProtectedFlow._get_flow_definition(flow_path) + if node.type == ExperimentNodeType.FLOW and get_flow_definition(node.path) == get_flow_definition(flow_path) ] if not start_nodes: raise ExperimentValueError(f"Flow {flow_path.as_posix()} not found in experiment {template.dir_name!r}.") diff --git a/src/promptflow/promptflow/_sdk/_submitter/run_submitter.py b/src/promptflow/promptflow/_sdk/_submitter/run_submitter.py index 204a0a9fb42..f0db9d7227b 100644 --- a/src/promptflow/promptflow/_sdk/_submitter/run_submitter.py +++ b/src/promptflow/promptflow/_sdk/_submitter/run_submitter.py @@ -107,11 +107,13 @@ def _submit_bulk_run( ) -> dict: logger.info(f"Submitting run {run.name}, log path: {local_storage.logger.file_path}") run_id = run.name - if flow.language == FlowLanguage.CSharp: + if flow.language != FlowLanguage.Python: # TODO: consider moving this to Operations - from promptflow.batch import CSharpExecutorProxy + from promptflow.batch._executor_proxy_factory import ExecutorProxyFactory - CSharpExecutorProxy.generate_metadata(flow_file=Path(flow.path), working_dir=Path(flow.code)) + ExecutorProxyFactory().get_executor_proxy_cls(flow.language).generate_flow_metadata( + flow_file=Path(flow.path), working_dir=Path(flow.code), dump=True + ) # TODO: shall we resolve connections here? connections = [] else: diff --git a/src/promptflow/promptflow/_sdk/_submitter/test_submitter.py b/src/promptflow/promptflow/_sdk/_submitter/test_submitter.py index 50a9da179a6..81fa869ec05 100644 --- a/src/promptflow/promptflow/_sdk/_submitter/test_submitter.py +++ b/src/promptflow/promptflow/_sdk/_submitter/test_submitter.py @@ -29,6 +29,7 @@ from ..._utils.dataclass_serializer import convert_eager_flow_output_to_dict from ..._utils.logger_utils import get_cli_sdk_logger from ...batch import APIBasedExecutorProxy, CSharpExecutorProxy +from ...batch._executor_proxy_factory import ExecutorProxyFactory from .._configuration import Configuration from ..entities._eager_flow import EagerFlow from .utils import ( @@ -171,7 +172,7 @@ def _resolve_connections(cls, flow: FlowBase, client): return SubmitterHelper.resolve_used_connections( flow=flow, - tools_meta=CSharpExecutorProxy.get_tool_metadata( + tools_meta=CSharpExecutorProxy.generate_tool_metadata( flow_file=flow.flow_dag_path, working_dir=flow.code, ), @@ -241,9 +242,14 @@ def init( # temp flow is generated, will use self.flow instead of self._origin_flow in the following context self._within_init_context = True - if self.flow.language == FlowLanguage.CSharp: + if self.flow.language != FlowLanguage.Python: + ExecutorProxyFactory().get_executor_proxy_cls(self.flow.language).generate_flow_metadata( + flow_file=self.flow.path, + working_dir=self.flow.code, + dump=True, + ) # TODO: consider move this to Operations - CSharpExecutorProxy.generate_metadata(self.flow.path, self.flow.code) + CSharpExecutorProxy.generate_flow_metadata(self.flow.path, self.flow.code) self._target_node = target_node self._enable_stream_output = stream_output diff --git a/src/promptflow/promptflow/_sdk/entities/_eager_flow.py b/src/promptflow/promptflow/_sdk/entities/_eager_flow.py index 63a01bfc847..9c591d563fd 100644 --- a/src/promptflow/promptflow/_sdk/entities/_eager_flow.py +++ b/src/promptflow/promptflow/_sdk/entities/_eager_flow.py @@ -7,7 +7,6 @@ from promptflow._constants import LANGUAGE_KEY, FlowLanguage from promptflow._sdk._constants import BASE_PATH_CONTEXT_KEY -from promptflow._sdk._utils import generate_flow_meta from promptflow._sdk.entities._flow import FlowBase from promptflow._sdk.entities._validation import SchemaValidatableMixin from promptflow.exceptions import ErrorTarget, UserErrorException @@ -97,16 +96,19 @@ def _resolve_entry_file(cls, entry: str, working_dir: Path) -> Optional[str]: # when entry file not found in working directory, return None since it can come from package return None - # TODO: no usage? On the other hand, according to our latest design, we'd better avoid touching Executable - # everywhere in devkit. def _init_executable(self, **kwargs): + # TODO(2991934): support environment variables here + from promptflow.batch._executor_proxy_factory import ExecutorProxyFactory from promptflow.contracts.flow import EagerFlow as ExecutableEagerFlow - # TODO(2991934): support environment variables here - meta_dict = generate_flow_meta( - flow_directory=self.code, - source_path=self.entry_file, - entry=self.entry, - dump=False, + meta_dict = ( + ExecutorProxyFactory() + .get_executor_proxy_cls(self.language) + .generate_flow_metadata( + # TODO: is it possible that there is no path? + flow_file=self.path, + working_dir=self.code, + dump=False, + ) ) return ExecutableEagerFlow.deserialize(meta_dict) diff --git a/src/promptflow/promptflow/_sdk/entities/_flow.py b/src/promptflow/promptflow/_sdk/entities/_flow.py index 390d262f960..11d19f76660 100644 --- a/src/promptflow/promptflow/_sdk/entities/_flow.py +++ b/src/promptflow/promptflow/_sdk/entities/_flow.py @@ -6,19 +6,14 @@ import json from os import PathLike from pathlib import Path -from typing import Dict, Optional, Tuple, Union +from typing import Dict, Optional, Union from marshmallow import Schema from promptflow._constants import LANGUAGE_KEY, FlowLanguage -from promptflow._sdk._constants import ( - BASE_PATH_CONTEXT_KEY, - DAG_FILE_NAME, - DEFAULT_ENCODING, - FLOW_TOOLS_JSON, - PROMPT_FLOW_DIR_NAME, -) +from promptflow._sdk._constants import BASE_PATH_CONTEXT_KEY, DEFAULT_ENCODING, FLOW_TOOLS_JSON, PROMPT_FLOW_DIR_NAME from promptflow._sdk.entities._connection import _Connection +from promptflow._sdk.entities._utils import get_flow_definition from promptflow._sdk.entities._validation import SchemaValidatableMixin from promptflow._utils.flow_utils import resolve_flow_path from promptflow._utils.logger_utils import get_cli_sdk_logger @@ -247,7 +242,7 @@ def __init__( ): super().__init__(path=path, code=code, dag=dag, **kwargs) - self._flow_dir, self._dag_file_name = self._get_flow_definition(self.code) + self._flow_dir, self._dag_file_name = get_flow_definition(self.code) self._executable = None self._params_override = params_override @@ -273,20 +268,6 @@ def tools_meta_path(self) -> Path: target_path.parent.mkdir(parents=True, exist_ok=True) return target_path - @classmethod - def _get_flow_definition(cls, flow, base_path=None) -> Tuple[Path, str]: - if base_path: - flow_path = Path(base_path) / flow - else: - flow_path = Path(flow) - - if flow_path.is_dir() and (flow_path / DAG_FILE_NAME).is_file(): - return flow_path, DAG_FILE_NAME - elif flow_path.is_file(): - return flow_path.parent, flow_path.name - - raise ValueError(f"Can't find flow with path {flow_path.as_posix()}.") - # region SchemaValidatableMixin @classmethod def _create_schema_for_validation(cls, context) -> Schema: diff --git a/src/promptflow/promptflow/_sdk/entities/_utils.py b/src/promptflow/promptflow/_sdk/entities/_utils.py new file mode 100644 index 00000000000..e9b66b9978f --- /dev/null +++ b/src/promptflow/promptflow/_sdk/entities/_utils.py @@ -0,0 +1,41 @@ +# --------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# --------------------------------------------------------- + +""" +Previously we have put some shared logic for flow in ProtectedFlow class. However, in practice, +it's hard to guarantee that we always use this class to create flow; on the other hand, the more + +""" +from os import PathLike +from pathlib import Path +from typing import Tuple, Union + +from promptflow._sdk._constants import DAG_FILE_NAME + + +def get_flow_definition( + flow_path: Union[str, Path, PathLike], base_path: Union[str, Path, PathLike, None] = None +) -> Tuple[Path, str]: + """Resolve flow path and return the flow directory path and the file name of the target yaml. + + :param flow_path: The path of the flow directory or the flow yaml file. It can either point to a + flow directory or a flow yaml file. + :type flow_path: Union[str, Path, PathLike] + :param base_path: The base path to resolve the flow path. If not specified, the flow path will be + resolved based on the current working directory. + :type base_path: Union[str, Path, PathLike] + :return: The flow directory path and the file name of the target yaml. + :rtype: Tuple[Path, str] + """ + if base_path: + flow_path = Path(base_path) / flow_path + else: + flow_path = Path(flow_path) + + if flow_path.is_dir() and (flow_path / DAG_FILE_NAME).is_file(): + return flow_path, DAG_FILE_NAME + elif flow_path.is_file(): + return flow_path.parent, flow_path.name + + raise ValueError(f"Can't find flow with path {flow_path.as_posix()}.") diff --git a/src/promptflow/promptflow/_sdk/operations/_flow_operations.py b/src/promptflow/promptflow/_sdk/operations/_flow_operations.py index c643ed60722..8f2a087fbec 100644 --- a/src/promptflow/promptflow/_sdk/operations/_flow_operations.py +++ b/src/promptflow/promptflow/_sdk/operations/_flow_operations.py @@ -31,7 +31,6 @@ _merge_local_code_and_additional_includes, copy_tree_respect_template_and_ignore_file, dump_flow_result, - generate_flow_meta, generate_flow_tools_json, generate_random_string, logger, @@ -410,7 +409,7 @@ def _export_flow_connections( return self._migrate_connections( connection_names=SubmitterHelper.get_used_connection_names( - tools_meta=CSharpExecutorProxy.get_tool_metadata( + tools_meta=CSharpExecutorProxy.generate_tool_metadata( flow_file=flow.flow_dag_path, working_dir=flow.code, ), @@ -835,13 +834,18 @@ def _generate_flow_meta( # No flow meta for DAG flow return {} + # TODO: is it possible that there is no flow.path? with self._resolve_additional_includes(flow.path) as new_flow_dag_path: - # TODO: support generate flow meta for csharp? - return generate_flow_meta( - flow_directory=new_flow_dag_path.parent, - source_path=flow.entry_file, - entry=flow.entry, - dump=dump, - timeout=timeout, - load_in_subprocess=load_in_subprocess, + from promptflow.batch._executor_proxy_factory import ExecutorProxyFactory + + return ( + ExecutorProxyFactory() + .get_executor_proxy_cls(flow.language) + .generate_flow_metadata( + flow_file=new_flow_dag_path, + working_dir=new_flow_dag_path.parent, + dump=dump, + timeout=timeout, + load_in_subprocess=load_in_subprocess, + ) ) diff --git a/src/promptflow/promptflow/azure/operations/_flow_operations.py b/src/promptflow/promptflow/azure/operations/_flow_operations.py index d3c297cd810..e5122cbcacf 100644 --- a/src/promptflow/promptflow/azure/operations/_flow_operations.py +++ b/src/promptflow/promptflow/azure/operations/_flow_operations.py @@ -36,6 +36,7 @@ from promptflow._sdk._telemetry import ActivityType, WorkspaceTelemetryMixin, monitor_operation from promptflow._sdk._utils import PromptflowIgnoreFile from promptflow._sdk._vendor._asset_utils import traverse_directory +from promptflow._sdk.entities._utils import get_flow_definition from promptflow._utils.logger_utils import get_cli_sdk_logger from promptflow.azure._constants._flow import DEFAULT_STORAGE from promptflow.azure._entities._flow import Flow @@ -43,6 +44,7 @@ from promptflow.azure._restclient.flow_service_caller import FlowServiceCaller from promptflow.azure.operations._artifact_utilities import _get_datastore_name, get_datastore_info from promptflow.azure.operations._fileshare_storeage_helper import FlowFileStorageClient +from promptflow.batch._executor_proxy_factory import ExecutorProxyFactory from promptflow.exceptions import SystemErrorException, UserErrorException logger = get_cli_sdk_logger() @@ -482,7 +484,11 @@ def _try_resolve_code_for_flow(cls, flow: Flow, ops: OperationOrchestrator, igno return # generate .promptflow/flow.json for eager flow - cls._generate_meta_for_eager_flow(code=code) + flow_directory, flow_file = get_flow_definition(code.path) + ExecutorProxyFactory().get_executor_proxy_cls(flow.language).generate_flow_metadata( + flow_file=flow_directory / flow_file, + working_dir=flow_directory, + ) if ignore_tools_json: ignore_file = code._ignore_file @@ -599,18 +605,3 @@ def _try_resolve_code_for_flow_to_file_share(cls, flow: Flow, ops: OperationOrch flow._code_uploaded = True # endregion - - @classmethod - def _generate_meta_for_eager_flow(cls, code): - from promptflow import load_flow as load_local_flow - from promptflow._sdk.entities._eager_flow import EagerFlow - - flow = load_local_flow(code.path) - if isinstance(flow, EagerFlow): - from promptflow.batch._executor_proxy_factory import ExecutorProxyFactory - - ExecutorProxyFactory().get_executor_proxy_cls(flow.language).generate_metadata( - flow_file=code.path, - # TODO: may need to resolve additional includes after supported - working_dir=code.path, - ) diff --git a/src/promptflow/promptflow/batch/_base_executor_proxy.py b/src/promptflow/promptflow/batch/_base_executor_proxy.py index c3cf3db159e..00cbc494a20 100644 --- a/src/promptflow/promptflow/batch/_base_executor_proxy.py +++ b/src/promptflow/promptflow/batch/_base_executor_proxy.py @@ -7,13 +7,13 @@ from datetime import datetime from json import JSONDecodeError from pathlib import Path -from typing import Any, Mapping, Optional +from typing import Any, Dict, Mapping, Optional import httpx from promptflow._constants import DEFAULT_ENCODING, LINE_TIMEOUT_SEC from promptflow._core._errors import MetaFileNotFound, MetaFileReadError, NotSupported, UnexpectedError -from promptflow._sdk._constants import FLOW_META_JSON, FLOW_TOOLS_JSON, PROMPT_FLOW_DIR_NAME +from promptflow._sdk._constants import FLOW_META_JSON, FLOW_META_JSON_GEN_TIMEOUT, FLOW_TOOLS_JSON, PROMPT_FLOW_DIR_NAME from promptflow._utils.async_utils import async_run_allowing_running_loop from promptflow._utils.exception_utils import ErrorResponse, ExceptionPresenter from promptflow._utils.logger_utils import bulk_logger @@ -29,17 +29,39 @@ class AbstractExecutorProxy: @classmethod - def get_tool_metadata(cls, flow_file: Path, working_dir: Optional[Path] = None) -> dict: + def generate_tool_metadata(cls, flow_file: Path, working_dir: Optional[Path] = None) -> dict: """Generate tool metadata file for the specified flow.""" return cls._get_tool_metadata(flow_file, working_dir or flow_file.parent) @classmethod - def generate_metadata(cls, flow_file: Path, working_dir: Path = None): - """Generate metadata for the flow and save them to files under .promptflow folder.""" - cls._generate_metadata(flow_file, working_dir or flow_file.parent) + def generate_flow_metadata( + cls, + flow_file: Path, + working_dir: Path, + dump: bool = True, + timeout: int = FLOW_META_JSON_GEN_TIMEOUT, + load_in_subprocess: bool = True, + ) -> Dict[str, Any]: + """Generate metadata for a specific flow. + + :param flow_file: The path of the flow file. + :type flow_file: Path + :param working_dir: The working directory to generate the flow metadata. It will impact the packages to load + and the location of generated flow.json file when dump is True. + :type working_dir: Path + :param dump: Whether to dump the metadata to .promptflow/flow.json. + :type dump: bool + :param timeout: The timeout for the flow execution. Default timeout is 60 seconds. + :type timeout: int + :param load_in_subprocess: Whether to load the flow in a subprocess. This parameter works for Python flow only. + :type load_in_subprocess: bool + :return: The metadata of the flow. + :rtype: Dict[str, Any] + """ + raise NotImplementedError() @classmethod - def _generate_metadata(cls, flow_file: Path, working_dir: Path): + def _generate_flow_metadata(cls, flow_file: Path, working_dir: Path): """Generate metadata for the flow and save them to files under .promptflow folder. including flow.json and flow.tools.json. """ @@ -170,17 +192,7 @@ async def destroy_if_all_generators_exhausted(self): def _get_flow_meta(self) -> dict: flow_meta_json_path = self.working_dir / PROMPT_FLOW_DIR_NAME / FLOW_META_JSON - if not flow_meta_json_path.is_file(): - raise MetaFileNotFound( - message_format=( - # TODO: pf flow validate should be able to generate flow.json - "Failed to fetch meta of inputs: cannot find {file_path}, please retry." - ), - file_path=flow_meta_json_path.absolute().as_posix(), - ) - - with open(flow_meta_json_path, mode="r", encoding=DEFAULT_ENCODING) as flow_meta_json_path: - return json.load(flow_meta_json_path) + return self._read_json_content(flow_meta_json_path, "meta of flow") def get_inputs_definition(self): """Get the inputs definition of an eager flow""" @@ -200,20 +212,26 @@ def get_inputs_definition(self): @classmethod def _get_tool_metadata(cls, flow_file: Path, working_dir: Path) -> dict: flow_tools_json_path = working_dir / PROMPT_FLOW_DIR_NAME / FLOW_TOOLS_JSON - if flow_tools_json_path.is_file(): - with open(flow_tools_json_path, mode="r", encoding=DEFAULT_ENCODING) as f: + return cls._read_json_content(flow_tools_json_path, "meta of tools") + + @classmethod + def _read_json_content(cls, file_path: Path, target: str) -> dict: + if file_path.is_file(): + with open(file_path, mode="r", encoding=DEFAULT_ENCODING) as f: try: return json.load(f) except json.JSONDecodeError: raise MetaFileReadError( - message_format="Failed to fetch meta of tools: {file_path} is not a valid json file.", - file_path=flow_tools_json_path.absolute().as_posix(), + message_format="Failed to fetch {target_obj}: {file_path} is not a valid json file.", + file_path=file_path.absolute().as_posix(), + target_obj=target, ) raise MetaFileNotFound( message_format=( - "Failed to fetch meta of tools: cannot find {file_path}, please build the flow project first." + "Failed to fetch meta of tools: cannot find {file_path}, " + "please build the flow project with extension first." ), - file_path=flow_tools_json_path.absolute().as_posix(), + file_path=file_path.absolute().as_posix(), ) @property diff --git a/src/promptflow/promptflow/batch/_csharp_executor_proxy.py b/src/promptflow/promptflow/batch/_csharp_executor_proxy.py index 60e846cafcf..7e8123c8d0c 100644 --- a/src/promptflow/promptflow/batch/_csharp_executor_proxy.py +++ b/src/promptflow/promptflow/batch/_csharp_executor_proxy.py @@ -1,13 +1,15 @@ # --------------------------------------------------------- # Copyright (c) Microsoft Corporation. All rights reserved. # --------------------------------------------------------- +import json import socket import subprocess import uuid from pathlib import Path -from typing import Optional +from typing import Any, Dict, Optional from promptflow._core._errors import UnexpectedError +from promptflow._sdk._constants import FLOW_META_JSON, FLOW_META_JSON_GEN_TIMEOUT, PROMPT_FLOW_DIR_NAME from promptflow.batch._csharp_base_executor_proxy import CSharpBaseExecutorProxy from promptflow.storage._run_storage import AbstractRunStorage @@ -46,9 +48,34 @@ def chat_output_name(self) -> Optional[str]: return self._chat_output_name @classmethod - def _generate_metadata(cls, flow_file: Path, working_dir: Path): - """Generate metadata for the flow and save them to files under .promptflow folder. - including flow.json and flow.tools.json. + def generate_flow_metadata( + cls, + flow_file: Path, + working_dir: Path, + dump: bool = True, + timeout: int = FLOW_META_JSON_GEN_TIMEOUT, + load_in_subprocess: bool = True, + ) -> Dict[str, Any]: + # TODO: timeout & dump doesn't take effect for now + # TODO: provide a way to skip dumping and directly read from flow.json + cls._dump_metadata( + flow_file=flow_file, + working_dir=working_dir, + ) + + from promptflow import load_flow + from promptflow._sdk.entities._eager_flow import EagerFlow + + flow = load_flow(flow_file) + if isinstance(flow, EagerFlow): + return json.load((working_dir / PROMPT_FLOW_DIR_NAME / FLOW_META_JSON).open()) + else: + return {} + + @classmethod + def _dump_metadata(cls, flow_file: Path, working_dir: Path): + """In csharp, we need to generate metadata based on a dotnet command for now and the metadata will + always be dumped. """ command = [ "dotnet", diff --git a/src/promptflow/promptflow/batch/_executor_proxy_factory.py b/src/promptflow/promptflow/batch/_executor_proxy_factory.py index 416faf2c8c0..7870bca9a82 100644 --- a/src/promptflow/promptflow/batch/_executor_proxy_factory.py +++ b/src/promptflow/promptflow/batch/_executor_proxy_factory.py @@ -1,12 +1,15 @@ -from typing import Dict +from typing import Dict, Type from promptflow._constants import FlowLanguage from promptflow._utils.async_utils import async_run_allowing_running_loop -from promptflow.batch import AbstractExecutorProxy, CSharpExecutorProxy, PythonExecutorProxy + +from ._base_executor_proxy import AbstractExecutorProxy +from ._csharp_executor_proxy import CSharpExecutorProxy +from ._python_executor_proxy import PythonExecutorProxy class ExecutorProxyFactory: - executor_proxy_classes: Dict[str, AbstractExecutorProxy] = { + executor_proxy_classes: Dict[str, Type[AbstractExecutorProxy]] = { FlowLanguage.Python: PythonExecutorProxy, FlowLanguage.CSharp: CSharpExecutorProxy, } @@ -14,11 +17,11 @@ class ExecutorProxyFactory: def __init__(self): pass - def get_executor_proxy_cls(self, language: str) -> AbstractExecutorProxy: + def get_executor_proxy_cls(self, language: str) -> Type[AbstractExecutorProxy]: return self.executor_proxy_classes[language] @classmethod - def register_executor(cls, language: str, executor_proxy_cls: AbstractExecutorProxy): + def register_executor(cls, language: str, executor_proxy_cls: Type[AbstractExecutorProxy]): """Register a executor proxy class for a specific program language. This method allows users to register a executor proxy class for a particular diff --git a/src/promptflow/promptflow/batch/_python_executor_proxy.py b/src/promptflow/promptflow/batch/_python_executor_proxy.py index 8b9f82592f8..993b49741ee 100644 --- a/src/promptflow/promptflow/batch/_python_executor_proxy.py +++ b/src/promptflow/promptflow/batch/_python_executor_proxy.py @@ -3,11 +3,12 @@ # --------------------------------------------------------- from pathlib import Path -from typing import Any, List, Mapping, Optional, Tuple +from typing import Any, Dict, List, Mapping, Optional, Tuple from promptflow._core._errors import UnexpectedError from promptflow._core.operation_context import OperationContext from promptflow._core.run_tracker import RunTracker +from promptflow._sdk._constants import FLOW_META_JSON_GEN_TIMEOUT from promptflow._utils.logger_utils import bulk_logger from promptflow.batch._base_executor_proxy import AbstractExecutorProxy from promptflow.contracts.run_mode import RunMode @@ -23,31 +24,32 @@ def __init__(self, flow_executor: FlowExecutor): self._flow_executor = flow_executor @classmethod - def _generate_metadata(cls, flow_file: Path, working_dir: Path): - """Generate metadata for the flow and save them to files under .promptflow folder. - including flow.json and flow.tools.json. - """ - + def generate_flow_metadata( + cls, + flow_file: Path, + working_dir: Path, + dump: bool = True, + timeout: int = FLOW_META_JSON_GEN_TIMEOUT, + load_in_subprocess: bool = True, + ) -> Dict[str, Any]: from promptflow import load_flow - from promptflow._sdk._utils import generate_flow_meta, generate_flow_tools_json + from promptflow._sdk._utils import generate_flow_meta from promptflow._sdk.entities._eager_flow import EagerFlow flow = load_flow(flow_file) if isinstance(flow, EagerFlow): # generate flow.json only for eager flow for now - generate_flow_meta( + return generate_flow_meta( flow_directory=working_dir, source_path=flow.entry_file, entry=flow.entry, - dump=True, + dump=dump, + timeout=timeout, + load_in_subprocess=load_in_subprocess, ) else: - # generate flow.tools.json for non-eager flow for now - generate_flow_tools_json( - flow_directory=working_dir, - dump=True, - used_packages_only=True, - ) + # we will also skip dump for now + return {} @classmethod async def create( diff --git a/src/promptflow/tests/executor/unittests/batch/test_csharp_executor_proxy.py b/src/promptflow/tests/executor/unittests/batch/test_csharp_executor_proxy.py index f78747378bd..3007d9d7674 100644 --- a/src/promptflow/tests/executor/unittests/batch/test_csharp_executor_proxy.py +++ b/src/promptflow/tests/executor/unittests/batch/test_csharp_executor_proxy.py @@ -105,13 +105,13 @@ def test_get_tool_metadata_succeed(self): with open(tool_meta_file, "w") as file: json.dump(expected_tool_meta, file, indent=4) - tool_meta = CSharpExecutorProxy.get_tool_metadata("", working_dir) + tool_meta = CSharpExecutorProxy.generate_tool_metadata("", working_dir) assert tool_meta == expected_tool_meta def test_get_tool_metadata_failed_with_file_not_found(self): working_dir = Path(mkdtemp()) with pytest.raises(MetaFileNotFound): - CSharpExecutorProxy.get_tool_metadata("", working_dir) + CSharpExecutorProxy.generate_tool_metadata("", working_dir) def test_get_tool_metadata_failed_with_content_not_json(self): working_dir = Path(mkdtemp()) @@ -120,7 +120,7 @@ def test_get_tool_metadata_failed_with_content_not_json(self): tool_meta_file.touch() with pytest.raises(MetaFileReadError): - CSharpExecutorProxy.get_tool_metadata("", working_dir) + CSharpExecutorProxy.generate_tool_metadata("", working_dir) def test_find_available_port(self): port = CSharpExecutorProxy.find_available_port()