From 3172b9e31268299586420360925aec28e07184ee Mon Sep 17 00:00:00 2001 From: Henrique Joaquim Date: Thu, 11 Apr 2024 15:01:36 +0100 Subject: [PATCH] [BugFix] Fix metadata creation (#6306) * fix metadata creation * fix metadata creation * adapt tests * adapt tests * adapt tests --------- Co-authored-by: hjoaquim --- .../core/openbb_core/app/model/metadata.py | 177 +++++++++--------- .../core/tests/app/model/test_metadata.py | 34 +++- .../core/tests/app/test_command_runner.py | 1 + 3 files changed, 121 insertions(+), 91 deletions(-) diff --git a/openbb_platform/core/openbb_core/app/model/metadata.py b/openbb_platform/core/openbb_core/app/model/metadata.py index eca37b061f99..70672dd15957 100644 --- a/openbb_platform/core/openbb_core/app/model/metadata.py +++ b/openbb_platform/core/openbb_core/app/model/metadata.py @@ -41,90 +41,95 @@ def scale_arguments(cls, v): containing the type and the columns. If the type is not one of the previous, the value is kept or trimmed to 80 characters. """ - for arg, arg_val in v.items(): - new_arg_val: Optional[Union[str, dict[str, Sequence[Any]]]] = None - - # Data - if isclass(type(arg_val)) and issubclass(type(arg_val), Data): - new_arg_val = { - "type": f"{type(arg_val).__name__}", - "columns": list(arg_val.model_dump().keys()), - } - - # List[Data] - if isinstance(arg_val, list) and issubclass(type(arg_val[0]), Data): - _columns = [list(d.model_dump().keys()) for d in arg_val] - ld_columns = ( - item for sublist in _columns for item in sublist - ) # flatten - new_arg_val = { - "type": f"List[{type(arg_val[0]).__name__}]", - "columns": list(set(ld_columns)), - } - - # DataFrame - elif isinstance(arg_val, pd.DataFrame): - df_columns = ( - list(arg_val.index.names) + arg_val.columns.tolist() - if any(index is not None for index in list(arg_val.index.names)) - else arg_val.columns.tolist() - ) - new_arg_val = { - "type": f"{type(arg_val).__name__}", - "columns": df_columns, - } - - # List[DataFrame] - elif isinstance(arg_val, list) and issubclass( - type(arg_val[0]), pd.DataFrame - ): - ldf_columns = [ - ( - list(df.index.names) + df.columns.tolist() - if any(index is not None for index in list(df.index.names)) - else df.columns.tolist() + arguments: Dict[str, Any] = {} + for item in ["provider_choices", "standard_params", "extra_params"]: + arguments[item] = {} + for arg, arg_val in v[item].items(): + new_arg_val: Optional[Union[str, dict[str, Sequence[Any]]]] = None + + # Data + if isclass(type(arg_val)) and issubclass(type(arg_val), Data): + new_arg_val = { + "type": f"{type(arg_val).__name__}", + "columns": list(arg_val.model_dump().keys()), + } + + # List[Data] + if isinstance(arg_val, list) and issubclass(type(arg_val[0]), Data): + _columns = [list(d.model_dump().keys()) for d in arg_val] + ld_columns = ( + item for sublist in _columns for item in sublist + ) # flatten + new_arg_val = { + "type": f"List[{type(arg_val[0]).__name__}]", + "columns": list(set(ld_columns)), + } + + # DataFrame + elif isinstance(arg_val, pd.DataFrame): + df_columns = ( + list(arg_val.index.names) + arg_val.columns.tolist() + if any(index is not None for index in list(arg_val.index.names)) + else arg_val.columns.tolist() ) - for df in arg_val - ] - new_arg_val = { - "type": f"List[{type(arg_val[0]).__name__}]", - "columns": ldf_columns, - } - - # Series - elif isinstance(arg_val, pd.Series): - new_arg_val = { - "type": f"{type(arg_val).__name__}", - "columns": list(arg_val.index.names) + [arg_val.name], - } - - # List[Series] - elif isinstance(arg_val, list) and isinstance(arg_val[0], pd.Series): - ls_columns = [ - ( - list(series.index.names) + [series.name] - if any(index is not None for index in list(series.index.names)) - else series.name - ) - for series in arg_val - ] - new_arg_val = { - "type": f"List[{type(arg_val[0]).__name__}]", - "columns": ls_columns, - } - - # ndarray - elif isinstance(arg_val, np.ndarray): - new_arg_val = { - "type": f"{type(arg_val).__name__}", - "columns": list(arg_val.dtype.names or []), - } - - else: - str_repr_arg_val = str(arg_val) - if len(str_repr_arg_val) > 80: - new_arg_val = str_repr_arg_val[:80] - - v[arg] = new_arg_val or arg_val - - return v + new_arg_val = { + "type": f"{type(arg_val).__name__}", + "columns": df_columns, + } + + # List[DataFrame] + elif isinstance(arg_val, list) and issubclass( + type(arg_val[0]), pd.DataFrame + ): + ldf_columns = [ + ( + list(df.index.names) + df.columns.tolist() + if any(index is not None for index in list(df.index.names)) + else df.columns.tolist() + ) + for df in arg_val + ] + new_arg_val = { + "type": f"List[{type(arg_val[0]).__name__}]", + "columns": ldf_columns, + } + + # Series + elif isinstance(arg_val, pd.Series): + new_arg_val = { + "type": f"{type(arg_val).__name__}", + "columns": list(arg_val.index.names) + [arg_val.name], + } + + # List[Series] + elif isinstance(arg_val, list) and isinstance(arg_val[0], pd.Series): + ls_columns = [ + ( + list(series.index.names) + [series.name] + if any( + index is not None for index in list(series.index.names) + ) + else series.name + ) + for series in arg_val + ] + new_arg_val = { + "type": f"List[{type(arg_val[0]).__name__}]", + "columns": ls_columns, + } + + # ndarray + elif isinstance(arg_val, np.ndarray): + new_arg_val = { + "type": f"{type(arg_val).__name__}", + "columns": list(arg_val.dtype.names or []), + } + + else: + str_repr_arg_val = str(arg_val) + if len(str_repr_arg_val) > 80: + new_arg_val = str_repr_arg_val[:80] + + arguments[item][arg] = new_arg_val or arg_val + + return arguments diff --git a/openbb_platform/core/tests/app/model/test_metadata.py b/openbb_platform/core/tests/app/model/test_metadata.py index 0368271c6ef5..37bf7377984d 100644 --- a/openbb_platform/core/tests/app/model/test_metadata.py +++ b/openbb_platform/core/tests/app/model/test_metadata.py @@ -12,7 +12,11 @@ def test_Metadata(): """Run Smoke test.""" m = Metadata( - arguments={"test": "test"}, + arguments={ + "provider_choices": {}, + "standard_params": {}, + "extra_params": {}, + }, route="test", timestamp=datetime.now(), duration=0, @@ -44,7 +48,7 @@ def test_fields(): { "data_list": { "type": "List[Data]", - "columns": ["open", "close", "volume"], + "columns": ["open", "volume", "close"], } }, ), @@ -101,8 +105,13 @@ def test_fields(): ) def test_scale_arguments(input_data, expected_output): """Test the scale_arguments method.""" + kwargs = { + "provider_choices": {}, + "standard_params": {}, + "extra_params": input_data, + } m = Metadata( - arguments=input_data, + arguments=kwargs, route="test", timestamp=datetime.now(), duration=0, @@ -112,9 +121,24 @@ def test_scale_arguments(input_data, expected_output): for arg in arguments: # pylint: disable=E1133 if "columns" in arguments[arg]: # compare the column names disregarding the order with the expected output - assert sorted(arguments[arg]["columns"]) == sorted( + assert sorted(arguments["extra_params"][arg]["columns"]) == sorted( expected_output[arg]["columns"] ) assert arguments[arg]["type"] == expected_output[arg]["type"] else: - assert m.arguments == expected_output + # assert m.arguments["extra_params"] == expected_output + keys = list(arguments["extra_params"].keys()) + expected_keys = list(expected_output.keys()) + assert sorted(keys) == sorted(expected_keys) + + for key in keys: + if "type" in arguments["extra_params"][key]: + assert ( + arguments["extra_params"][key]["type"] + == expected_output[key]["type"] + ) + assert sorted(arguments["extra_params"][key]["columns"]) == sorted( + expected_output[key]["columns"] + ) + else: + assert arguments["extra_params"][key] == expected_output[key] diff --git a/openbb_platform/core/tests/app/test_command_runner.py b/openbb_platform/core/tests/app/test_command_runner.py index 446d64ce061c..d56acfca82c4 100644 --- a/openbb_platform/core/tests/app/test_command_runner.py +++ b/openbb_platform/core/tests/app/test_command_runner.py @@ -352,6 +352,7 @@ def __init__(self, results): """Initialize the mock object.""" self.results = results self.extra = {} + self.extra["metadata"] = {"test": "test"} mock_get_command.return_value = other_mock_func mock_execute_func.return_value = MockOBBject(results=[1, 2, 3, 4])