Skip to content

Commit

Permalink
[BugFix] Fix metadata creation (OpenBB-finance#6306)
Browse files Browse the repository at this point in the history 
* fix metadata creation

* fix metadata creation

* adapt tests

* adapt tests

* adapt tests

---------

Co-authored-by: hjoaquim <h.joaquim@campus.fct.unl.pt>
  • Loading branch information
@hjoaquim
hjoaquim and hjoaquim authored Apr 11, 2024
1 parent 00cc450 commit 3172b9e
Show file tree
Hide file tree
Showing 3 changed files with 121 additions and 91 deletions.
177 changes: 91 additions & 86 deletions openbb_platform/core/openbb_core/app/model/metadata.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,90 +41,95 @@ def scale_arguments(cls, v):
containing the type and the columns. If the type is not one of the previous, the
value is kept or trimmed to 80 characters.
"""
for arg, arg_val in v.items():
new_arg_val: Optional[Union[str, dict[str, Sequence[Any]]]] = None

# Data
if isclass(type(arg_val)) and issubclass(type(arg_val), Data):
new_arg_val = {
"type": f"{type(arg_val).__name__}",
"columns": list(arg_val.model_dump().keys()),
}

# List[Data]
if isinstance(arg_val, list) and issubclass(type(arg_val[0]), Data):
_columns = [list(d.model_dump().keys()) for d in arg_val]
ld_columns = (
item for sublist in _columns for item in sublist
) # flatten
new_arg_val = {
"type": f"List[{type(arg_val[0]).__name__}]",
"columns": list(set(ld_columns)),
}

# DataFrame
elif isinstance(arg_val, pd.DataFrame):
df_columns = (
list(arg_val.index.names) + arg_val.columns.tolist()
if any(index is not None for index in list(arg_val.index.names))
else arg_val.columns.tolist()
)
new_arg_val = {
"type": f"{type(arg_val).__name__}",
"columns": df_columns,
}

# List[DataFrame]
elif isinstance(arg_val, list) and issubclass(
type(arg_val[0]), pd.DataFrame
):
ldf_columns = [
(
list(df.index.names) + df.columns.tolist()
if any(index is not None for index in list(df.index.names))
else df.columns.tolist()
arguments: Dict[str, Any] = {}
for item in ["provider_choices", "standard_params", "extra_params"]:
arguments[item] = {}
for arg, arg_val in v[item].items():
new_arg_val: Optional[Union[str, dict[str, Sequence[Any]]]] = None

# Data
if isclass(type(arg_val)) and issubclass(type(arg_val), Data):
new_arg_val = {
"type": f"{type(arg_val).__name__}",
"columns": list(arg_val.model_dump().keys()),
}

# List[Data]
if isinstance(arg_val, list) and issubclass(type(arg_val[0]), Data):
_columns = [list(d.model_dump().keys()) for d in arg_val]
ld_columns = (
item for sublist in _columns for item in sublist
) # flatten
new_arg_val = {
"type": f"List[{type(arg_val[0]).__name__}]",
"columns": list(set(ld_columns)),
}

# DataFrame
elif isinstance(arg_val, pd.DataFrame):
df_columns = (
list(arg_val.index.names) + arg_val.columns.tolist()
if any(index is not None for index in list(arg_val.index.names))
else arg_val.columns.tolist()
)
for df in arg_val
]
new_arg_val = {
"type": f"List[{type(arg_val[0]).__name__}]",
"columns": ldf_columns,
}

# Series
elif isinstance(arg_val, pd.Series):
new_arg_val = {
"type": f"{type(arg_val).__name__}",
"columns": list(arg_val.index.names) + [arg_val.name],
}

# List[Series]
elif isinstance(arg_val, list) and isinstance(arg_val[0], pd.Series):
ls_columns = [
(
list(series.index.names) + [series.name]
if any(index is not None for index in list(series.index.names))
else series.name
)
for series in arg_val
]
new_arg_val = {
"type": f"List[{type(arg_val[0]).__name__}]",
"columns": ls_columns,
}

# ndarray
elif isinstance(arg_val, np.ndarray):
new_arg_val = {
"type": f"{type(arg_val).__name__}",
"columns": list(arg_val.dtype.names or []),
}

else:
str_repr_arg_val = str(arg_val)
if len(str_repr_arg_val) > 80:
new_arg_val = str_repr_arg_val[:80]

v[arg] = new_arg_val or arg_val

return v
new_arg_val = {
"type": f"{type(arg_val).__name__}",
"columns": df_columns,
}

# List[DataFrame]
elif isinstance(arg_val, list) and issubclass(
type(arg_val[0]), pd.DataFrame
):
ldf_columns = [
(
list(df.index.names) + df.columns.tolist()
if any(index is not None for index in list(df.index.names))
else df.columns.tolist()
)
for df in arg_val
]
new_arg_val = {
"type": f"List[{type(arg_val[0]).__name__}]",
"columns": ldf_columns,
}

# Series
elif isinstance(arg_val, pd.Series):
new_arg_val = {
"type": f"{type(arg_val).__name__}",
"columns": list(arg_val.index.names) + [arg_val.name],
}

# List[Series]
elif isinstance(arg_val, list) and isinstance(arg_val[0], pd.Series):
ls_columns = [
(
list(series.index.names) + [series.name]
if any(
index is not None for index in list(series.index.names)
)
else series.name
)
for series in arg_val
]
new_arg_val = {
"type": f"List[{type(arg_val[0]).__name__}]",
"columns": ls_columns,
}

# ndarray
elif isinstance(arg_val, np.ndarray):
new_arg_val = {
"type": f"{type(arg_val).__name__}",
"columns": list(arg_val.dtype.names or []),
}

else:
str_repr_arg_val = str(arg_val)
if len(str_repr_arg_val) > 80:
new_arg_val = str_repr_arg_val[:80]

arguments[item][arg] = new_arg_val or arg_val

return arguments
34 changes: 29 additions & 5 deletions openbb_platform/core/tests/app/model/test_metadata.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,11 @@
def test_Metadata():
"""Run Smoke test."""
m = Metadata(
arguments={"test": "test"},
arguments={
"provider_choices": {},
"standard_params": {},
"extra_params": {},
},
route="test",
timestamp=datetime.now(),
duration=0,
Expand Down Expand Up @@ -44,7 +48,7 @@ def test_fields():
{
"data_list": {
"type": "List[Data]",
"columns": ["open", "close", "volume"],
"columns": ["open", "volume", "close"],
}
},
),
Expand Down Expand Up @@ -101,8 +105,13 @@ def test_fields():
)
def test_scale_arguments(input_data, expected_output):
"""Test the scale_arguments method."""
kwargs = {
"provider_choices": {},
"standard_params": {},
"extra_params": input_data,
}
m = Metadata(
arguments=input_data,
arguments=kwargs,
route="test",
timestamp=datetime.now(),
duration=0,
Expand All @@ -112,9 +121,24 @@ def test_scale_arguments(input_data, expected_output):
for arg in arguments: # pylint: disable=E1133
if "columns" in arguments[arg]:
# compare the column names disregarding the order with the expected output
assert sorted(arguments[arg]["columns"]) == sorted(
assert sorted(arguments["extra_params"][arg]["columns"]) == sorted(
expected_output[arg]["columns"]
)
assert arguments[arg]["type"] == expected_output[arg]["type"]
else:
assert m.arguments == expected_output
# assert m.arguments["extra_params"] == expected_output
keys = list(arguments["extra_params"].keys())
expected_keys = list(expected_output.keys())
assert sorted(keys) == sorted(expected_keys)

for key in keys:
if "type" in arguments["extra_params"][key]:
assert (
arguments["extra_params"][key]["type"]
== expected_output[key]["type"]
)
assert sorted(arguments["extra_params"][key]["columns"]) == sorted(
expected_output[key]["columns"]
)
else:
assert arguments["extra_params"][key] == expected_output[key]
1 change: 1 addition & 0 deletions openbb_platform/core/tests/app/test_command_runner.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -352,6 +352,7 @@ def __init__(self, results):
"""Initialize the mock object."""
self.results = results
self.extra = {}
self.extra["metadata"] = {"test": "test"}

mock_get_command.return_value = other_mock_func
mock_execute_func.return_value = MockOBBject(results=[1, 2, 3, 4])
Expand Down

0 comments on commit 3172b9e

Please sign in to comment.