Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Skill class #47

Merged
merged 44 commits into from
May 20, 2021
Merged
Show file tree
Hide file tree
Changes from 29 commits
Commits
Show all changes
44 commits
Select commit Hold shift + click to select a range
e0eaa90
Create skill.py
jsmariegaard May 7, 2021
b183b6b
use the new skill class
jsmariegaard May 7, 2021
9f84d39
adapt tests to changes
jsmariegaard May 7, 2021
d59fa18
references
jsmariegaard May 7, 2021
cf358ac
Updated notebooks
jsmariegaard May 7, 2021
ef725f3
Merge branch 'main' into skill-class
jsmariegaard May 10, 2021
a4883d9
style, sort_values, to_html
jsmariegaard May 10, 2021
0e888f8
Merge branch 'main' into skill-class
jsmariegaard May 11, 2021
485df6d
Update test_multimodelcompare.py
jsmariegaard May 11, 2021
450f09b
Merge branch 'main' into skill-class
jsmariegaard May 11, 2021
b1ccd3b
minor
jsmariegaard May 11, 2021
f3ebfef
get only actual metrics in valid_metrics
jsmariegaard May 11, 2021
86e97d8
mark best
jsmariegaard May 12, 2021
1b0f1cd
Merge branch 'main' into skill-class
jsmariegaard May 12, 2021
29911e2
rename
jsmariegaard May 12, 2021
e99e967
SkillDataFrame class, obs_name, mod_names and more
jsmariegaard May 14, 2021
046082a
Update test_multimodelcompare.py
jsmariegaard May 14, 2021
a74f18b
plot_grid, plot_line, plot_bar
jsmariegaard May 14, 2021
ae34aac
reversed cmap and more pandas functions wrapped
jsmariegaard May 14, 2021
4a8d58a
sel, _reverse_colormap
jsmariegaard May 14, 2021
c13a207
docstrings and minor
jsmariegaard May 16, 2021
1d341c7
field_names and docstrings
jsmariegaard May 16, 2021
5faf7ba
add skill to documentation
jsmariegaard May 16, 2021
f148df9
First tests added
jsmariegaard May 16, 2021
fea4faa
clean up
jsmariegaard May 16, 2021
6c43c38
updated notebooks
jsmariegaard May 16, 2021
0377858
Update Multi_variable_comparison.ipynb
jsmariegaard May 16, 2021
baf87d3
make sure index has a name
jsmariegaard May 17, 2021
c132c71
datetime axis, fmt, to_excel
jsmariegaard May 17, 2021
506c48e
select multiple model/observation etc; select columns, select by id
jsmariegaard May 17, 2021
cbd481f
reduce multiindex, sel() docstring,
jsmariegaard May 18, 2021
f0be9d6
metrics as list or not
jsmariegaard May 19, 2021
a8f108c
docstrings and KeyError
jsmariegaard May 19, 2021
d71ef57
test sel()
jsmariegaard May 19, 2021
d3d3967
fix non-list check
jsmariegaard May 19, 2021
e87ae60
ndims to ndim
jsmariegaard May 19, 2021
8a329f2
more tests
jsmariegaard May 19, 2021
311ef88
more tests added
jsmariegaard May 19, 2021
5e16ced
reduce only index for multiindex, and keep index when selecting from …
jsmariegaard May 19, 2021
d0c20be
more tests
jsmariegaard May 19, 2021
eef9e4d
lin_slope background gradient, docstrings and clean up
jsmariegaard May 19, 2021
3cd539b
lin_slope
jsmariegaard May 19, 2021
f2898c8
show skill examples
jsmariegaard May 20, 2021
ae629d5
updated notebooks
jsmariegaard May 20, 2021
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions docs/api.rst
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,12 @@ Compare
:inherited-members:
:exclude-members: keys, values, get, items

Skill
-------------
.. autoclass:: fmskill.skill.AggregatedSkill
:members:
:inherited-members:

Spatial Skill
-------------
.. autoclass:: fmskill.spatial.SpatialSkill
Expand Down
36 changes: 21 additions & 15 deletions fmskill/compare.py
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
import fmskill.metrics as mtr
from fmskill.observation import PointObservation, TrackObservation
from fmskill.plot import scatter
from fmskill.skill import AggregatedSkill
from fmskill.spatial import SpatialSkill


Expand Down Expand Up @@ -290,7 +291,9 @@ def _parse_metric(self, metric):
return [mtr.bias, mtr.rmse, mtr.urmse, mtr.mae, mtr.cc, mtr.si, mtr.r2]

if isinstance(metric, str):
valid_metrics = [x[0] for x in getmembers(mtr, isfunction)]
valid_metrics = [
x[0] for x in getmembers(mtr, isfunction) if x[0][0] != "_"
]

if metric.lower() in valid_metrics:
metric = getattr(mtr, metric.lower())
Expand Down Expand Up @@ -318,7 +321,7 @@ def skill(
end: Union[str, datetime] = None,
area: List[float] = None,
df: pd.DataFrame = None,
) -> pd.DataFrame:
) -> AggregatedSkill:
"""Aggregated skill assessment of model(s)

Parameters
Expand Down Expand Up @@ -408,7 +411,7 @@ def skill(

res = self._groupby_df(df.drop(columns=["x", "y"]), by, metrics)
res = self._add_as_field_if_not_in_index(df, skilldf=res)
return res
return AggregatedSkill(res)

def _add_as_field_if_not_in_index(
self, df, skilldf, fields=["model", "observation", "variable"]
Expand Down Expand Up @@ -928,7 +931,7 @@ def skill(
end: Union[str, datetime] = None,
area: List[float] = None,
df: pd.DataFrame = None,
) -> pd.DataFrame:
) -> AggregatedSkill:
"""Skill assessment of model(s)

Parameters
Expand Down Expand Up @@ -959,8 +962,8 @@ def skill(

Returns
-------
pd.DataFrame
skill assessment as a dataframe
AggregatedSkill
skill assessment object

See also
--------
Expand Down Expand Up @@ -1008,7 +1011,7 @@ def score(
end: Union[str, datetime] = None,
area: List[float] = None,
df: pd.DataFrame = None,
) -> pd.DataFrame:
) -> float:
"""Model skill score

Parameters
Expand Down Expand Up @@ -1057,7 +1060,7 @@ def score(
end=end,
area=area,
df=df,
)
).df
values = df[metric.__name__].values
if len(values) == 1:
values = values[0]
Expand Down Expand Up @@ -1195,6 +1198,7 @@ def __init__(self, observation, modeldata):
else:
self.df[self.mod_names[j]] = df[self.mod_names[j]]

self.df.index.name = "datetime"
self.df.dropna(inplace=True)

def plot_timeseries(
Expand Down Expand Up @@ -1306,6 +1310,7 @@ def __init__(self, observation, modeldata):
else:
self.df[self.mod_names[j]] = df[self.mod_names[j]]

self.df.index.name = "datetime"
self.df = self.df.dropna()

def _obs_mod_xy_distance_acceptable(self, df_mod, df_obs):
Expand Down Expand Up @@ -1411,6 +1416,7 @@ def _construct_all_df(self):
res = res.append(df[cols])

self._all_df = res.sort_index()
self._all_df.index.name = "datetime"

def __init__(self):
self.comparers = {}
Expand Down Expand Up @@ -1495,7 +1501,7 @@ def mean_skill(
end: Union[str, datetime] = None,
area: List[float] = None,
df: pd.DataFrame = None,
) -> pd.DataFrame:
) -> AggregatedSkill:
"""Weighted mean skill of model(s) over all observations (of same variable)

Parameters
Expand Down Expand Up @@ -1527,8 +1533,8 @@ def mean_skill(

Returns
-------
pd.DataFrame
mean skill assessment as a dataframe
AggregatedSkill
mean skill assessment as a skill object

See also
--------
Expand Down Expand Up @@ -1563,7 +1569,7 @@ def mean_skill(

# skill assessment
metrics = self._parse_metric(metrics)
skilldf = self.skill(df=df, metrics=metrics)
skilldf = self.skill(df=df, metrics=metrics).df

# weights
weights = self._parse_weights(weights, obs_names)
Expand All @@ -1581,7 +1587,7 @@ def mean_skill(

# output
res = self._add_as_field_if_not_in_index(df, res, fields=["model", "variable"])
return res.astype({"n": int})
return AggregatedSkill(res.astype({"n": int}))

def _mean_skill_by(self, skilldf, mod_names, var_names):
by = []
Expand Down Expand Up @@ -1647,7 +1653,7 @@ def score(
end: Union[str, datetime] = None,
area: List[float] = None,
df: pd.DataFrame = None,
) -> pd.DataFrame:
) -> float:
"""Weighted mean score of model(s) over all observations
NOTE: will take simple mean over different variables

Expand Down Expand Up @@ -1720,7 +1726,7 @@ def score(
end=end,
area=area,
df=df,
)
).df

if n_models == 1:
score = df[metric.__name__].values.mean()
Expand Down
9 changes: 4 additions & 5 deletions fmskill/metrics.py
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@
from typing import Tuple
import warnings
import numpy as np
from scipy.stats import linregress
from scipy.stats import linregress as _linregress
import scipy.stats
from scipy import odr

Expand Down Expand Up @@ -203,7 +203,7 @@ def model_efficiency_factor(obs: np.ndarray, model: np.ndarray) -> float:
See Also
--------
nash_sutcliffe_efficiency
root_mean_square_error
root_mean_squared_error

"""
assert obs.size == model.size
Expand Down Expand Up @@ -240,14 +240,13 @@ def corrcoef(obs, model, weights=None) -> float:


def rho(obs: np.ndarray, model: np.ndarray) -> float:
"""alias for Spearman rank correlation coefficient"""
"""alias for spearmanr"""
return spearmanr(obs, model)


def spearmanr(obs: np.ndarray, model: np.ndarray) -> float:
"""Spearman rank correlation coefficient


The rank correlation coefficient is similar to the Pearson correlation coefficient but
applied to ranked quantities and is useful to quantify a monotonous relationship

Expand Down Expand Up @@ -341,7 +340,7 @@ def _linear_regression(
return np.nan

if reg_method == "ols":
reg = linregress(obs, model)
reg = _linregress(obs, model)
intercept = reg.intercept
slope = reg.slope
elif reg_method == "odr":
Expand Down
Loading