Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Tune] Fix best trial in ProgressReporter with nan #31276

Merged
merged 2 commits into from
Jan 4, 2023
Merged

[Tune] Fix best trial in ProgressReporter with nan #31276

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
77158b4
[Tune] Fix best trial in ProgressReporter with nan
Yard1 Dec 21, 2022
4b45031
Merge branch 'ray-project:master' into progress_reporter_nan
Yard1 Jan 3, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions python/ray/tune/progress_reporter.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,9 +12,10 @@
from typing import Any, Callable, Dict, List, Optional, Tuple, Union

import numpy as np
from ray._private.dict import flatten_dict
import pandas as pd

import ray
from ray._private.dict import flatten_dict
from ray.tune.callback import Callback
from ray.tune.logger import pretty_print
from ray.tune.result import (
Expand Down Expand Up @@ -432,7 +433,7 @@ def _current_best_trial(self, trials: List[Trial]):
if not t.last_result:
continue
metric_value = unflattened_lookup(metric, t.last_result, default=None)
if metric_value is None:
if pd.isnull(metric_value):
continue
if not best_trial or metric_value * metric_op > best_metric:
best_metric = metric_value * metric_op
Expand Down
28 changes: 28 additions & 0 deletions python/ray/tune/tests/test_progress_reporter.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
from unittest.mock import MagicMock, Mock, patch

import pytest
import numpy as np

from ray import tune
from ray._private.test_utils import run_string_as_driver
Expand Down Expand Up @@ -513,6 +514,33 @@ def testBestTrialZero(self):
best_trial, metric = reporter._current_best_trial([trial1, trial2, trial3])
assert best_trial == trial2

def testBestTrialNan(self):
trial1 = Trial("", config={}, stub=True)
trial1.last_result = {"metric": np.nan, "config": {}}

trial2 = Trial("", config={}, stub=True)
trial2.last_result = {"metric": 0, "config": {}}

trial3 = Trial("", config={}, stub=True)
trial3.last_result = {"metric": 2, "config": {}}

reporter = TuneReporterBase(metric="metric", mode="min")
best_trial, metric = reporter._current_best_trial([trial1, trial2, trial3])
assert best_trial == trial2

trial1 = Trial("", config={}, stub=True)
trial1.last_result = {"metric": np.nan, "config": {}}

trial2 = Trial("", config={}, stub=True)
trial2.last_result = {"metric": 0, "config": {}}

trial3 = Trial("", config={}, stub=True)
trial3.last_result = {"metric": 2, "config": {}}

reporter = TuneReporterBase(metric="metric", mode="max")
best_trial, metric = reporter._current_best_trial([trial1, trial2, trial3])
assert best_trial == trial3

def testTimeElapsed(self):
# Sun Feb 7 14:18:40 2016 -0800
# (time of the first Ray commit)
Expand Down