support string alg in tune

flaml/tune/tune.py

@@ -344,7 +344,7 @@ def easy_objective(config):
  # do cleanup operation here
  return
  ```
- search_alg: An instance of BlendSearch as the search algorithm
+ search_alg: An instance/string of the search algorithm
  to be used. The same instance can be used for iterative tuning.
  e.g.,
 
@@ -481,12 +481,25 @@ def easy_objective(config):
  else:
  logger.setLevel(logging.CRITICAL)
 
- from .searcher.blendsearch import BlendSearch, CFO
+ from .searcher.blendsearch import BlendSearch, CFO, RandomSearch
 
  if lexico_objectives is not None:
- logger.warning("If lexico_objectives is not None, search_alg is forced to be CFO")
- search_alg = None
- if search_alg is None:
+ if "modes" not in lexico_objectives.keys():
+ lexico_objectives["modes"] = ["min"] * len(lexico_objectives["metrics"])
+ for t_metric, t_mode in zip(lexico_objectives["metrics"], lexico_objectives["modes"]):
+ if t_metric not in lexico_objectives["tolerances"].keys():
+ lexico_objectives["tolerances"][t_metric] = 0
+ if t_metric not in lexico_objectives["targets"].keys():
+ lexico_objectives["targets"][t_metric] = -float("inf") if t_mode == "min" else float("inf")
+ if search_alg is None or isinstance(search_alg, str):
+ if isinstance(search_alg, str):
+ assert search_alg in [
+ "BlendSearch",
+ "CFO",
+ "CFOCat",
+ "RandomSearch",
+ ], f"search_alg={search_alg} is not recognized. 'BlendSearch', 'CFO', 'CFOcat' and 'RandomSearch' are supported."
+
  flaml_scheduler_resource_attr = (
  flaml_scheduler_min_resource
  ) = flaml_scheduler_max_resource = flaml_scheduler_reduction_factor = None
@@ -500,20 +513,30 @@ def easy_objective(config):
  flaml_scheduler_max_resource = max_resource
  flaml_scheduler_reduction_factor = reduction_factor
  scheduler = None
- if lexico_objectives is None:
- try:
- import optuna as _
+ if lexico_objectives:
+ # TODO: Modify after supporting BlendSearch in lexicographic optimization
+ SearchAlgorithm = CFO
+ logger.info(
+ f"Using search algorithm {SearchAlgorithm.__name__} for lexicographic optimization. Note that when providing other search algorithms, we use CFO instead temporarily."
+ )
+ metric = lexico_objectives["metrics"][0] or DEFAULT_METRIC
+ else:
+ if not search_alg or search_alg == "BlendSearch":
+ try:
+ import optuna as _
 
- SearchAlgorithm = BlendSearch
+ SearchAlgorithm = BlendSearch
+ logger.info("Using search algorithm {}.".format(SearchAlgorithm.__name__))
+ except ImportError:
+ if search_alg == "BlendSearch":
+ raise ValueError("To use BlendSearch, run: pip install flaml[blendsearch]")
+ else:
+ SearchAlgorithm = CFO
+ logger.warning("Using CFO for search. To use BlendSearch, run: pip install flaml[blendsearch]")
+ else:
+ SearchAlgorithm = locals()[search_alg]
  logger.info("Using search algorithm {}.".format(SearchAlgorithm.__name__))
- except ImportError:
- SearchAlgorithm = CFO
- logger.warning("Using CFO for search. To use BlendSearch, run: pip install flaml[blendsearch]")
  metric = metric or DEFAULT_METRIC
- else:
- SearchAlgorithm = CFO
- logger.info("Using search algorithm {}.".format(SearchAlgorithm.__name__))
- metric = lexico_objectives["metrics"][0] or DEFAULT_METRIC
  search_alg = SearchAlgorithm(
  metric=metric,
  mode=mode,
@@ -535,8 +558,12 @@ def easy_objective(config):
  )
  else:
  if metric is None or mode is None:
- metric = metric or search_alg.metric or DEFAULT_METRIC
- mode = mode or search_alg.mode
+ if lexico_objectives:
+ metric = lexico_objectives["metrics"][0] or metric or search_alg.metric or DEFAULT_METRIC
+ mode = lexico_objectives["modes"][0] or mode or search_alg.mode
+ else:
+ metric = metric or search_alg.metric or DEFAULT_METRIC
+ mode = mode or search_alg.mode
  if ray_available and use_ray:
  if ray_version.startswith("1."):
  from ray.tune.suggest import ConcurrencyLimiter
@@ -555,6 +582,13 @@ def easy_objective(config):
  ):
  search_alg.use_incumbent_result_in_evaluation = use_incumbent_result_in_evaluation
  searcher = search_alg.searcher if isinstance(search_alg, ConcurrencyLimiter) else search_alg
+ if lexico_objectives:
+ # TODO: Modify after supporting BlendSearch in lexicographic optimization
+ assert search_alg.__class__.__name__ in [
+ "CFO",
+ ], "If lexico_objectives is not None, the search_alg must be CFO for now."
+ search_alg.lexico_objective = lexico_objectives
+
  if isinstance(searcher, BlendSearch):
  setting = {}
  if time_budget_s:

setup.py

@@ -85,7 +85,7 @@
  "tensorboardX==2.6", # test_forecast_panel
  "requests<2.29.0", # https://github.com/docker/docker-py/issues/3113
  "packaging",
- "pydantic",
+ "pydantic==1.10.9",
  "sympy",
  "wolframalpha",
  "joblib<1.3.0", # temp solution for joblib 1.3.0 issue, no need once https://github.com/joblib/joblib-spark/pull/48 is merged
@@ -137,7 +137,7 @@
  "benchmark": ["catboost>=0.26", "psutil==5.8.0", "xgboost==1.3.3", "pandas==1.1.4"],
  "openai": ["openai==0.27.8", "diskcache"],
  "autogen": ["openai==0.27.8", "diskcache", "docker"],
- "mathchat": ["openai==0.27.8", "diskcache", "docker", "sympy", "pydantic", "wolframalpha"],
+ "mathchat": ["openai==0.27.8", "diskcache", "docker", "sympy", "pydantic==1.10.9", "wolframalpha"],
  "synapse": [
  "joblibspark>=0.5.0",
  "optuna==2.8.0",

test/tune/test_tune.py

@@ -1,13 +1,14 @@
 """Require: pip install flaml[test,ray]
 """
-from flaml import BlendSearch
+from flaml import BlendSearch, CFO
 import time
 import os
 from sklearn.model_selection import train_test_split
 import sklearn.metrics
 import sklearn.datasets
 import xgboost as xgb
 import logging
+import math
 
 try:
  from ray.tune.integration.xgboost import TuneReportCheckpointCallback
@@ -20,6 +21,32 @@
 logger.setLevel(logging.INFO)
 
 
+def _BraninCurrin(config):
+ # Rescale brain
+ x_1 = 15 * config["x1"] - 5
+ x_2 = 15 * config["x2"]
+ # Brain function
+ t1 = x_2 - 5.1 / (4 * math.pi**2) * x_1**2 + 5 / math.pi * x_1 - 6
+ t2 = 10 * (1 - 1 / (8 * math.pi)) * math.cos(x_1)
+ brain_result = t1**2 + t2 + 10
+ # Currin function
+ xc_1 = config["x1"]
+ xc_2 = config["x2"]
+ factor1 = 1 - math.exp(-1 / (2 * xc_2))
+ numer = 2300 * pow(xc_1, 3) + 1900 * pow(xc_1, 2) + 2092 * xc_1 + 60
+ denom = 100 * pow(xc_1, 3) + 500 * pow(xc_1, 2) + 4 * xc_1 + 20
+ currin_result = factor1 * numer / denom
+ return {"brain": brain_result, "currin": currin_result}
+
+
+def _easy_objective(config):
+ # Hyperparameters
+ width, height, step = config["width"], config["height"], config["steps"]
+
+ # get_result
+ return {"mean_loss": (0.1 + width * step / 100) ** (-1) + height * 0.1}
+
+
 def test_nested_run():
  from flaml import AutoML, tune
 
@@ -352,6 +379,72 @@ def evaluate_config_empty(config):
  )
 
 
+def test_passing_search_alg():
+ from flaml import tune
+
+ # search_space
+ so_search_space = {
+ "steps": 100,
+ "width": tune.uniform(0, 20),
+ "height": tune.uniform(-100, 100),
+ }
+ mo_search_space = {
+ "x1": tune.uniform(lower=0.000001, upper=1.0),
+ "x2": tune.uniform(lower=0.000001, upper=1.0),
+ }
+
+ # lexicographic objectives
+ lexico_objectives = {}
+ lexico_objectives["metrics"] = ["brain", "currin"]
+ lexico_objectives["tolerances"] = {"brain": 10.0, "currin": 0.0}
+ lexico_objectives["targets"] = {"brain": 0.0, "currin": 0.0}
+ lexico_objectives["modes"] = ["min", "min"]
+
+ ## Passing search_alg through string
+ # Non lexico tune
+ tune.run(
+ _easy_objective,
+ search_alg="BlendSearch",
+ metric="mean_loss",
+ mode="min",
+ num_samples=10,
+ config=so_search_space,
+ )
+ # lexico tune
+ tune.run(
+ _BraninCurrin, search_alg="CFO", num_samples=10, config=mo_search_space, lexico_objectives=lexico_objectives
+ )
+ tune.run(
+ _BraninCurrin,
+ search_alg="BlendSearch",
+ num_samples=10,
+ config=mo_search_space,
+ lexico_objectives=lexico_objectives,
+ )
+
+ ## Passing search_alg through instance
+ so_bs = BlendSearch(time_budget_s=5, metric="mean_loss", mode="min")
+ # TODO: We will change CFO into blendsearch in the future
+ mo_bs = CFO(time_budget_s=5)
+ # Non lexico tune
+ tune.run(
+ _easy_objective,
+ search_alg=so_bs,
+ metric="mean_loss",
+ mode="min",
+ num_samples=10,
+ config=so_search_space,
+ )
+ # lexico tune
+ tune.run(
+ _BraninCurrin,
+ search_alg=mo_bs,
+ num_samples=10,
+ config=mo_search_space,
+ lexico_objectives=lexico_objectives,
+ )
+
+
 def test_xgboost_bs():
  _test_xgboost()