Add support for the OpenML test server

amlb/benchmarks/openml.py

@@ -16,7 +16,7 @@ def is_openml_benchmark(benchmark: str) -> bool:
         supported_types = ['s', 't']
 
         if oml_id.isdecimal():
-            return domain == "openml" and oml_type in supported_types
+            return domain in ("openml", "test.openml") and oml_type in supported_types
     return False
 
 
@@ -26,6 +26,10 @@ def load_oml_benchmark(benchmark: str) -> Tuple[str, Optional[str], List[Namespa
     path = None  # benchmark file does not exist on disk
     name = benchmark  # name is later passed as cli input again for containers, it needs to remain parsable
 
+    if domain == "test.openml":
+        log.debug("Setting openml server to the test server.")
+        openml.config.server = "https://test.openml.org/api/v1/xml"
+
     if openml.config.retry_policy != "robot":
         log.debug(
             "Setting openml retry_policy from '%s' to 'robot'." % openml.config.retry_policy)
@@ -38,7 +42,8 @@ def load_oml_benchmark(benchmark: str) -> Tuple[str, Optional[str], List[Namespa
         data = openml.datasets.get_dataset(t.dataset_id, download_data=False, download_qualities=False)
         tasks = [Namespace(name=str_sanitize(data.name),
                            description=data.description,
-                           openml_task_id=t.id)]
+                           openml_task_id=t.id,
+                           id="{}/t/{}".format(domain, t.id))]
     elif oml_type == 's':
         log.info("Loading openml suite %s.", oml_id)
         suite = openml.study.get_suite(oml_id)
@@ -50,7 +55,8 @@ def load_oml_benchmark(benchmark: str) -> Tuple[str, Optional[str], List[Namespa
         for tid, did in zip(suite.tasks, suite.data):
             tasks.append(Namespace(name=str_sanitize(datasets.loc[did]['name']),
                                    description=f"{openml.config.server.replace('/api/v1/xml', '')}/d/{did}",
-                                   openml_task_id=tid))
+                                   openml_task_id=tid,
+                                   id="{}/t/{}".format(domain, tid)))
     else:
         raise ValueError(f"The oml_type is {oml_type} but must be 's' or 't'")
     return name, path, tasks

amlb/resources.py

@@ -210,14 +210,14 @@ def _validate_task(self, task, lenient=False):
 
         conf = 'id'
         if task[conf] is None:
-            task[conf] = ("openml.org/t/{}".format(task.openml_task_id) if task['openml_task_id'] is not None
-                          else "openml.org/d/{}".format(task.openml_dataset_id) if task['openml_dataset_id'] is not None
-                          else ((task.dataset['id'] if isinstance(task.dataset, (dict, Namespace))
-                                 else task.dataset if isinstance(task.dataset, str)
-                                 else None) or task.name) if task['dataset'] is not None
+            task[conf] = (((task.dataset['id'] if isinstance(task.dataset, (dict, Namespace))
+                            else task.dataset if isinstance(task.dataset, str)
+                            else None) or task.name) if task['dataset'] is not None
                           else None)
             if not lenient and task[conf] is None:
-                raise ValueError("task definition must contain one property among ['openml_task_id', 'dataset']")
+                raise ValueError("task definition must contain an ID or the property "
+                                 "'dataset' to create an ID, but task "
+                                 "definition is {task}".format(task=str(task)))
 
         conf = 'metric'
         if task[conf] is None:

docs/README.md

@@ -127,7 +127,8 @@ positional arguments:
                         benchmark description file, or an openml suite or
                         task. OpenML references should be formatted as
                         'openml/s/X' and 'openml/t/Y', for studies and tasks
-                        respectively. Defaults to `test`.
+                        respectively. Use 'test.openml/s/X' for the OpenML test
+                        server. Defaults to `test`.
   constraint            The constraint definition to use as defined by default
                         in resources/constraints.yaml. Defaults to `test`.
 

runbenchmark.py

@@ -31,7 +31,8 @@
                     help="The benchmark type to run as defined by default in resources/benchmarks/{benchmark}.yaml,"
                          "\na path to a benchmark description file, or an openml suite or task."
                          "\nOpenML references should be formatted as 'openml/s/X' and 'openml/t/Y',"
-                         "\nfor studies and tasks respectively."
+                         "\nfor studies and tasks respectively. Use 'test.openml/s/X' for the "
+                         "\nOpenML test server."
                          "\n(default: '%(default)s')")
 parser.add_argument('constraint', type=str, nargs='?', default='test',
                     help="The constraint definition to use as defined by default in resources/constraints.yaml."