Add versions 2023 q2

amlb/runners/aws.py

@@ -1110,7 +1110,7 @@ def _ec2_startup_script(self, instance_key, script_params="", timeout_secs=-1):
   - pip3 install -U awscli wheel
   - aws s3 cp '{s3_input}' /s3bucket/input --recursive
   - aws s3 cp '{s3_user}' /s3bucket/user --recursive
-  - docker run {docker_options} -v /s3bucket/input:/input -v /s3bucket/output:/output -v /s3bucket/user:/custom --rm {image} {params} -i /input -o /output -u /custom -s skip -Xrun_mode=aws.docker {extra_params}
+  - docker run {docker_options} -v /s3bucket/input:/input -v /s3bucket/output:/output -u "0:0" -v /s3bucket/user:/custom --rm {image} {params} -i /input -o /output -u /custom -s skip -Xrun_mode=aws.docker {extra_params}
   - aws s3 cp /s3bucket/output '{s3_output}' --recursive
   #- rm -f /var/lib/cloud/instance/sem/config_scripts_user
 

amlb/runners/docker.py

@@ -101,12 +101,14 @@ def _image_exists(self, image):
 
     def _run_container_build_command(self, image, cache):
         log.info(f"Building docker image {image}.")
-        run_cmd("docker build {options} -t {container} -f {script} .".format(
+        run_cmd("docker build {options} {build_extra_options} -t {container} -f {script} .".format(
             options="" if cache else "--no-cache",
             container=image,
-            script=self._script),
+            script=self._script,
+            build_extra_options=rconfig().docker.build_extra_options,
+        ),
             _live_output_=rconfig().setup.live_output,
-            _activity_timeout_=rconfig().setup.activity_timeout
+            _activity_timeout_=rconfig().setup.activity_timeout,
         )
         log.info(f"Successfully built docker image {image}.")
 

frameworks/TPOT/__init__.py

@@ -24,7 +24,7 @@ def run(dataset: Dataset, config: TaskConfig):
         ),
     )
     if config.measure_inference_time:
-        data["inference_subsample_files"] = dataset.inference_subsample_files(fmt="parquet")
+        data["inference_subsample_files"] = dataset.inference_subsample_files(fmt="parquet", scikit_safe=True)
 
     def process_results(results):
         if isinstance(results.probabilities, str) and results.probabilities == "predictions":

frameworks/autosklearn/requirements.txt

@@ -1,2 +1,3 @@
 packaging<22.0
 openml
+pandas<2.0  # https://github.com/automl/auto-sklearn/issues/1672

resources/config.yaml

@@ -33,7 +33,7 @@ frameworks:              # configuration namespace for the frameworks definition
     - '{root}/resources/frameworks.yaml'
   root_module: frameworks     # the default python module under which the frameworks modules are defined.
   allow_duplicates: false     # if true, the last definition is used.
-  tags: ['stable', 'latest', '2020Q2', '2021Q3']  # the list of supported tags when looking up frameworks:
+  tags: ['stable', 'latest', '2020Q2', '2021Q3', '2023Q2']  # the list of supported tags when looking up frameworks:
                               # for example frmwk:latest will look for framework frmwk in a frameworks_latest.yaml file if present.
 
 benchmarks:                     # configuration namespace for the benchmarks definitions.
@@ -114,6 +114,7 @@ container: &container          # parent configuration namespace for container mo
 docker:                        # configuration namespace for docker: it inherits from `container` namespace.
   <<: *container
   run_extra_options: '--shm-size=2048M'
+  build_extra_options: ''
 
 singularity:                   # configuration namespace for docker: it inherits from `container` namespace.
   <<: *container
@@ -150,6 +151,9 @@ aws:                    # configuration namespace for AWS mode.
       us-east-1:
         ami: ami-053b0d53c279acc90
         description: Canonical, Ubuntu, 22.04 LTS, amd64 jammy image build on 2023-05-16
+      us-east-2:
+        ami: ami-024e6efaf93d85776
+        description: Canonical, Ubuntu, 22.04 LTS, amd64 jammy image build on 2023-05-16
       us-west-1:
         ami: ami-0f8e81a3da6e2510a
         description: Canonical, Ubuntu, 22.04 LTS, amd64 jammy image build on 2023-05-16

resources/frameworks_2023Q2.yaml

@@ -0,0 +1,119 @@
+---
+
+#########################
+### AutoML frameworks ###
+#########################
+
+AutoGluon:
+  abstract: true
+  version: "0.8.0"
+
+AutoGluon_benchmark:
+  extends: AutoGluon
+  description: "AutoGluon with 'best_quality' preset provides the most accurate overall predictor"
+  params:
+    _save_artifacts: ['leaderboard', 'info']
+    presets: best_quality
+    _leaderboard_test: True
+
+AutoGluon_hq:
+  extends: AutoGluon
+  description: "AutoGluon with 'high_quality' preset provides generally fast inference speed with high accuracy"
+  params:
+    _save_artifacts: ['leaderboard', 'info']
+    presets: high_quality
+    _leaderboard_test: True
+
+AutoGluon_hq_il001:
+  extends: AutoGluon
+  description: "AutoGluon ~3x faster inference at slight performance loss to 'high quality' (self-reported)."
+  params:
+    _save_artifacts: ['leaderboard', 'info']
+    presets: high_quality
+    _leaderboard_test: True
+    infer_limit: 0.01
+
+autosklearn:
+  version: '0.15.0'
+
+autosklearn2:
+  extends: autosklearn
+  params:
+    _askl2: true
+
+AutoWEKA:
+  version: '2.6'
+
+flaml:
+  version: '1.2.4'
+
+GAMA:
+  abstract: true
+  version: '22.0.0'
+
+GAMA_benchmark:
+  extends: GAMA
+  params:
+    goal: performance
+
+H2OAutoML:
+  version: '3.40.0.4'
+
+lightautoml:
+  version: '0.3.7.3'
+
+mljarsupervised:
+  abstract: true
+  version: '0.11.5'
+
+mljarsupervised_benchmark:
+  extends: mljarsupervised
+  description: "MLJar using 'Compete' mode to provide most accurate predictor"
+  params:
+    mode: Compete
+
+mljarsupervised_perform:
+  extends: mljarsupervised
+  description: "MLJar using 'Perform' mode to provide fast inference"
+  params:
+    mode: Perform
+
+MLPlan:
+  abstract: true
+  version: '0.2.5'
+
+MLPlanWEKA:
+  extends: MLPlan
+  params:
+    _backend: weka
+
+mlr3automl:
+  version: '#f667900292e3ded64bb419285c71cd5d1d2c4301'
+  project: https://github.com/a-hanf/mlr3automl
+
+TPOT:
+  version: '0.12.0'
+
+
+#######################################
+### Non AutoML reference frameworks ###
+#######################################
+
+constantpredictor: {}
+
+DecisionTree:
+  version: '1.2.2'
+
+RandomForest:
+  version: '1.2.2'
+  params:
+    n_estimators: 2000
+
+ranger:
+  version: 'stable'
+
+TunedRandomForest:
+  version: '1.2.2'
+  params:
+    n_estimators: 2000
+