Merge for V2.1

.github/workflows/run_all_frameworks.yml

@@ -15,7 +15,7 @@ jobs:
       skip_baseline: ${{ steps.find-required-tests.outputs.skip_baseline }}
       skip_evaluation: ${{ steps.find-required-tests.outputs.skip_evaluation }}
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - name: pull base branch
         run: |
           git fetch --unshallow origin $GITHUB_BASE_REF
@@ -52,7 +52,7 @@ jobs:
           skip_evaluation=0
           if [[ $is_common -eq 0 ]];
           then
-            FRAMEWORKS='["autogluon", "autosklearn", "autoxgboost", "gama", "h2oautoml", "mlplanweka", "mlr3automl", "tpot", "tunedrandomforest"]'
+            FRAMEWORKS='["autogluon", "autosklearn", "gama", "h2oautoml", "mlplanweka", "mlr3automl", "tpot", "tunedrandomforest"]'
             TASKS='["iris", "kc2", "cholesterol"]'
             BENCHMARK='["test"]'
           else
@@ -90,17 +90,17 @@ jobs:
         task: [iris, kc2, cholesterol]
       fail-fast:  false
     steps:
-      - uses: actions/checkout@v2
-      - name: Setup Python 3.8
-        uses: actions/setup-python@v2
+      - uses: actions/checkout@v3
+      - name: Setup Python 3.9
+        uses: actions/setup-python@v4
         with:
-          python-version: '3.8'
+          python-version: '3.9'
       - name: Create venv
         run: python -m venv venv
-      - uses: actions/cache@v2
+      - uses: actions/cache@v3
         id: cache
         with:
-          path: /home/runner/work/automlbenchmark/automlbenchmark/venv/lib/python3.8/site-packages
+          path: /home/runner/work/automlbenchmark/automlbenchmark/venv/lib/python3.9/site-packages
           key: pip-v3-${{ hashFiles('**/requirements.txt') }}
           restore-keys: |
             pip-v3-
@@ -126,24 +126,24 @@ jobs:
 
     strategy:
       matrix:
-        python-version: [3.8]
+        python-version: [3.9]
         framework: ${{ fromJson(needs.detect_changes.outputs.frameworks) }}
         task: ${{ fromJson(needs.detect_changes.outputs.tasks) }}
         benchmark: ${{ fromJson(needs.detect_changes.outputs.benchmark) }}
       fail-fast:  true  # not sure about this one, but considering the big workload it might be nicer
 
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
     - name: Setup Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v4
       with:
         python-version: ${{ matrix.python-version }}
     - name: Create venv
       run: python -m venv venv
-    - uses: actions/cache@v2
+    - uses: actions/cache@v3
       id: cache
       with:
-        path: /home/runner/work/automlbenchmark/automlbenchmark/venv/lib/python3.8/site-packages
+        path: /home/runner/work/automlbenchmark/automlbenchmark/venv/lib/python3.9/site-packages
         key: pip-v3-${{ hashFiles('**/requirements.txt') }}
         restore-keys: |
           pip-v3-
@@ -152,8 +152,10 @@ jobs:
       run: |
         source venv/bin/activate
         python -m pip install --upgrade pip
-        pip install -r requirements.txt
+        python -m pip install -r requirements.txt
     - name: Run ${{ matrix.framework }} on ${{ matrix.task }}
       run: |
         source venv/bin/activate
         python runbenchmark.py ${{ matrix.framework }} ${{ matrix.benchmark }} test -f 0 -t ${{ matrix.task }} -e
+      env:
+        GITHUB_PAT: ${{ secrets.PUBLIC_ACCESS_GITHUB_PAT }}

.github/workflows/run_pytest.yaml

@@ -0,0 +1,35 @@
+name: Light-weight Unit Tests
+
+on: pull_request
+
+
+jobs:
+  pytest_not_stress:
+    name: pytest
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Setup Python 3.9
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.9'
+      - name: Create venv
+        run: python -m venv venv
+      - uses: actions/cache@v3
+        id: cache
+        with:
+          path: /home/runner/work/automlbenchmark/automlbenchmark/venv/lib/python3.9/site-packages
+          key: pip-v3-${{ hashFiles('**/requirements.txt') }}-${{ hashFiles('**/requirements-dev.txt') }}
+          restore-keys: |
+            pip-v3-
+      - name: Install Requirements
+        if: steps.cache.outputs.cache-hit != 'true'
+        run: |
+          source venv/bin/activate
+          python -m pip install --upgrade pip
+          pip install -r requirements.txt
+          pip install -r requirements-dev.txt
+      - name: Run unit tests
+        run: |
+          source venv/bin/activate
+          python -m pytest -vv -m "not stress and not openml_upload"

.github/workflows/runbenchmark.yml

@@ -0,0 +1,38 @@
+name: Run `runbenchmark.py`
+
+on:
+  workflow_dispatch:
+    inputs:
+      python_version:
+        default: '3.9'
+      os:
+        default: 'ubuntu-latest'
+      framework:
+        default: 'randomforest'
+      options:
+        default: ''
+
+jobs:
+  run_benchmark:
+    runs-on: ${{ inputs.os }}
+    steps:
+      - name: Runs on ${{ inputs.os }}
+        run: echo Preparing to benchmark ${{ inputs.framework }} ${{ inputs.options }}
+      - name: Checkout ${{ github.ref_name }}
+        uses: actions/checkout@v3
+        with:
+          ref: ${{ github.ref_name }}
+      - name: Install Py${{ inputs.python_version }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ inputs.python_version }}
+      - name: Install benchmark dependencies
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install -r requirements.txt
+      - name: Install ${{ inputs.framework }}
+        run: python runbenchmark.py ${{ inputs.framework }} -s only
+        env:
+          GITHUB_PAT: ${{ secrets.PUBLIC_ACCESS_GITHUB_PAT }}
+      - name: Benchmark ${{ inputs.framework }}
+        run: python runbenchmark.py ${{ inputs.framework }} ${{ inputs.options }}

.gitignore

@@ -16,6 +16,7 @@ venv/
 .idea/
 *.iml
 *.swp
+launch.json
 
 # tmp files
 .ipynb_checkpoints/
@@ -47,3 +48,4 @@ Singularityfile
 #whitelist
 !reports/**/*.csv
 !reports/**/*.png
+!docs/img/**/*.png

amlb/__version__.py

@@ -1,3 +1,3 @@
 # major.minor.patch[-label]
 _dev_version = "dev"
-__version__ = "2.0.6"
+__version__ = "dev"