Merge branch 'main' into launcher-test-args-fixups

.bumpversion.cfg

@@ -1,10 +1,12 @@
 [bumpversion]
-current_version = 0.5.1
+current_version = 0.6.0
 commit = True
 tag = True
 
-[bumpversion:file:doc/source/conf.py]
+[bumpversion:file:doc/source/version.py]
 
-[bumpversion:file:mlos_core/_version.py]
+[bumpversion:file:mlos_core/mlos_core/version.py]
 
-[bumpversion:file:mlos_bench/_version.py]
+[bumpversion:file:mlos_bench/mlos_bench/version.py]
+
+[bumpversion:file:mlos_viz/mlos_viz/version.py]

.devcontainer/Dockerfile

@@ -47,16 +47,18 @@ RUN umask 0002 \
  && /opt/conda/bin/conda install -v -y -n base conda-libmamba-solver \
  && /opt/conda/bin/conda config --set solver libmamba \
  && /opt/conda/bin/conda list -n base \
- && /opt/conda/bin/conda clean -v -y -a
-
-# Update the base. This helps save space by making sure the same version
-# python is used for both the base env and mlos env.
-RUN umask 0002 \
- && /opt/conda/bin/conda update -v -y -n base -c defaults --all \
- && /opt/conda/bin/conda update -v -y -n base -c defaults conda python \
  && /opt/conda/bin/conda clean -v -y -a \
  && /opt/conda/bin/conda run -n base pip cache purge
 
+# No longer relevant since we're using conda-forge in the environment files by default now.
+## Update the base. This helps save space by making sure the same version
+## python is used for both the base env and mlos env.
+#RUN umask 0002 \
+# && /opt/conda/bin/conda update -v -y -n base -c defaults --all \
+# && /opt/conda/bin/conda update -v -y -n base -c defaults conda python \
+# && /opt/conda/bin/conda clean -v -y -a \
+# && /opt/conda/bin/conda run -n base pip cache purge
+
 # Install some additional editor packages for the base environment.
 RUN umask 0002 \
  && /opt/conda/bin/conda run -n base pip install --no-cache-dir -U pynvim

.devcontainer/devcontainer.json

@@ -45,7 +45,6 @@
  // Adjust the python interpreter path to point to the conda environment
  "python.defaultInterpreterPath": "/opt/conda/envs/mlos/bin/python",
  "python.testing.pytestPath": "/opt/conda/envs/mlos/bin/pytest",
- "python.formatting.autopep8Path": "/opt/conda/envs/mlos/bin/autopep8",
  "python.linting.pylintPath": "/opt/conda/envs/mlos/bin/pylint",
  "pylint.path": [
  "/opt/conda/envs/mlos/bin/pylint"
@@ -58,7 +57,6 @@
  // Add the IDs of extensions you want installed when the container is created.
  "extensions": [
  "davidanson.vscode-markdownlint",
- "donjayamanne.githistory",
  "donjayamanne.python-environment-manager",
  "eamodio.gitlens",
  "editorconfig.editorconfig",
@@ -67,14 +65,13 @@
  "github.vscode-github-actions",
  "github.vscode-pull-request-github",
  "gruntfuggly.todo-tree",
- "hashicorp.terraform",
  "huntertran.auto-markdown-toc",
  "ibm.output-colorizer",
  "lextudio.restructuredtext",
  "matangover.mypy",
  "ms-azuretools.vscode-docker",
- "ms-python.autopep8",
- "ms-python.flake8",
+ "ms-python.black-formatter",
+ "ms-python.isort",
  "ms-python.pylint",
  "ms-python.python",
  "ms-python.vscode-pylance",
@@ -83,9 +80,9 @@
  "redhat.vscode-yaml",
  "stkb.rewrap",
  "streetsidesoftware.code-spell-checker",
+ "tamasfe.even-better-toml",
  "trond-snekvik.simple-rst",
- "tyriar.sort-lines",
- "waderyan.gitblame"
+ "tyriar.sort-lines"
  ]
  }
  }

.devcontainer/scripts/common/prep-deps-files.sh

@@ -27,7 +27,7 @@ tmpdir=$(mktemp -d)
 get_python_deps() {
  local pkg="$1"
  touch /tmp/conda-tmp/$pkg.requirements.txt
- python3 /tmp/conda-tmp/$pkg/setup.py egg_info --egg-base "$tmpdir/"
+ (cd /tmp/conda-tmp/$pkg && python3 setup.py egg_info --egg-base "$tmpdir/")
  cat "$tmpdir/$pkg.egg-info/requires.txt" \
  | grep -v -e '^\[' -e '^\s*$' \
  | grep -v '^mlos-' \

.devcontainer/scripts/prep-container-build

@@ -31,9 +31,10 @@ fi
 mkdir -p .devcontainer/tmp/
 cp -v conda-envs/mlos.yml .devcontainer/tmp/mlos.yml
 for pkg in mlos_core mlos_bench mlos_viz; do
- mkdir -p .devcontainer/tmp/$pkg
+ mkdir -p .devcontainer/tmp/$pkg/$pkg
+ cp -v $pkg/pyproject.toml .devcontainer/tmp/$pkg/pyproject.toml
  cp -v $pkg/setup.py .devcontainer/tmp/$pkg/setup.py
- cp -v $pkg/_version.py .devcontainer/tmp/$pkg/_version.py
+ cp -v $pkg/$pkg/version.py .devcontainer/tmp/$pkg/$pkg/version.py
 done
 cp -v doc/requirements.txt .devcontainer/tmp/doc.requirements.txt
 

.devcontainer/scripts/prep-container-build.ps1

@@ -27,9 +27,11 @@ New-Item -Type Directory .devcontainer/tmp
 
 Copy-Item conda-envs/mlos.yml .devcontainer/tmp/mlos.yml
 foreach ($pkg in @('mlos_core', 'mlos_bench', 'mlos_viz')) {
- New-Item -Type Directory ".devcontainer/tmp/${pkg}"
- Copy-Item "$pkg/setup.py" ".devcontainer/tmp/${pkg}/setup.py"
- Copy-Item "$pkg/_version.py" ".devcontainer/tmp/${pkg}/_version.py"
+ New-Item -Type Directory ".devcontainer/tmp/$pkg"
+ New-Item -Type Directory ".devcontainer/tmp/$pkg/$pkg"
+ Copy-Item "$pkg/setup.py" ".devcontainer/tmp/$pkg/setup.py"
+ Copy-Item "$pkg/pyproject.toml" ".devcontainer/tmp/$pkg/pyproject.toml"
+ Copy-Item "$pkg/$pkg/version.py" ".devcontainer/tmp/$pkg/$pkg/version.py"
 }
 Copy-Item doc/requirements.txt .devcontainer/tmp/doc.requirements.txt
 

.git-blame-ignore-revs

@@ -0,0 +1,2 @@
+# Reformat with black, isort, docformatter, etc.
+e40ac28317c61ea90345d3499986957b0e1c9134

.github/workflows/build-dist-test.ps1

@@ -18,7 +18,7 @@ Set-Location mlos_core
 if (Test-Path dist) {
  Remove-Item -Recurse -Force dist
 }
-conda run -n $env:CONDA_ENV_NAME python setup.py bdist_wheel
+conda run -n $env:CONDA_ENV_NAME python -m build
 Set-Location ..
 $mlos_core_whl = (Resolve-Path mlos_core/dist/mlos_core-*-py3-none-any.whl | Select-Object -ExpandProperty Path)
 Write-Host "mlos_core_whl: $mlos_core_whl"
@@ -32,7 +32,7 @@ Set-Location mlos_bench
 if (Test-Path dist) {
  Remove-Item -Recurse -Force dist
 }
-conda run -n $env:CONDA_ENV_NAME python setup.py bdist_wheel
+conda run -n $env:CONDA_ENV_NAME python -m build
 Set-Location ..
 $mlos_bench_whl = (Resolve-Path mlos_bench/dist/mlos_bench-*-py3-none-any.whl | Select-Object -ExpandProperty Path)
 Write-Host "mlos_bench_whl: $mlos_bench_whl"
@@ -46,7 +46,7 @@ Set-Location mlos_viz
 if (Test-Path dist) {
  Remove-Item -Recurse -Force dist
 }
-conda run -n $env:CONDA_ENV_NAME python setup.py bdist_wheel
+conda run -n $env:CONDA_ENV_NAME python -m build
 Set-Location ..
 $mlos_viz_whl = (Resolve-Path mlos_viz/dist/mlos_viz-*-py3-none-any.whl | Select-Object -ExpandProperty Path)
 Write-Host "mlos_viz_whl: $mlos_viz_whl"

.github/workflows/codeql.yml

@@ -11,6 +11,10 @@
 #
 name: "CodeQL"
 
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }}
+ cancel-in-progress: true
+
 on:
  push:
  branches: [ "main" ]

.github/workflows/devcontainer.yml

@@ -21,6 +21,10 @@ on:
  schedule:
  - cron: "1 0 * * *"
 
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }}
+ cancel-in-progress: true
+
 jobs:
  DevContainer:
  runs-on: ubuntu-latest

.github/workflows/linux.yml

@@ -14,6 +14,10 @@ on:
  schedule:
  - cron: "1 0 * * *"
 
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }}
+ cancel-in-progress: true
+
 jobs:
  Linux:
  runs-on: ubuntu-latest
@@ -76,21 +80,21 @@ jobs:
  with:
  #path: ${{ env.CONDA_DIR }}/envs/${{ env.CONDA_ENV_NAME }}
  path: ${{ env.CONDA_DIR }}/pkgs
- key: conda-${{ runner.os }}-${{ env.CONDA_ENV_NAME }}-${{ hashFiles('conda-envs/${{ env.CONDA_ENV_NAME }}.yml') }}-${{ hashFiles('mlos_*/setup.py') }}-${{ env.cache_cur_date }}-${{ env.cache_cur_hour }}
+ key: conda-${{ runner.os }}-${{ env.CONDA_ENV_NAME }}-${{ hashFiles('conda-envs/${{ env.CONDA_ENV_NAME }}.yml') }}-${{ hashFiles('mlos_*/pyproject.toml') }}-${{ hashFiles('mlos_*/setup.py') }}-${{ env.cache_cur_date }}-${{ env.cache_cur_hour }}
  restore-keys: |
- conda-${{ runner.os }}-${{ env.CONDA_ENV_NAME }}-${{ hashFiles('conda-envs/${{ env.CONDA_ENV_NAME }}.yml') }}-${{ hashFiles('mlos_*/setup.py') }}-${{ env.cache_cur_date }}-${{ env.cache_prev_hour }}
- conda-${{ runner.os }}-${{ env.CONDA_ENV_NAME }}-${{ hashFiles('conda-envs/${{ env.CONDA_ENV_NAME }}.yml') }}-${{ hashFiles('mlos_*/setup.py') }}-${{ env.cache_cur_date }}
+ conda-${{ runner.os }}-${{ env.CONDA_ENV_NAME }}-${{ hashFiles('conda-envs/${{ env.CONDA_ENV_NAME }}.yml') }}-${{ hashFiles('mlos_*/pyproject.toml') }}-${{ hashFiles('mlos_*/setup.py') }}-${{ env.cache_cur_date }}-${{ env.cache_prev_hour }}
+ conda-${{ runner.os }}-${{ env.CONDA_ENV_NAME }}-${{ hashFiles('conda-envs/${{ env.CONDA_ENV_NAME }}.yml') }}-${{ hashFiles('mlos_*/pyproject.toml') }}-${{ hashFiles('mlos_*/setup.py') }}-${{ env.cache_cur_date }}
 
  - name: Restore cached pip packages
  id: restore-pip-cache
  if: ${{ github.event_name != 'schedule' }}
  uses: actions/cache@v4
  with:
  path: ${{ env.PIP_CACHE_DIR }}
- key: conda-${{ runner.os }}-${{ env.CONDA_ENV_NAME }}-${{ hashFiles('conda-envs/${{ env.CONDA_ENV_NAME }}.yml') }}-${{ hashFiles('mlos_*/setup.py') }}-${{ env.cache_cur_date }}-${{ env.cache_cur_hour }}
+ key: conda-${{ runner.os }}-${{ env.CONDA_ENV_NAME }}-${{ hashFiles('conda-envs/${{ env.CONDA_ENV_NAME }}.yml') }}-${{ hashFiles('mlos_*/pyproject.toml') }}-${{ hashFiles('mlos_*/setup.py') }}-${{ env.cache_cur_date }}-${{ env.cache_cur_hour }}
  restore-keys: |
- conda-${{ runner.os }}-${{ env.CONDA_ENV_NAME }}-${{ hashFiles('conda-envs/${{ env.CONDA_ENV_NAME }}.yml') }}-${{ hashFiles('mlos_*/setup.py') }}-${{ env.cache_cur_date }}-${{ env.cache_prev_hour }}
- conda-${{ runner.os }}-${{ env.CONDA_ENV_NAME }}-${{ hashFiles('conda-envs/${{ env.CONDA_ENV_NAME }}.yml') }}-${{ hashFiles('mlos_*/setup.py') }}-${{ env.cache_cur_date }}
+ conda-${{ runner.os }}-${{ env.CONDA_ENV_NAME }}-${{ hashFiles('conda-envs/${{ env.CONDA_ENV_NAME }}.yml') }}-${{ hashFiles('mlos_*/pyproject.toml') }}-${{ hashFiles('mlos_*/setup.py') }}-${{ env.cache_cur_date }}-${{ env.cache_prev_hour }}
+ conda-${{ runner.os }}-${{ env.CONDA_ENV_NAME }}-${{ hashFiles('conda-envs/${{ env.CONDA_ENV_NAME }}.yml') }}-${{ hashFiles('mlos_*/pyproject.toml') }}-${{ hashFiles('mlos_*/setup.py') }}-${{ env.cache_cur_date }}
 
  - name: Log some environment variables for debugging
  run: |

.github/workflows/markdown-link-check.yml

@@ -12,6 +12,10 @@ on:
  merge_group:
  types: [checks_requested]
 
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }}
+ cancel-in-progress: true
+
 jobs:
  # Check in-repo markdown links
  markdown-link-check:

.github/workflows/windows.yml

@@ -15,6 +15,10 @@ on:
  schedule:
  - cron: "1 0 * * *"
 
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }}
+ cancel-in-progress: true
+
 jobs:
  Windows:
  runs-on: windows-latest
@@ -64,10 +68,10 @@ jobs:
  uses: actions/cache@v4
  with:
  path: ${{ env.CONDA }}/envs/mlos
- key: conda-${{ runner.os }}-${{ env.CONDA_ENV_NAME }}-${{ hashFiles('conda-envs/${{ env.CONDA_ENV_YML }}') }}-${{ hashFiles('mlos_*/setup.py') }}-${{ env.cache_cur_date }}-${{ env.cache_cur_hour }}
+ key: conda-${{ runner.os }}-${{ env.CONDA_ENV_NAME }}-${{ hashFiles('conda-envs/${{ env.CONDA_ENV_YML }}') }}-${{ hashFiles('mlos_*/pyproject.toml') }}-${{ hashFiles('mlos_*/setup.py') }}-${{ env.cache_cur_date }}-${{ env.cache_cur_hour }}
  restore-keys: |
- conda-${{ runner.os }}-${{ env.CONDA_ENV_NAME }}-${{ hashFiles('conda-envs/${{ env.CONDA_ENV_YML }}') }}-${{ hashFiles('mlos_*/setup.py') }}-${{ env.cache_cur_date }}-${{ env.cache_prev_hour }}
- conda-${{ runner.os }}-${{ env.CONDA_ENV_NAME }}-${{ hashFiles('conda-envs/${{ env.CONDA_ENV_YML }}') }}-${{ hashFiles('mlos_*/setup.py') }}-${{ env.cache_cur_date }}
+ conda-${{ runner.os }}-${{ env.CONDA_ENV_NAME }}-${{ hashFiles('conda-envs/${{ env.CONDA_ENV_YML }}') }}-${{ hashFiles('mlos_*/pyproject.toml') }}-${{ hashFiles('mlos_*/setup.py') }}-${{ env.cache_cur_date }}-${{ env.cache_prev_hour }}
+ conda-${{ runner.os }}-${{ env.CONDA_ENV_NAME }}-${{ hashFiles('conda-envs/${{ env.CONDA_ENV_YML }}') }}-${{ hashFiles('mlos_*/pyproject.toml') }}-${{ hashFiles('mlos_*/setup.py') }}-${{ env.cache_cur_date }}
 
  - name: Log some environment variables for debugging
  run: |

.gitignore

@@ -1,3 +1,6 @@
+# Ignore git directory (ripgrep)
+.git/
+
 # Byte-compiled / optimized / DLL files
 __pycache__/
 *.py[cod]
@@ -150,7 +153,9 @@ build/*.build-stamp
 .vscode/
 !.vscode/settings.json
 !.vscode/extensions.json
+!.vscode/launch.json
 !.devcontainer/build/
+!.devcontainer/scripts/
 
 # Test configs. May contain sensitive information
 # like Azure credentials, DB password, etc.

.vscode/extensions.json

@@ -1,7 +1,6 @@
 {
  "recommendations": [
  "davidanson.vscode-markdownlint",
- "donjayamanne.githistory",
  "donjayamanne.python-environment-manager",
  "eamodio.gitlens",
  "editorconfig.editorconfig",
@@ -10,14 +9,13 @@
  "github.vscode-github-actions",
  "github.vscode-pull-request-github",
  "gruntfuggly.todo-tree",
- "hashicorp.terraform",
  "huntertran.auto-markdown-toc",
  "ibm.output-colorizer",
  "lextudio.restructuredtext",
  "matangover.mypy",
  "ms-azuretools.vscode-docker",
- "ms-python.autopep8",
- "ms-python.flake8",
+ "ms-python.black-formatter",
+ "ms-python.isort",
  "ms-python.pylint",
  "ms-python.python",
  "ms-python.vscode-pylance",
@@ -26,8 +24,8 @@
  "redhat.vscode-yaml",
  "stkb.rewrap",
  "streetsidesoftware.code-spell-checker",
+ "tamasfe.even-better-toml",
  "trond-snekvik.simple-rst",
- "tyriar.sort-lines",
- "waderyan.gitblame"
+ "tyriar.sort-lines"
  ]
 }

.vscode/settings.json

@@ -125,11 +125,8 @@
  ],
  "esbonio.sphinx.confDir": "${workspaceFolder}/doc/source",
  "esbonio.sphinx.buildDir": "${workspaceFolder}/doc/build/",
- "autopep8.args": [
- "--experimental"
- ],
  "[python]": {
- "editor.defaultFormatter": "ms-python.autopep8",
+ "editor.defaultFormatter": "ms-python.black-formatter",
  "editor.formatOnSave": true,
  "editor.formatOnSaveMode": "modifications"
  },

CONTRIBUTING.md

@@ -62,17 +62,35 @@ Simply open the project in VSCode and follow the prompts to build and open the d
 We expect development to follow a typical "forking" style workflow:
 
 1. Fork a copy of the [MLOS repo in Github](https://github.com/microsoft/MLOS).
-2. Create a development (a.k.a. topic) branch off of `main` to work on changes.
+1. Create a development (a.k.a. topic) branch off of `main` to work on changes.
 
  For instance:
 
  ```shell
  git checkout -b YourDevName/some-topic-description main
  ```
 
-3. Submit changes for inclusion as a [Pull Request on Github](https://github.com/microsoft/MLOS/pulls).
-4. PRs are associated with [Github Issues](https://github.com/microsoft/MLOS/issues) and need [MLOS-committers](https://github.com/orgs/microsoft/teams/MLOS-committers) to sign-off (in addition to other CI pipeline checks like tests and lint checks to pass).
-5. Once approved, the PR can be completed using a squash merge in order to keep a nice linear history.
+1. Ensure all of the lint checks and tests pass.
+
+ The easiest way to do this is to run the `make` commands that are also used in the CI pipeline:
+
+ ```shell
+ # All at once.
+ make all
+
+ # Or individually (for easier debugging)
+ make check
+ make test
+ make dist-test
+ make doc-test
+ ```
+
+1. Submit changes for inclusion as a [Pull Request on Github](https://github.com/microsoft/MLOS/pulls).
+
+ > Please try to keep PRs small whenver possible and don't include unnecessaary formatting changes.
+
+1. PRs are associated with [Github Issues](https://github.com/microsoft/MLOS/issues) and need [MLOS-committers](https://github.com/orgs/microsoft/teams/MLOS-committers) to sign-off (in addition to other CI pipeline checks like tests and lint checks to pass).
+1. Once approved, the PR can be completed using a squash merge in order to keep a nice linear history.
 
 ## Distributing