microsoft · motus · May 3, 2023 · Apr 22, 2023 · Apr 22, 2023 · Apr 22, 2023
@@ -12,6 +12,7 @@
         "dataframe",
         "discretization",
         "discretize",
+        "drivername",
         "dtype",
         "duckdb",
         "emukit",

@@ -119,9 +119,10 @@ mypy: conda-env build/mypy.mlos_core.${CONDA_ENV_NAME}.build-stamp build/mypy.ml
 build/mypy.mlos_core.${CONDA_ENV_NAME}.build-stamp: $(MLOS_CORE_PYTHON_FILES)
 build/mypy.mlos_bench.${CONDA_ENV_NAME}.build-stamp: $(MLOS_BENCH_PYTHON_FILES) build/mypy.mlos_core.${CONDA_ENV_NAME}.build-stamp
 
+NON_MYPY_FILES := scripts/dmypy-wrapper.sh build/conda-env.${CONDA_ENV_NAME}.build-stamp build/mypy.mlos_core.${CONDA_ENV_NAME}.build-stamp setup.cfg
 build/mypy.%.${CONDA_ENV_NAME}.build-stamp: scripts/dmypy-wrapper.sh build/conda-env.${CONDA_ENV_NAME}.build-stamp setup.cfg
 	conda run -n ${CONDA_ENV_NAME} scripts/dmypy-wrapper.sh \
-		$(filter-out scripts/dmypy-wrapper.sh build/*.build-stamp setup.cfg,$+)
+		$(filter-out $(NON_MYPY_FILES),$+)
 	touch $@
 
 
@@ -204,6 +205,7 @@ mlos_bench/dist/tmp/mlos-bench-latest.tar: PACKAGE_NAME := mlos-bench
 	cd $(MODULE_NAME)/ && conda run -n ${CONDA_ENV_NAME} python3 setup.py sdist --formats tar
 	ls $(MODULE_NAME)/dist/$(PACKAGE_NAME)-*.tar
 	! ( tar tf $(MODULE_NAME)/dist/$(PACKAGE_NAME)-*.tar | grep -m1 tests/ )
+	[ "$(MODULE_NAME)" != "mlos_bench" ] || tar tf $(MODULE_NAME)/dist/$(PACKAGE_NAME)-*.tar | grep -m1 mlos_bench/config/
 	cd $(MODULE_NAME)/dist/tmp && ln -s ../$(PACKAGE_NAME)-*.tar $(PACKAGE_NAME)-latest.tar
 
 %-latest-py3-none-any.whl: build/conda-env.${CONDA_ENV_NAME}.build-stamp
@@ -214,6 +216,8 @@ mlos_bench/dist/tmp/mlos-bench-latest.tar: PACKAGE_NAME := mlos-bench
 	ls $(MODULE_NAME)/dist/$(MODULE_NAME)-*-py3-none-any.whl
 	# Check to make sure the tests were excluded from the wheel.
 	! ( unzip -t $(MODULE_NAME)/dist/$(MODULE_NAME)-*-py3-none-any.whl | grep -m1 tests/ )
+	# Check to make sure the mlos_bench module has the config directory.
+	[ "$(MODULE_NAME)" != "mlos_bench" ] || unzip -t $(MODULE_NAME)/dist/$(MODULE_NAME)-*-py3-none-any.whl | grep -m1 mlos_bench/config/
 	cd $(MODULE_NAME)/dist/tmp && ln -s ../$(MODULE_NAME)-*-py3-none-any.whl $(MODULE_NAME)-latest-py3-none-any.whl
 
 .PHONY: dist-test-env-clean
@@ -227,7 +231,7 @@ dist-test-env: dist build/dist-test-env.$(PYTHON_VERSION).build-stamp
 
 build/dist-test-env.$(PYTHON_VERSION).build-stamp: build/conda-env.${CONDA_ENV_NAME}.build-stamp
 # Use the same version of python as the one we used to build the wheels.
-build/dist-test-env.$(PYTHON_VERSION).build-stamp: PYTHON_VERS_REQ=$(shell conda list -n ${CONDA_ENV_NAME} | egrep '^python\s+' | sed -r -e 's/^python\s+//' | cut -d' ' -f1)
+build/dist-test-env.$(PYTHON_VERSION).build-stamp: PYTHON_VERS_REQ=$(shell conda list -n ${CONDA_ENV_NAME} | egrep '^python\s+' | sed -r -e 's/^python\s+//' | cut -d' ' -f1 | cut -d. -f1-2)
 build/dist-test-env.$(PYTHON_VERSION).build-stamp: mlos_core/dist/tmp/mlos_core-latest-py3-none-any.whl mlos_bench/dist/tmp/mlos_bench-latest-py3-none-any.whl
 	# Create a clean test environment for checking the wheel files.
 	$(MAKE) dist-test-env-clean
@@ -236,6 +240,8 @@ build/dist-test-env.$(PYTHON_VERSION).build-stamp: mlos_core/dist/tmp/mlos_core-
 	conda run -n mlos-dist-test-$(PYTHON_VERSION) pip install "mlos_core/dist/tmp/mlos_core-latest-py3-none-any.whl[full-tests]"
 	# Test a clean install of the mlos_bench wheel.
 	conda run -n mlos-dist-test-$(PYTHON_VERSION) pip install "mlos_bench/dist/tmp/mlos_bench-latest-py3-none-any.whl[full-tests]"
+	# Test that the config dir for mlos_bench got distributed.
+	test -e `conda env list | grep "mlos-dist-test-$(PYTHON_VERSION) " | awk '{ print $$2 }'`/lib/python$(PYTHON_VERS_REQ)/site-packages/mlos_bench/config/README.md
 	touch $@
 
 .PHONY: dist-test
@@ -338,7 +344,7 @@ build/check-doc.build-stamp: doc/build/html/index.html doc/build/html/htmlcov/in
 	test -s doc/build/html/generated/mlos_bench.environments.Environment.html
 	test -s doc/build/html/api/mlos_core/mlos_core.html
 	test -s doc/build/html/api/mlos_bench/mlos_bench.html
-	grep -q options: doc/build/html/api/mlos_bench/mlos_bench.run.html
+	grep -q -e '--config CONFIG' doc/build/html/api/mlos_bench/mlos_bench.run.html
 	# Check doc logs for errors (but skip over some known ones) ...
 	@cat doc/build/log.txt \
 		| egrep -C1 -e WARNING -e CRITICAL -e ERROR \

@@ -186,7 +186,7 @@ All we have to do is specifying the [`Storage`](./mlos_bench/storage/) and [`Opt
 }
 ```
 
-> **NOTE:** A working example of this configuration can be found in our repository at [azure-redis-1shot.jsonc](examples/azure-redis-1shot.jsonc).
+> **NOTE:** A working example of this configuration can be found in our repository at [azure-redis-1shot.jsonc](./mlos_bench/config/cli/azure-redis-1shot.jsonc).
 
 The only difference between the two configurations is that the latter has the `optimizer` parameter instead of using a fixed set of tunables via `tunable_values` option.
 It also stores the results of each trial in the SQLite3 database (configured via `storage`).

@@ -0,0 +1,9 @@
+# `config` Examples Overview
+
+The [`config`](./) directory contains a collection of scripts and config snippets that are used to configure the `mlos_bench` components.
+
+These are meant to be used as examples and starting points for your own configuration, though some can be included as-is in your configuration (e.g. linux kernel configs).
+
+In general the `config` directory layout follows that of the `mlos_bench` module/directory layout (e.g. `remote` and `local` `Environments` making using of `Services`, etc., each with their own `json` configs and shell scripts.).
+
+Full end-to-end examples are provided in the [`cli`](./cli/) directory, and typically and make use of the [`CompositeEnvironments`](./environments/composite/) to combine multiple [`Environments`](./environments/), also referencing [`Services`](./services/), [`Storage`](./storage/), and [`Optimizer`](./optimizers/) configs, into a single [`mlos_bench`](../run.py) run.
@@ -0,0 +1,19 @@
+{
+    "redis": {
+        "cost": 1,
+        "params": {
+            "hz": {
+                "description": "Background tasks frequency",
+                "type": "int",
+                "default": 10,
+                "range": [1, 500]
+            },
+            "dynamic_hz": {
+                "description": "Enable adaptive hz",
+                "type": "categorical",
+                "default": "yes",
+                "values": ["yes", "no"]
+            }
+        }
+    }
+}
@@ -4,6 +4,7 @@
         "name": "Generate Redis configuration",
         "class": "mlos_bench.environments.local.LocalFileShareEnv",
 
+        "include_tunables": ["environments/apps/redis/redis-tunables.jsonc"],
         "config": {
             "tunable_params": ["redis"],
 
@@ -13,12 +14,13 @@
             "dump_params_file": "redis-params.json",
 
             "setup": [
-                "applications/generate_redis_config.py redis-params.json redis.cfg"
+                "environment/apps/redis/local/scripts/generate_redis_config.py redis-params.json redis.cfg"
             ],
 
+            // TODO: improve handling of relative paths
             "upload": [
                 {"from": "redis.cfg", "to": "$experimentId/$trialId/input/redis.cfg"},
-                {"from": "redis/", "to": "$experimentId/$trialId/scripts"}
+                {"from": "remote/scripts/", "to": "$experimentId/$trialId/scripts"}
             ]
         }
     },
@@ -28,8 +30,8 @@
         "name": "Benchmark Redis on Linux",
         "class": "mlos_bench.environments.remote.RemoteEnv",
 
+        "include_tunables": ["environments/apps/redis/redis-tunables.jsonc"],
         "config": {
-
             "tunable_params": ["redis"],
 
             "required_args": ["experimentId", "trialId", "mountPoint"],
@@ -61,8 +63,8 @@
         "name": "Download and post-process Redis benchmark data",
         "class": "mlos_bench.environments.local.LocalFileShareEnv",
 
+        "include_tunables": ["environments/apps/redis/redis-tunables.jsonc"],
         "config": {
-
             "tunable_params": ["redis"],
 
             "required_args": ["experimentId", "trialId"],
@@ -75,7 +77,7 @@
             ],
 
             "run": [
-                "applications/process_redis_results.py redis-bench-output/results.csv redis-benchmark.csv"
+                "environments/apps/redis/local/scripts/process_redis_results.py redis-bench-output/results.csv redis-benchmark.csv"
             ],
 
             // Read benchmark results from a local CSV file:

@@ -0,0 +1,25 @@
+{
+    "name": "Azure VM Ubuntu Redis",
+    "class": "mlos_bench.environments.CompositeEnv",
+
+    "include_tunables": [
+        "environments/vm/azure/azure-vm-tunables.jsonc",
+        "environments/os/linux/boot/linux-boot-tunables.jsonc",
+        "environments/os/linux/runtime/linux-runtime-tunables.jsonc",
+        "environments/apps/redis/redis-tunables.jsonc"
+    ],
+
+    "include_services": [
+        "services/remote/azure/service-linux-vm-ops.jsonc",
+        "services/remote/azure/service-fileshare.jsonc"
+    ],
+
+    "config": {
+        "include_children": [
+            "environments/vm/azure/provision-ubuntu-vm.jsonc",
+            "environments/os/linux/boot/boot-ubuntu.jsonc",
+            "environments/os/linux/runtime/setup-ubuntu.jsonc",
+            "environments/apps/redis/redis.jsonc"
+        ]
+    }
+}
@@ -3,10 +3,15 @@
     "name": "Mock environment",
     "class": "mlos_bench.environments.mock_env.MockEnv",
 
-    "include_tunables": ["tunables.jsonc"],
+    "include_tunables": [
+        "environments/apps/redis/redis-tunables.jsonc",
+        "environments/os/linux/runtime/linux-runtime-tunables.jsonc",
+        "environments/os/linux/boot/linux-boot-tunables.jsonc",
+        "environments/vm/azure/azure-vm-tunables.jsonc"
+    ],
 
     "config": {
-        "tunable_params": ["provision", "boot", "kernel", "redis"],
+        "tunable_params": ["azure-vm", "linux-kernel-boot", "linux-kernel-runtime", "redis"],
         "required_args": ["experimentId", "trialId"],
         "seed": 42,         // Seed for the random noise generator. Omit to produce noise-free data.
         "range": [60, 120]  // Range of the generated output value of the benchmark.

@@ -11,9 +11,10 @@
         "name": "Generate boot parameters for a Ubuntu VM",
         "class": "mlos_bench.environments.local.LocalFileShareEnv",
 
-        "config": {
+        "include_tunables": ["environments/os/linux/boot/linux-boot-tunables.jsonc"],
 
-            "tunable_params": ["boot"],
+        "config": {
+            "tunable_params": ["linux-kernel-boot"],
 
             // Values provided from the global config/optimizer:
             "required_args": ["experimentId", "trialId"],
@@ -26,8 +27,11 @@
             // Dump tunable parameters to a local JSON file:
             "dump_params_file": "boot-params.json",
 
+            // TODO: expand "setup" command to support array syntax like devcontainers so we can execute without a shell.
+            // In particular this should let us resolve the path to the script more easily.
+            // Alternatively: we could provide a "CWD" argument to the setup command.
             "setup": [
-                "linux-boot/generate_grub_config.py boot-params.json grub.cfg"
+                "environment/os/linux/boot/generate_grub_config.py boot-params.json grub.cfg"
             ],
 
             "upload": [
@@ -44,9 +48,10 @@
         "name": "Apply boot parameters to a remote Ubuntu VM",
         "class": "mlos_bench.environments.RemoteEnv",
 
-        "config": {
+        "include_tunables": ["environments/os/linux/boot/linux-boot-tunables.jsonc"],
 
-            "tunable_params": ["boot"],
+        "config": {
+            "tunable_params": ["linux-kernel-boot"],
 
             "required_args": [
                 "storageAccountName",

@@ -0,0 +1,54 @@
+{
+    "linux-kernel-boot": {
+        "cost": 300,
+        // See Also: https://www.kernel.org/doc/html/latest/admin-guide/kernel-parameters.html
+        // NOTE: This is an incomplete list.
+        "params": {
+            "align_va_addr": {
+                "type": "categorical",
+                "default": "on",
+                "values": ["on", "off"]
+            },
+            "idle": {
+                "type": "categorical",
+                "default": "halt",
+                "values": ["halt", "poll", "nomwait"]
+            },
+            "ima.ahash_bufsize": {
+                "type": "int",
+                "default": 4096,
+                "range": [0, 65536]
+            },
+            "noautogroup": {
+                "type": "categorical",
+                "default": null,
+                "values": [null, "noautogroup"]
+            },
+            "nohugevmalloc": {
+                "type": "categorical",
+                "default": null,
+                "values": [null, "nohugevmalloc"]
+            },
+            "nohalt": {
+                "type": "categorical",
+                "default": null,
+                "values": [null, "nohalt"]
+            },
+            "nohz": {
+                "type": "categorical",
+                "default": null,
+                "values": [null, "nohz"]
+            },
+            "no-kvmapf": {
+                "type": "categorical",
+                "default": null,
+                "values": [null, "no-kvmapf"]
+            },
+            "nopvspin": {
+                "type": "categorical",
+                "default": null,
+                "values": [null, "nopvspin"]
+            }
+        }
+    }
+}