Merged PR 1054: Prepwork for OSSing

- include some boilerplate from the original MLOS repo
- adding license headers on scripts
- add rules for checking licenseheaders

.azure-pipelines/linux.yml

@@ -161,6 +161,13 @@ jobs:
  docker exec --user root mlos-core-devcontainer chown -R vscode /opt/conda/pkgs/cache /var/cache/pip
  displayName: 'Fixup vscode uid/gid in the running container'
 
+ - bash: |
+ set -x
+ docker exec --user vscode mlos-core-devcontainer make CONDA_INFO_LEVEL=-v licenseheaders
+ # licenseheaders changes the contents of the files, so make this check fail if there are any changes detected
+ git diff --no-pager --exit-code
+ displayName: 'Check that all of the python files have licenseheaders'
+
  - bash: |
  set -x
  docker exec --user vscode mlos-core-devcontainer make CONDA_INFO_LEVEL=-v conda-env

.devcontainer/Dockerfile

@@ -1,3 +1,6 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+
 FROM mcr.microsoft.com/devcontainers/miniconda:3 AS base
 
 # Add some additional packages for the devcontainer terminal environment.

.devcontainer/build/Dockerfile

@@ -1,3 +1,6 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+
 # Start with npm
 FROM node:lts-slim
 

.devcontainer/build/build-devcontainer-cli.ps1

@@ -1,6 +1,9 @@
 # Requires -Version 5.0
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+
 # A script to build the devcontainer-cli image.
-#
+
 $ErrorActionPreference = 'Stop'
 
 # Make sure we're in the root of the repository.

.devcontainer/build/build-devcontainer-cli.sh

@@ -1,4 +1,8 @@
 #!/bin/bash
+##
+## Copyright (c) Microsoft Corporation.
+## Licensed under the MIT License.
+##
 
 set -x
 

.devcontainer/build/build-devcontainer.ps1

@@ -1,6 +1,9 @@
 # Requires -Version 5.0
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+
 # A script to build the devcontainer image.
-#
+
 $ErrorActionPreference = 'Stop'
 
 # Make sure we're in the root of the repository.

.devcontainer/build/build-devcontainer.sh

@@ -1,4 +1,8 @@
 #!/bin/bash
+##
+## Copyright (c) Microsoft Corporation.
+## Licensed under the MIT License.
+##
 
 set -x
 

.devcontainer/scripts/common/prep-deps-files.sh

@@ -1,4 +1,8 @@
 #!/bin/bash
+##
+## Copyright (c) Microsoft Corporation.
+## Licensed under the MIT License.
+##
 
 # Prep some files for the devcontainer build in a way that's cross platform.
 # We do this by running this script from a container that has the tools we need

.devcontainer/scripts/prep-container-build

@@ -1,4 +1,6 @@
 #!/bin/sh
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
 
 set -eu
 

.devcontainer/scripts/prep-container-build.ps1

@@ -1,4 +1,7 @@
 # Requires -Version 3.0
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+
 # A script to prepare the build environment for the devcontainer.
 
 $ErrorActionPreference = 'Stop'

.devcontainer/scripts/run-cspell.sh

@@ -1,4 +1,8 @@
 #!/bin/bash
+##
+## Copyright (c) Microsoft Corporation.
+## Licensed under the MIT License.
+##
 
 set -x
 

.devcontainer/scripts/run-devcontainer.sh

@@ -1,4 +1,8 @@
 #!/bin/bash
+##
+## Copyright (c) Microsoft Corporation.
+## Licensed under the MIT License.
+##
 
 # Quick hacky script to start a devcontainer in a non-vscode shell for testing.
 # See Also:

CODE_OF_CONDUCT.md

@@ -0,0 +1,9 @@
+# Microsoft Open Source Code of Conduct
+
+This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/).
+
+Resources:
+
+- [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/)
+- [Microsoft Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/)
+- Contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with questions or concerns

CONTRIBUTING.md

@@ -0,0 +1,37 @@
+# Contributing to MLOS
+
+This project welcomes contributions and suggestions.
+Most contributions require you to agree to a Contributor License Agreement (CLA) declaring that you have the right to, and actually do, grant us the rights to use your contribution.
+For details, visit https://cla.opensource.microsoft.com.
+
+When you submit a pull request, a CLA bot will automatically determine whether you need to provide a CLA and decorate the PR appropriately (e.g., status check, comment).
+Simply follow the instructions provided by the bot. You will only need to do this once across all repos using our CLA.
+
+This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/).
+For more information see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) or contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments.
+
+## Details
+
+[`main`](https://github.com/microsoft/MLOS/tree/main) is considered the primary development branch.
+
+We expect development to follow a typical "gitflow" style workflow:
+
+1. Fork a copy of the [MLOS repo in Github](https://github.com/microsoft/MLOS).
+2. Create a development (a.k.a. topic) branch off of `main` to work on changes.
+
+ ```shell
+ git checkout -b YourDevName/some-topic-description main
+ ```
+
+3. Submit changes for inclusion as a [Pull Request on Github](https://github.com/microsoft/MLOS/pulls).
+4. PRs are associated with [Github Issues](https://github.com/microsoft/MLOS/issues) and need [MLOS-committers](https://github.com/orgs/microsoft/teams/MLOS-committers) to sign-off (in addition to other CI pipeline checks like tests and lint checks to pass).
+5. Once approved, the PR can be completed using a squash merge in order to keep a nice linear history.
+
+### Caveats
+
+There are consumers of MLOS internal to Microsoft that use an internal copy of the Github repo targetting code that is not open-sourced.
+This arrangement sometimes means porting changes from the internal repo to Github (and vise-versa).
+When that happens, the changes are submitted as a PR as described above, with the slight modification of (once approved and passing tests) using a rebase based merge instead of a squash merge in order to allow detecting duplicate patches between the public and private repos.
+
+Additionally, to try and catch breaking changes we run some extra internal integration tests as well.
+If they do find issues, we encourage a conversation to occur on how to resolve them in the PRs.

Makefile

@@ -1,3 +1,6 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+
 CONDA_ENV_NAME ?= mlos_core
 PYTHON_VERSION := $(shell echo "${CONDA_ENV_NAME}" | sed -r -e 's/^mlos_core[-]?//')
 ENV_YML := conda-envs/${CONDA_ENV_NAME}.yml
@@ -38,7 +41,7 @@ clean-conda-env:
  rm -f build/conda-env.${CONDA_ENV_NAME}.build-stamp
 
 .PHONY: check
-check: pycodestyle pydocstyle pylint # cspell
+check: pycodestyle pydocstyle pylint # cspell licenseheaders
 
 .PHONY: pycodestyle
 pycodestyle: conda-env build/pycodestyle.mlos_core.${CONDA_ENV_NAME}.build-stamp build/pycodestyle.mlos_bench.${CONDA_ENV_NAME}.build-stamp
@@ -63,6 +66,17 @@ build/pydocstyle.%.${CONDA_ENV_NAME}.build-stamp: build/conda-env.${CONDA_ENV_NA
  conda run -n ${CONDA_ENV_NAME} pydocstyle $(filter-out setup.cfg,$+)
  touch $@
 
+.PHONY: licenseheaders
+licenseheaders: build/licenseheaders.${CONDA_ENV_NAME}.build-stamp
+
+build/licenseheaders-prereqs.${CONDA_ENV_NAME}.build-stamp: build/conda-env.${CONDA_ENV_NAME}.build-stamp
+ conda run -n ${CONDA_ENV_NAME} pip install licenseheaders
+ touch $@
+
+build/licenseheaders.${CONDA_ENV_NAME}.build-stamp: build/licenseheaders-prereqs.${CONDA_ENV_NAME}.build-stamp $(PYTHON_FILES) doc/mit-license.tmpl
+ conda run -n ${CONDA_ENV_NAME} licenseheaders -t doc/mit-license.tmpl -E .py .sh .ps1 -x mlos_bench/setup.py mlos_core/setup.py
+ touch $@
+
 .PHONY: cspell
 ifeq ($(DOCKER),)
 cspell:
@@ -361,6 +375,8 @@ clean-check:
  rm -f build/pydocstyle.${CONDA_ENV_NAME}.build-stamp
  rm -f build/pydocstyle.mlos_core.${CONDA_ENV_NAME}.build-stamp
  rm -f build/pydocstyle.mlos_bench.${CONDA_ENV_NAME}.build-stamp
+ rm -f build/licenseheaders.${CONDA_ENV_NAME}.build-stamp
+ rm -f build/licenseheaders-prereqs.${CONDA_ENV_NAME}.build-stamp
 
 .PHONY: clean-test
 clean-test:

NOTICE

@@ -0,0 +1,3 @@
+NOTICES
+
+This repository incorporates material as listed below or described in the code.

SECURITY.md

@@ -0,0 +1,41 @@
+<!-- BEGIN MICROSOFT SECURITY.MD V0.0.5 BLOCK -->
+
+## Security
+
+Microsoft takes the security of our software products and services seriously, which includes all source code repositories managed through our GitHub organizations, which include [Microsoft](https://github.com/Microsoft), [Azure](https://github.com/Azure), [DotNet](https://github.com/dotnet), [AspNet](https://github.com/aspnet), [Xamarin](https://github.com/xamarin), and [our GitHub organizations](https://opensource.microsoft.com/).
+
+If you believe you have found a security vulnerability in any Microsoft-owned repository that meets [Microsoft's definition of a security vulnerability](https://docs.microsoft.com/en-us/previous-versions/tn-archive/cc751383(v=technet.10)), please report it to us as described below.
+
+## Reporting Security Issues
+
+**Please do not report security vulnerabilities through public GitHub issues.**
+
+Instead, please report them to the Microsoft Security Response Center (MSRC) at [https://msrc.microsoft.com/create-report](https://msrc.microsoft.com/create-report).
+
+If you prefer to submit without logging in, send email to [secure@microsoft.com](mailto:secure@microsoft.com). If possible, encrypt your message with our PGP key; please download it from the [Microsoft Security Response Center PGP Key page](https://www.microsoft.com/en-us/msrc/pgp-key-msrc).
+
+You should receive a response within 24 hours. If for some reason you do not, please follow up via email to ensure we received your original message. Additional information can be found at [microsoft.com/msrc](https://www.microsoft.com/msrc). 
+
+Please include the requested information listed below (as much as you can provide) to help us better understand the nature and scope of the possible issue:
+
+ * Type of issue (e.g. buffer overflow, SQL injection, cross-site scripting, etc.)
+ * Full paths of source file(s) related to the manifestation of the issue
+ * The location of the affected source code (tag/branch/commit or direct URL)
+ * Any special configuration required to reproduce the issue
+ * Step-by-step instructions to reproduce the issue
+ * Proof-of-concept or exploit code (if possible)
+ * Impact of the issue, including how an attacker might exploit the issue
+
+This information will help us triage your report more quickly.
+
+If you are reporting for a bug bounty, more complete reports can contribute to a higher bounty award. Please visit our [Microsoft Bug Bounty Program](https://microsoft.com/msrc/bounty) page for more details about our active programs.
+
+## Preferred Languages
+
+We prefer all communications to be in English.
+
+## Policy
+
+Microsoft follows the principle of [Coordinated Vulnerability Disclosure](https://www.microsoft.com/en-us/msrc/cvd).
+
+<!-- END MICROSOFT SECURITY.MD BLOCK -->

conftest.py

@@ -1,3 +1,7 @@
+#
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+#
 """
 Provides some pytest configuration overrides for both modules.
 """

doc/mit-license.tmpl

@@ -0,0 +1,2 @@
+Copyright (c) Microsoft Corporation.
+Licensed under the MIT License.

doc/nginx-docker.sh

@@ -1,4 +1,8 @@
 #!/bin/bash
+##
+## Copyright (c) Microsoft Corporation.
+## Licensed under the MIT License.
+##
 
 # A quick script to start a local webserver for testing the sphinx documentation.
 

doc/source/_templates/numpydoc_docstring.py

@@ -1,3 +1,7 @@
+#
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+#
 {{index}}
 {{summary}}
 {{extended_summary}}

doc/source/conf.py

@@ -1,3 +1,8 @@
+#
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+#
+
 # Configuration file for the Sphinx documentation builder.
 #
 # This file only contains a selection of the most common options. For a full

mlos_bench/config/applications/generate_redis_config.py

@@ -1,4 +1,8 @@
 #!/usr/bin/env python3
+#
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+#
 """
 Helper script to generate Redis config from tunable parameters JSON.
 

mlos_bench/config/applications/process_redis_results.py

@@ -1,4 +1,8 @@
 #!/usr/bin/env python3
+#
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+#
 """
 Script for post-processing redis-benchmark results.
 """

mlos_bench/config/linux-boot/generate_grub_config.py

@@ -1,4 +1,8 @@
 #!/usr/bin/env python3
+#
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+#
 """
 Helper script to generate GRUB config from tunable parameters JSON.
 

mlos_bench/config/linux-setup/generate_kernel_config_script.py

@@ -1,4 +1,8 @@
 #!/usr/bin/env python3
+#
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+#
 """
 Helper script to generate a script to update kernel parameters from tunables JSON.
 

mlos_bench/examples/linux-kernel/cleanup-os-boot-time.sh

@@ -1,4 +1,8 @@
 #!/bin/bash
+##
+## Copyright (c) Microsoft Corporation.
+## Licensed under the MIT License.
+##
 
 # Script to restore boot-time parameters of VM to original state.
 # This script should be run in the VM.

mlos_bench/examples/linux-kernel/cleanup-os-runtime.sh

@@ -1,4 +1,8 @@
 #!/bin/bash
+##
+## Copyright (c) Microsoft Corporation.
+## Licensed under the MIT License.
+##
 
 # Script to restore runtime parameters of VM to original state.
 # This script should be run in the VM.

mlos_bench/examples/linux-kernel/common-boot-time.sh

@@ -1,3 +1,8 @@
+##
+## Copyright (c) Microsoft Corporation.
+## Licensed under the MIT License.
+##
+
 # Environment variables, distribution check, root access for boot-time parameters.
 # This file should be in the VM.
 

mlos_bench/examples/linux-kernel/common-runtime.sh

@@ -1,3 +1,8 @@
+##
+## Copyright (c) Microsoft Corporation.
+## Licensed under the MIT License.
+##
+
 # Environment variables and root access for runtime parameters.
 # This file should be in the VM.
 

mlos_bench/examples/linux-kernel/create_new_grub_cfg.py

@@ -1,4 +1,8 @@
 #!/usr/bin/env python3
+#
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+#
 """
 Python script to parse through JSON and create new config file.
 

mlos_bench/examples/linux-kernel/parse_runtime_params.py

@@ -1,4 +1,8 @@
 #!/usr/bin/env python3
+#
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+#
 """
 Python script to parse through JSON, store runtime parameter paths, and
 create script to apply new runtime parameters in VM.

mlos_bench/examples/linux-kernel/prepare-os-boot-time.sh

@@ -1,4 +1,8 @@
 #!/bin/bash
+##
+## Copyright (c) Microsoft Corporation.
+## Licensed under the MIT License.
+##
 
 # Script to store old grub config and reboot VM (if necessary).
 # Config file created in scheduler should have been moved to

mlos_bench/examples/linux-kernel/prepare-os-runtime.sh

@@ -1,4 +1,8 @@
 #!/bin/bash
+##
+## Copyright (c) Microsoft Corporation.
+## Licensed under the MIT License.
+##
 
 # Script to apply new runtime parameters of booted VM.
 # This script should be run in the VM.