Merge pull request #238 from facebookresearch/release-v0.1.8

Release v0.1.8

.github/workflows/ci.yaml

@@ -40,13 +40,13 @@ jobs:
                   chmod +x bazel
                   sudo mv bazel /usr/local/bin/bazel
                   sudo apt install clang-9 patchelf
-                  python -m pip install -r compiler_gym/requirements.txt -r examples/requirements.txt -r leaderboard/requirements.txt -r tests/requirements.txt
+                  python -m pip install -r compiler_gym/requirements.txt -r examples/requirements.txt -r tests/requirements.txt
               if: matrix.os == 'ubuntu-latest'
 
             - name: Install dependencies (macOS)
               run: |
                   brew install bazelisk zlib
-                  python -m pip install -r compiler_gym/requirements.txt -r examples/requirements.txt -r leaderboard/requirements.txt -r tests/requirements.txt
+                  python -m pip install -r compiler_gym/requirements.txt -r examples/requirements.txt -r tests/requirements.txt
               env:
                   LDFLAGS: -L/usr/local/opt/zlib/lib
                   CPPFLAGS: -I/usr/local/opt/zlib/include
@@ -84,13 +84,13 @@ jobs:
                   chmod +x bazel
                   sudo mv bazel /usr/local/bin/bazel
                   sudo apt install clang-9 patchelf
-                  python -m pip install -r compiler_gym/requirements.txt -r examples/requirements.txt -r leaderboard/requirements.txt -r tests/requirements.txt
+                  python -m pip install -r compiler_gym/requirements.txt -r examples/requirements.txt -r tests/requirements.txt
               if: matrix.os == 'ubuntu-latest'
 
             - name: Install dependencies (macos)
               run: |
                   brew install bazelisk zlib
-                  python -m pip install -r compiler_gym/requirements.txt -r examples/requirements.txt -r leaderboard/requirements.txt -r tests/requirements.txt
+                  python -m pip install -r compiler_gym/requirements.txt -r examples/requirements.txt -r tests/requirements.txt
               env:
                   LDFLAGS: -L/usr/local/opt/zlib/lib
                   CPPFLAGS: -I/usr/local/opt/zlib/include
@@ -110,6 +110,21 @@ jobs:
                   CC: clang
                   CXX: clang++
                   BAZEL_BUILD_OPTS: --config=ci
+              if: matrix.os == 'macos-latest'
+
+            - name: Test with coverage
+              run: make install-test-cov
+              env:
+                  CC: clang
+                  CXX: clang++
+                  BAZEL_BUILD_OPTS: --config=ci
+              if: matrix.os == 'ubuntu-latest'
+
+            - name: Upload coverage to Codecov
+              uses: codecov/codecov-action@v1
+              with:
+                  files: ./coverage.xml
+              if: matrix.os == 'ubuntu-latest'
 
             - name: Uninstall
               run: make purge

.gitignore

@@ -1,12 +1,13 @@
+__pycache__
 .DS_Store
 .env
-/*.egg-info
 /.act
 /.clwb
 /.vscode
+/*.egg-info
 /bazel-*
 /build
+/coverage.xml
 /dist
 /node_modules
 /package-lock.json
-__pycache__

BUILD.bazel

@@ -14,8 +14,8 @@ exports_files([
 py_library(
     name = "CompilerGym",
     data = [
-        "//compiler_gym/third_party/cBench:benchmarks_list",
-        "//compiler_gym/third_party/cBench:crc32",
+        "//compiler_gym/third_party/cbench:benchmarks_list",
+        "//compiler_gym/third_party/cbench:crc32",
     ],
     deps = [
         "//compiler_gym",

CHANGELOG.md

@@ -1,3 +1,90 @@
+## Release 0.1.8 (2021-04-30)
+
+This release introduces some significant changes to the way that benchmarks are
+managed, introducing a new dataset API. This enabled us to add support for
+millions of new benchmarks and a more efficient implementation for the LLVM
+environment, but this will require some migrating of old code to the new
+interfaces (see "Migration Checklist" below). Some of the key changes of this
+release are:
+
+- **[Core API change]** We have added a Python
+  [Benchmark](https://facebookresearch.github.io/CompilerGym/compiler_gym/datasets.html#compiler_gym.datasets.Benchmark)
+  class ([#190](https://github.com/facebookresearch/CompilerGym/pull/190)). The
+  `env.benchmark` attribute is now an instance of this class rather than a
+  string ([#222](https://github.com/facebookresearch/CompilerGym/pull/222)).
+- **[Core behavior change]** Environments will no longer select benchmarks
+  randomly. Now `env.reset()` will now always select the last-used benchmark,
+  unless the `benchmark` argument is provided or `env.benchmark` has been set.
+  If no benchmark is specified, a default is used.
+- **[API deprecations]** We have added a new
+  [Dataset](https://facebookresearch.github.io/CompilerGym/compiler_gym/datasets.html#compiler_gym.datasets.Dataset)
+  class hierarchy
+  ([#191](https://github.com/facebookresearch/CompilerGym/pull/191),
+  [#192](https://github.com/facebookresearch/CompilerGym/pull/192)). All
+  datasets are now available without needing to be downloaded first, and a new
+  [Datasets](https://facebookresearch.github.io/CompilerGym/compiler_gym/datasets.html#compiler_gym.datasets.Datasets)
+  class can be used to iterate over them
+  ([#200](https://github.com/facebookresearch/CompilerGym/pull/200)). We have
+  deprecated the old dataset management operations, the
+  `compiler_gym.bin.datasets` script, and removed the `--dataset` and
+  `--ls_benchmark` flags from the command line tools.
+- **[RPC interface change]** The `StartSession` RPC endpoint now accepts a list
+  of initial observations to compute. This removes the need for an immediate
+  call to `Step`, reducing environment reset time by 15-21%
+  ([#189](https://github.com/facebookresearch/CompilerGym/pull/189)).
+- [LLVM] We have added several new datasets of benchmarks, including the Csmith
+  and llvm-stress program generators
+  ([#207](https://github.com/facebookresearch/CompilerGym/pull/207)), a dataset
+  of OpenCL kernels
+  ([#208](https://github.com/facebookresearch/CompilerGym/pull/208)), and a
+  dataset of compilable C functions
+  ([#210](https://github.com/facebookresearch/CompilerGym/pull/210)). See [the
+  docs](https://facebookresearch.github.io/CompilerGym/llvm/index.html#datasets)
+  for an overview.
+- `CompilerEnv` now takes an optional `Logger` instance at construction time for
+  fine-grained control over logging output
+  ([#187](https://github.com/facebookresearch/CompilerGym/pull/187)).
+- [LLVM] The ModuleID and source_filename of LLVM-IR modules are now anonymized
+  to prevent unintentional overfitting to benchmarks by name
+  ([#171](https://github.com/facebookresearch/CompilerGym/pull/171)).
+- [docs] We have added a [Feature
+  Stability](https://facebookresearch.github.io/CompilerGym/about.html#feature-stability)
+  section to the documentation
+  ([#196](https://github.com/facebookresearch/CompilerGym/pull/196)).
+- Numerous bug fixes and improvements.
+
+Please use this checklist when updating code for the previous CompilerGym release:
+
+* Review code that accesses the `env.benchmark` property and update to
+  `env.benchmark.uri` if a string name is required. Setting this attribute by
+  string (`env.benchmark = "benchmark://a-v0/b"`) and comparison to string types
+  (`env.benchmark == "benchmark://a-v0/b"`) still work.
+* Review code that calls `env.reset()` without first setting a benchmark.
+  Previously, calling `env.reset()` would select a random benchmark. Now,
+  `env.reset()` always selects the last used benchmark, or a predetermined
+  default if none is specified.
+* Review code that relies on `env.benchmark` being `None` to select benchmarks
+  randomly. Now, `env.benchmark` is always set to the previously used benchmark,
+  or a predetermined default benchmark if none has been specified. Setting
+  `env.benchmark = None` will raise an error. Select a benchmark randomly by
+  sampling from the `env.datasets.benchmark_uris()` iterator.
+* Remove calls to `env.require_dataset()` and related operations. These are no
+  longer required.
+* Remove accesses to `env.benchmarks`. An iterator over available benchmark URIs
+  is now available at `env.datasets.benchmark_uris()`, but the list of URIs
+  cannot be relied on to be fully enumerable (the LLVM environments have over
+  2^32 URIs).
+* Review code that accesses `env.observation_space` and update to
+  `env.observation_space_spec` where necessary
+  ([#228](https://github.com/facebookresearch/CompilerGym/pull/228)).
+* Update compiler service implementations to support the updated RPC interface
+  by removing the deprecated `GetBenchmarks` RPC endpoint and replacing it with
+  `Dataset` classes. See the [example
+  service](https://github.com/facebookresearch/CompilerGym/tree/development/examples/example_compiler_gym_service)
+  for details.
+* [LLVM] Update references to the `poj104-v0` dataset to `poj104-v1`.
+* [LLVM] Update references to the `cBench-v1` dataset to `cbench-v1`.
+
 ## Release 0.1.7 (2021-04-01)
 
 This release introduces [public
@@ -46,11 +133,11 @@ semantics validation, and improving the datasets. Many thanks to @JD-at-work,
 - Added default reward spaces for `CompilerEnv` that are derived from scalar
   observations (thanks @bwasti!)
 - Added a new Q learning example (thanks @JD-at-work!).
-- *Deprecation:* The next release v0.1.5 will introduce a new datasets API that
-  is easier to use and more flexible. In preparation for this, the `Dataset`
-  class has been renamed to `LegacyDataset`, the following dataset operations
-  have been marked deprecated: `activate()`, `deactivate()`, and `delete()`. The
-  `GetBenchmarks()` RPC interface method has also been marked deprecated..
+- *Deprecation:* The v0.1.8 release will introduce a new datasets API that is
+  easier to use and more flexible. In preparation for this, the `Dataset` class
+  has been renamed to `LegacyDataset`, the following dataset operations have
+  been marked deprecated: `activate()`, `deactivate()`, and `delete()`. The
+  `GetBenchmarks()` RPC interface method has also been marked deprecated.
 - [llvm] Improved semantics validation using LLVM's memory, thread, address, and
   undefined behavior sanitizers.
 - Numerous bug fixes and improvements.

CONTRIBUTING.md

@@ -1,4 +1,16 @@
-# Contributing
+# Contributing  <!-- omit in toc -->
+
+**Table of Contents**
+
+- [How to Contribute](#how-to-contribute)
+- [Pull Requests](#pull-requests)
+- [Leaderboard Submissions](#leaderboard-submissions)
+- [Code Style](#code-style)
+- [Contributor License Agreement ("CLA")](#contributor-license-agreement-cla)
+
+---
+
+## How to Contribute
 
 We want to make contributing to CompilerGym as easy and transparent
 as possible. The most helpful ways to contribute are:
@@ -16,8 +28,10 @@ as possible. The most helpful ways to contribute are:
    * Pull requests. Please see below for details. The easiest way to get stuck
      is to grab an [unassigned "Good first issue"
      ticket](https://github.com/facebookresearch/CompilerGym/issues?q=is%3Aopen+is%3Aissue+no%3Aassignee+label%3A%22Good+first+issue%22)!
-   * Add new features not on the roadmap. Examples could include adding support
-     for new compilers, producing research results using CompilerGym, etc.
+   * Add new features not on [the
+     roadmap](https://facebookresearch.github.io/CompilerGym/about.html#roadmap).
+     Examples could include adding support for new compilers, producing research
+     results using CompilerGym, etc.
 
 
 ## Pull Requests
@@ -32,9 +46,9 @@ We actively welcome your pull requests.
 3. If you've added code that should be tested, add tests.
 4. If you've changed APIs, update the [documentation](/docs/source).
 5. Ensure the `make test` suite passes.
-6. Make sure your code lints (see "Code Style" below).
-7. If you haven't already, complete the Contributor License Agreement
-   ("CLA").
+6. Make sure your code lints (see [Code Style](#code-style) below).
+7. If you haven't already, complete the [Contributor License Agreement
+   ("CLA")](#contributor-license-agreement-cla).
 
 
 ## Leaderboard Submissions
@@ -49,12 +63,13 @@ and file a [Pull Request](#pull-requests). Please include:
 3. A write-up of your approach. You may use the
    [submission template](/leaderboard/SUBMISSION_TEMPLATE.md) as a guide.
 
-We do not require that you submit the source code for your approach. Once you
-submit your pull request we will validate your results CSV files and may ask
-clarifying questions if we feel that those would be useful to improve
-reproducibility. Please [take a look
-here](https://github.com/facebookresearch/CompilerGym/pull/117) for an example
-of a well-formed pull request submission.
+Please make sure to update to the latest CompilerGym release prior to
+submission. We do not require that you submit the source code for your approach,
+though we encourage that you make it publicly available. Once you submit your
+pull request we will validate your results CSV files and may ask clarifying
+questions if we feel that those would be useful to improve reproducibility.
+[Take a look here](https://github.com/facebookresearch/CompilerGym/pull/117) for
+an example of a well-formed pull request submission.
 
 
 ## Code Style
@@ -69,15 +84,17 @@ is simple:
   style](https://google.github.io/styleguide/cppguide.html) with 100
   character line length and `camelCaseFunctionNames()`.
 
-We use [pre-commit](/.pre-commit-config.yaml) to format our code to
-enforce these rules. Before submitting pull requests, please run
-pre-commit to ensure the code is correctly formatted.
+We use [pre-commit](https://pre-commit.com/) to ensure that code is formatted
+prior to committing. Before submitting pull requests, please run pre-commit. See
+the [config file](/.pre-commit-config.yaml) for installation and usage
+instructions.
 
 Other common sense rules we encourage are:
 
 * Prefer descriptive names over short ones.
 * Split complex code into small units.
 * When writing new features, add tests.
+* Make tests deterministic.
 * Prefer easy-to-use code over easy-to-read, and easy-to-read code over
   easy-to-write.
 

Makefile

@@ -34,6 +34,12 @@ Post-installation Tests
         usually not needed for interactive development since `make test` runs
         the same tests without having to install anything.
 
+    make install-test-cov
+        The same as `make install-test`, but with python test coverage
+        reporting. A summary of test coverage is printed at the end of execution
+        and the full details are recorded in a coverage.xml file in the project
+        root directory.
+
     make install-fuzz
         Run the fuzz testing suite against an installed CompilerGym package.
         Fuzz tests are tests that generate their own inputs and run in a loop
@@ -185,7 +191,7 @@ docs/source/contributing.rst: CONTRIBUTING.md
 
 docs/source/installation.rst: README.md
 	echo "..\n  Generated from $<. Do not edit!\n" > $@
-	sed -n '/^## Installation/,$$p' $< | sed -n '/^## Trying/q;p' | $(PANDOC) --from=markdown --to=rst >> $@
+	sed -n '/^## Installation/,$$p' $< | sed -n '/^### Building/q;p' | $(PANDOC) --from=markdown --to=rst >> $@
 
 GENERATED_DOCS := \
 	docs/source/changelog.rst \
@@ -215,20 +221,30 @@ test:
 itest:
 	$(IBAZEL) $(BAZEL_OPTS) test $(BAZEL_TEST_OPTS) //...
 
-install-test-datasets:
-	cd .. && $(PYTHON) -m compiler_gym.bin.datasets --env=llvm-v0 --download=cBench-v1 >/dev/null
 
-install-test: install-test-datasets
+# Since we can't run compiler_gym from the project root we need to jump through
+# some hoops to run pytest "out of tree" by creating an empty directory and
+# symlinking the test directory into it so that pytest can be invoked.
+define run_pytest_suite
 	mkdir -p /tmp/compiler_gym/wheel_tests
-	rm -f /tmp/compiler_gym/wheel_tests/tests
+	rm -f /tmp/compiler_gym/wheel_tests/tests /tmp/compiler_gym/wheel_tests/tox.ini
 	ln -s $(ROOT)/tests /tmp/compiler_gym/wheel_tests
-	cd /tmp/compiler_gym/wheel_tests && pytest tests -n auto -k "not fuzz"
+	ln -s $(ROOT)/tox.ini /tmp/compiler_gym/wheel_tests
+	cd /tmp/compiler_gym/wheel_tests && pytest tests $(1) --benchmark-disable -n auto -k "not fuzz"
+endef
+
+install-test:
+	$(call run_pytest_suite,)
+
+install-test-cov:
+	$(call run_pytest_suite,--cov=compiler_gym --cov-report=xml)
+	@mv /tmp/compiler_gym/wheel_tests/coverage.xml .
 
 # The minimum number of seconds to run the fuzz tests in a loop for. Override
 # this at the commandline, e.g. `FUZZ_SECONDS=1800 make fuzz`.
 FUZZ_SECONDS ?= 300
 
-install-fuzz: install-test-datasets
+install-fuzz:
 	mkdir -p /tmp/compiler_gym/wheel_fuzz_tests
 	rm -f /tmp/compiler_gym/wheel_fuzz_tests/tests
 	ln -s $(ROOT)/tests /tmp/compiler_gym/wheel_fuzz_tests