Rename Dataset tuple to LegacyDataset.

In preparation for introducing a new Dataset class.

Issue #45.

compiler_gym/bin/datasets.py

@@ -139,7 +139,13 @@
 import humanize
 from absl import app, flags
 
-from compiler_gym.datasets.dataset import Dataset, activate, deactivate, delete, require
+from compiler_gym.datasets.dataset import (
+    LegacyDataset,
+    activate,
+    deactivate,
+    delete,
+    require,
+)
 from compiler_gym.util.flags.env_from_flags import env_from_flags
 from compiler_gym.util.tabulate import tabulate
 
@@ -183,7 +189,7 @@ def enumerate_directory(name: str, path: Path):
     for path in path.iterdir():
         if not path.is_file() or not path.name.endswith(".json"):
             continue
-        dataset = Dataset.from_json_file(path)
+        dataset = LegacyDataset.from_json_file(path)
         rows.append(
             (dataset.name, dataset.license, dataset.file_count, dataset.size_bytes)
         )

compiler_gym/datasets/__init__.py

@@ -3,6 +3,12 @@
 # This source code is licensed under the MIT license found in the
 # LICENSE file in the root directory of this source tree.
 """Manage datasets of benchmarks."""
-from compiler_gym.datasets.dataset import Dataset, activate, deactivate, delete, require
+from compiler_gym.datasets.dataset import (
+    LegacyDataset,
+    activate,
+    deactivate,
+    delete,
+    require,
+)
 
-__all__ = ["Dataset", "require", "activate", "deactivate", "delete"]
+__all__ = ["LegacyDataset", "require", "activate", "deactivate", "delete"]

compiler_gym/datasets/dataset.py

@@ -15,7 +15,7 @@
 from compiler_gym.util.download import download
 
 
-class Dataset(NamedTuple):
+class LegacyDataset(NamedTuple):
     """A collection of benchmarks for use by an environment."""
 
     name: str
@@ -48,11 +48,11 @@ class Dataset(NamedTuple):
     """A list of platforms supported by this dataset. Allowed platforms 'macos' and 'linux'."""
 
     @classmethod
-    def from_json_file(cls, path: Path) -> "Dataset":
+    def from_json_file(cls, path: Path) -> "LegacyDataset":
         """Construct a dataset form a JSON metadata file.
 
         :param path: Path of the JSON metadata.
-        :return: A Dataset instance.
+        :return: A LegacyDataset instance.
         """
         try:
             with open(str(path), "rb") as f:
@@ -136,7 +136,7 @@ def deactivate(env, name: str) -> bool:
         return True
 
 
-def require(env, dataset: Union[str, Dataset]) -> bool:
+def require(env, dataset: Union[str, LegacyDataset]) -> bool:
     """Require that the given dataset is available to the environment.
 
     This will download and activate the dataset if it is not already installed.
@@ -151,12 +151,14 @@ def require(env, dataset: Union[str, Dataset]) -> bool:
 
     :param env: The environment that this dataset is required for.
     :param dataset: The name of the dataset to download, the URL of the dataset,
-        or a :class:`Dataset` instance.
+        or a :class:`LegacyDataset` instance.
     :return: :code:`True` if the dataset was downloaded, or :code:`False` if the
         dataset was already available.
     """
 
-    def download_and_unpack_archive(url: str, sha256: Optional[str] = None) -> Dataset:
+    def download_and_unpack_archive(
+        url: str, sha256: Optional[str] = None
+    ) -> LegacyDataset:
         json_files_before = {
             f
             for f in env.inactive_datasets_site_path.iterdir()
@@ -173,9 +175,9 @@ def download_and_unpack_archive(url: str, sha256: Optional[str] = None) -> Datas
         new_json = json_files_after - json_files_before
         if not len(new_json):
             raise OSError(f"Downloaded dataset {url} contains no metadata JSON file")
-        return Dataset.from_json_file(list(new_json)[0])
+        return LegacyDataset.from_json_file(list(new_json)[0])
 
-    def unpack_local_archive(path: Path) -> Dataset:
+    def unpack_local_archive(path: Path) -> LegacyDataset:
         if not path.is_file():
             raise FileNotFoundError(f"File not found: {path}")
         json_files_before = {
@@ -193,12 +195,12 @@ def unpack_local_archive(path: Path) -> Dataset:
         new_json = json_files_after - json_files_before
         if not len(new_json):
             raise OSError(f"Downloaded dataset {url} contains no metadata JSON file")
-        return Dataset.from_json_file(list(new_json)[0])
+        return LegacyDataset.from_json_file(list(new_json)[0])
 
     with fasteners.InterProcessLock(env.datasets_site_path / "LOCK"):
         # Resolve the name and URL of the dataset.
         sha256 = None
-        if isinstance(dataset, Dataset):
+        if isinstance(dataset, LegacyDataset):
             name, url = dataset.name, dataset.url
         elif isinstance(dataset, str):
             # Check if we have already downloaded the dataset.

compiler_gym/envs/compiler_env.py

@@ -19,7 +19,7 @@
 from gym.spaces import Space
 
 from compiler_gym.compiler_env_state import CompilerEnvState
-from compiler_gym.datasets.dataset import Dataset, require
+from compiler_gym.datasets.dataset import LegacyDataset, require
 from compiler_gym.service import (
     CompilerGymServiceConnection,
     ConnectionOpts,
@@ -103,9 +103,9 @@ class CompilerEnv(gym.Env):
         to store benchmarks.
     :vartype datasets_site_path: Optional[Path]
 
-    :ivar available_datasets: A mapping from dataset name to :class:`Dataset`
+    :ivar available_datasets: A mapping from dataset name to :class:`LegacyDataset`
         objects that are available to download.
-    :vartype available_datasets: Dict[str, Dataset]
+    :vartype available_datasets: Dict[str, LegacyDataset]
 
     :ivar observation: A view of the available observation spaces that permits
         on-demand computation of observations.
@@ -196,7 +196,7 @@ def __init__(
         self._service_endpoint: Union[str, Path] = service
         self._connection_settings = connection_settings or ConnectionOpts()
         self.datasets_site_path: Optional[Path] = None
-        self.available_datasets: Dict[str, Dataset] = {}
+        self.available_datasets: Dict[str, LegacyDataset] = {}
 
         # The benchmark that is currently being used, and the benchmark that
         # the user requested. Those do not always correlate, since the user
@@ -818,7 +818,7 @@ def _reward_view_type(self):
         """
         return RewardView
 
-    def require_datasets(self, datasets: List[Union[str, Dataset]]) -> None:
+    def require_datasets(self, datasets: List[Union[str, LegacyDataset]]) -> None:
         """Require that the given datasets are available to the environment.
 
         Example usage:
@@ -834,8 +834,11 @@ def require_datasets(self, datasets: List[Union[str, Dataset]]) -> None:
 
         :param datasets: A list of datasets to require. Each dataset is the name
             of an available dataset, the URL of a dataset to download, or a
-            :class:`Dataset` instance.
+            :class:`LegacyDataset` instance.
+
+        :return: Whether a new dataset was downloaded.
         """
+        self.logger.debug("Requiring datasets: %s", datasets)
         dataset_installed = False
         for dataset in datasets:
             dataset_installed |= require(self, dataset)
@@ -849,15 +852,16 @@ def require_datasets(self, datasets: List[Union[str, Dataset]]) -> None:
                 ),
             )
             self.make_manifest_file()
+        return dataset_installed
 
-    def require_dataset(self, dataset: Union[str, Dataset]) -> None:
+    def require_dataset(self, dataset: Union[str, LegacyDataset]) -> bool:
         """Require that the given dataset is available to the environment.
 
         Alias for
         :meth:`env.require_datasets([dataset]) <compiler_gym.envs.CompilerEnv.require_datasets>`.
 
         :param dataset: The name of the dataset to download, the URL of the dataset, or a
-            :class:`Dataset` instance.
+            :class:`LegacyDataset` instance.
         """
         return self.require_datasets([dataset])
 
@@ -885,7 +889,7 @@ def make_manifest_file(self) -> Path:
                     )
         return manifest_path
 
-    def register_dataset(self, dataset: Dataset) -> bool:
+    def register_dataset(self, dataset: LegacyDataset) -> bool:
         """Register a new dataset.
 
         After registering, the dataset name may be used by
@@ -894,13 +898,13 @@ def register_dataset(self, dataset: Dataset) -> bool:
 
         Example usage:
 
-            >>> my_dataset = Dataset(name="my-dataset-v0", ...)
+            >>> my_dataset = LegacyDataset(name="my-dataset-v0", ...)
             >>> env = gym.make("llvm-v0")
             >>> env.register_dataset(my_dataset)
             >>> env.require_dataset("my-dataset-v0")
             >>> env.benchmark = "my-dataset-v0/1"
 
-        :param dataset: A :class:`Dataset` instance describing the new dataset.
+        :param dataset: A :class:`LegacyDataset` instance describing the new dataset.
         :return: :code:`True` if the dataset was added, else :code:`False`.
         :raises ValueError: If a dataset with this name is already registered.
         """

compiler_gym/envs/llvm/datasets.py

@@ -19,7 +19,7 @@
 
 import fasteners
 
-from compiler_gym.datasets.dataset import Dataset
+from compiler_gym.datasets.dataset import LegacyDataset
 from compiler_gym.util.download import download
 from compiler_gym.util.runfiles_path import cache_path, runfiles_path, site_data_path
 from compiler_gym.util.timer import Timer
@@ -42,7 +42,7 @@
     _COMPILE_ARGS = []
 
 LLVM_DATASETS = [
-    Dataset(
+    LegacyDataset(
         name="blas-v0",
         url="https://dl.fbaipublicfiles.com/compiler_gym/llvm_bitcodes-10.0.0-blas-v0.tar.bz2",
         license="BSD 3-Clause",
@@ -52,7 +52,7 @@
         size_bytes=3969036,
         sha256="e724a8114709f8480adeb9873d48e426e8d9444b00cddce48e342b9f0f2b096d",
     ),
-    Dataset(
+    LegacyDataset(
         name="cBench-v0",
         url="https://dl.fbaipublicfiles.com/compiler_gym/llvm_bitcodes-10.0.0-cBench-v0-macos.tar.bz2",
         license="BSD 3-Clause",
@@ -63,7 +63,7 @@
         sha256="072a730c86144a07bba948c49afe543e4f06351f1cb17f7de77f91d5c1a1b120",
         platforms=["macos"],
     ),
-    Dataset(
+    LegacyDataset(
         name="cBench-v0",
         url="https://dl.fbaipublicfiles.com/compiler_gym/llvm_bitcodes-10.0.0-cBench-v0-linux.tar.bz2",
         license="BSD 3-Clause",
@@ -74,7 +74,7 @@
         sha256="9b5838a90895579aab3b9375e8eeb3ed2ae58e0ad354fec7eb4f8b31ecb4a360",
         platforms=["linux"],
     ),
-    Dataset(
+    LegacyDataset(
         name="github-v0",
         url="https://dl.fbaipublicfiles.com/compiler_gym/llvm_bitcodes-10.0.0-github-v0.tar.bz2",
         license="CC BY 4.0",
@@ -84,7 +84,7 @@
         size_bytes=725974100,
         sha256="880269dd7a5c2508ea222a2e54c318c38c8090eb105c0a87c595e9dd31720764",
     ),
-    Dataset(
+    LegacyDataset(
         name="linux-v0",
         url="https://dl.fbaipublicfiles.com/compiler_gym/llvm_bitcodes-10.0.0-linux-v0.tar.bz2",
         license="GPL-2.0",
@@ -94,7 +94,7 @@
         size_bytes=516031044,
         sha256="a1ae5c376af30ab042c9e54dc432f89ce75f9ebaee953bc19c08aff070f12566",
     ),
-    Dataset(
+    LegacyDataset(
         name="mibench-v0",
         url="https://dl.fbaipublicfiles.com/compiler_gym/llvm_bitcodes-10.0.0-mibench-v0.tar.bz2",
         license="BSD 3-Clause",
@@ -104,7 +104,7 @@
         size_bytes=238480,
         sha256="128c090c40b955b99fdf766da167a5f642018fb35c16a1d082f63be2e977eb13",
     ),
-    Dataset(
+    LegacyDataset(
         name="npb-v0",
         url="https://dl.fbaipublicfiles.com/compiler_gym/llvm_bitcodes-10.0.0-npb-v0.tar.bz2",
         license="NASA Open Source Agreement v1.3",
@@ -114,7 +114,7 @@
         size_bytes=2287444,
         sha256="793ac2e7a4f4ed83709e8a270371e65b724da09eaa0095c52e7f4209f63bb1f2",
     ),
-    Dataset(
+    LegacyDataset(
         name="opencv-v0",
         url="https://dl.fbaipublicfiles.com/compiler_gym/llvm_bitcodes-10.0.0-opencv-v0.tar.bz2",
         license="Apache 2.0",
@@ -124,7 +124,7 @@
         size_bytes=21903008,
         sha256="003df853bd58df93572862ca2f934c7b129db2a3573bcae69a2e59431037205c",
     ),
-    Dataset(
+    LegacyDataset(
         name="poj104-v0",
         url="https://dl.fbaipublicfiles.com/compiler_gym/llvm_bitcodes-10.0.0-poj104-v0.tar.bz2",
         license="BSD 3-Clause",
@@ -134,7 +134,7 @@
         size_bytes=304207752,
         sha256="6254d629887f6b51efc1177788b0ce37339d5f3456fb8784415ed3b8c25cce27",
     ),
-    Dataset(
+    LegacyDataset(
         name="polybench-v0",
         url="https://dl.fbaipublicfiles.com/compiler_gym/llvm_bitcodes-10.0.0-polybench-v0.tar.bz2",
         license="BSD 3-Clause",
@@ -144,7 +144,7 @@
         size_bytes=162624,
         sha256="968087e68470e5b44dc687dae195143000c7478a23d6631b27055bb3bb3116b1",
     ),
-    Dataset(
+    LegacyDataset(
         name="tensorflow-v0",
         url="https://dl.fbaipublicfiles.com/compiler_gym/llvm_bitcodes-10.0.0-tensorflow-v0.tar.bz2",
         license="Apache 2.0",

tests/compiler_env_test.py

@@ -8,14 +8,14 @@
 import gym
 import pytest
 
-from compiler_gym.datasets import Dataset
+from compiler_gym.datasets import LegacyDataset
 from compiler_gym.envs import CompilerEnv
 from tests.test_main import main
 
 pytest_plugins = ["tests.pytest_plugins.llvm"]
 
 
-def make_dataset(**kwargs) -> Dataset:
+def make_dataset(**kwargs) -> LegacyDataset:
     default_kwargs = {
         "name": "test-dataset-v0",
         "url": "https://dl.fbaipublicfiles.com/compiler_gym/llvm_bitcodes-10.0.0-blas-v0.tar.bz2",
@@ -27,7 +27,7 @@ def make_dataset(**kwargs) -> Dataset:
         "sha256": "e724a8114709f8480adeb9873d48e426e8d9444b00cddce48e342b9f0f2b096d",
     }
     default_kwargs.update(kwargs)
-    return Dataset(**default_kwargs)
+    return LegacyDataset(**default_kwargs)
 
 
 def test_register_dataset(env: CompilerEnv):