Example searchpath plugin

examples/configure_hydra/job_name/no_config_file_override.py

@@ -1,11 +1,11 @@
 # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved
 import hydra
-from hydra import HydraConfig
+from hydra.plugins.common.utils import HydraConfig
 
 
 @hydra.main()
 def experiment(_cfg):
-    print(HydraConfig().hydra.job.name)
+    print(HydraConfig.instance().hydra.job.name)
 
 
 if __name__ == "__main__":

examples/configure_hydra/job_name/with_config_file_override.py

@@ -1,11 +1,11 @@
 # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved
 import hydra
-from hydra import HydraConfig
+from hydra.plugins.common.utils import HydraConfig
 
 
 @hydra.main(config_path="config.yaml")
 def experiment(_cfg):
-    print(HydraConfig().hydra.job.name)
+    print(HydraConfig.instance().hydra.job.name)
 
 
 if __name__ == "__main__":

examples/configure_hydra/logging/main.py → examples/configure_hydra/logging/my_app.py

@@ -2,15 +2,17 @@
 import logging
 import sys
 
+from omegaconf import DictConfig
+
 import hydra
 
 log = logging.getLogger(__name__)
 
 
 @hydra.main(config_path="conf/config.yaml")
-def experiment(_cfg):
+def my_app(_cfg: DictConfig) -> None:
     log.info("Info level message")
 
 
 if __name__ == "__main__":
-    sys.exit(experiment())
+    sys.exit(my_app())

examples/tutorial/8_working_directory/my_app.py

@@ -1,11 +1,13 @@
 # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved
 import os
 
+from omegaconf import DictConfig
+
 import hydra
 
 
 @hydra.main()
-def my_app(_cfg):
+def my_app(_cfg: DictConfig) -> None:
     print("Working directory : {}".format(os.getcwd()))
 
 

examples/tutorial/8_working_directory/original_cwd.py

@@ -1,16 +1,17 @@
 # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved
 import os
 
+from omegaconf import DictConfig
+
 import hydra
-from hydra import utils
 
 
 @hydra.main()
-def my_app(_cfg):
-    print("Current working directory  : {}".format(os.getcwd()))
-    print("Original working directory : {}".format(utils.get_original_cwd()))
-    print("to_absolute_path('foo')    : {}".format(utils.to_absolute_path("foo")))
-    print("to_absolute_path('/foo')   : {}".format(utils.to_absolute_path("/foo")))
+def my_app(_cfg: DictConfig) -> None:
+    print("Current working directory : {}".format(os.getcwd()))
+    print("Orig working directory    : {}".format(hydra.utils.get_original_cwd()))
+    print("to_absolute_path('foo')   : {}".format(hydra.utils.to_absolute_path("foo")))
+    print("to_absolute_path('/foo')  : {}".format(hydra.utils.to_absolute_path("/foo")))
 
 
 if __name__ == "__main__":

hydra/_internal/core_plugins/package_config_source.py

@@ -37,26 +37,29 @@ def load_config(self, config_path: str) -> ConfigResult:
             raise ConfigLoadError(f"PackageConfigSource: Config not found: {full_path}")
 
     def exists(self, config_path: str) -> bool:
-        full_path = f"{self.path}/{config_path}"
-        module_name, resource_name = PackageConfigSource._split_module_and_resource(
-            full_path
-        )
-        try:
-            return resource_exists(module_name, resource_name)
-        except ImportError:
-            return False
+        return self.get_type(config_path=config_path) != ObjectType.NOT_FOUND
 
     def get_type(self, config_path: str) -> ObjectType:
         full_path = self.concat(self.path, config_path)
         module_name, resource_name = PackageConfigSource._split_module_and_resource(
             full_path
         )
-        if resource_exists(module_name, resource_name):
-            if resource_isdir(module_name, resource_name):
-                return ObjectType.GROUP
+
+        try:
+            if resource_exists(module_name, resource_name):
+                if resource_isdir(module_name, resource_name):
+                    return ObjectType.GROUP
+                else:
+                    return ObjectType.CONFIG
             else:
-                return ObjectType.CONFIG
-        else:
+                return ObjectType.NOT_FOUND
+        except NotImplementedError:
+            raise NotImplementedError(
+                "Unable to load {}/{}, are you missing an __init__.py?".format(
+                    module_name, resource_name
+                )
+            )
+        except ImportError:
             return ObjectType.NOT_FOUND
 
     def list(self, config_path: str, results_filter: Optional[ObjectType]) -> List[str]:

hydra/_internal/utils.py

@@ -3,7 +3,6 @@
 import inspect
 import os
 import sys
-from argparse import ArgumentParser
 from os.path import dirname, join, normpath, realpath
 from typing import Any, Optional, Sequence, Tuple
 
@@ -149,7 +148,7 @@ def create_config_search_path(search_path_dir: Optional[str]) -> ConfigSearchPat
 
 
 def run_hydra(
-    args_parser: ArgumentParser,
+    args_parser: argparse.ArgumentParser,
     task_function: TaskFunction,
     config_path: Optional[str],
     strict: Optional[bool],
@@ -216,7 +215,7 @@ def _get_exec_command() -> str:
         return executable
 
 
-def get_args_parser() -> ArgumentParser:
+def get_args_parser() -> argparse.ArgumentParser:
     from .. import __version__
 
     parser = argparse.ArgumentParser(add_help=False, description="Hydra")

hydra/core/plugins.py

@@ -2,6 +2,7 @@
 import importlib
 import inspect
 import pkgutil
+import warnings
 from typing import Any, List, Optional, Type
 
 from omegaconf import DictConfig
@@ -97,15 +98,28 @@ def _get_all_subclasses_in(
             for importer, modname, ispkg in pkgutil.walk_packages(
                 path=mdl.__path__, prefix=mdl.__name__ + ".", onerror=lambda x: None
             ):
-                loaded_mod = importer.find_module(modname).load_module(modname)
-                for name, obj in inspect.getmembers(loaded_mod):
-                    if inspect.isclass(obj):
-                        if (
-                            supertype is None
-                            or issubclass(obj, supertype)
-                            and not inspect.isabstract(obj)
-                        ):
-                            ret[obj.__name__] = obj
+                try:
+                    loaded_mod = importer.find_module(modname).load_module(modname)
+                except ImportError as e:
+                    warnings.warn(
+                        message=f"\n"
+                        f"\tError importing '{modname}'.\n"
+                        f"\tPlugin is incompatible with this Hydra version or buggy.\n"
+                        f"\tRecommended to uninstall or upgrade plugin.\n"
+                        f"\t\t{type(e).__name__} : {e}",
+                        category=UserWarning,
+                    )
+                    loaded_mod = None
+
+                if loaded_mod is not None:
+                    for name, obj in inspect.getmembers(loaded_mod):
+                        if inspect.isclass(obj):
+                            if (
+                                supertype is None
+                                or issubclass(obj, supertype)
+                                and not inspect.isabstract(obj)
+                            ):
+                                ret[obj.__name__] = obj
 
         result: List[Type[Plugin]] = []
         for v in ret.values():

news/253.docs

@@ -0,0 +1 @@
+Working examples are provided for all Hydra plugins in [plugins/examples](https://github.com/facebookresearch/hydra/tree/master/plugins/examples)

plugins/examples/example_plugin/setup.py

@@ -4,7 +4,7 @@
 with open("README.md", "r") as fh:
     LONG_DESC = fh.read()
     setup(
-        name="hydra-example",
+        name="hydra-example-plugin",
         version="0.1.0",
         author="Omry Yadan",
         author_email="omry@fb.com",

plugins/examples/example_searchpath_plugin/MANIFEST.in

@@ -0,0 +1,3 @@
+global-exclude *.pyc
+global-exclude __pycache__
+recursive-include *.yaml

plugins/examples/example_searchpath_plugin/README.md

@@ -0,0 +1,10 @@
+# Hydra example SearchPath plugin
+
+This plugin provides an example for how to write a SearchPathPlugin that can manipulate the search path.
+Typical use cases includes:
+ * A framework that wants to allow user code to discover its configurations and be able to compose with them.
+ * A plugin that wants to extend Hydra or another plugin by providing additional configs in existing config groups like `hydra/launcher`.
+ * A plugin that can replace the default configuration of another plugin or of Hydra itself by prepending its configurations before those that it want to replace.
+
+SearchPath plugins are discovered and enabled automatically once they are installed.
+You can use `python foo.py hydra.verbose=true --cfg` to see the search path and the installed plugins.

plugins/examples/example_searchpath_plugin/arbitrary_package/__init__.py

@@ -0,0 +1 @@
+# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved

plugins/examples/example_searchpath_plugin/arbitrary_package/conf/__init__.py

@@ -0,0 +1 @@
+# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved

plugins/examples/example_searchpath_plugin/arbitrary_package/conf/hydra/output/my_default_output_dir.yaml

@@ -0,0 +1,3 @@
+hydra:
+  run:
+    dir: /tmp/${now:%Y-%m-%d}/${now:%H-%M-%S}

plugins/examples/example_searchpath_plugin/hydra_plugins/__init__.py

@@ -0,0 +1,3 @@
+# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved
+# type: ignore
+__path__ = __import__("pkgutil").extend_path(__path__, __name__)

plugins/examples/example_searchpath_plugin/hydra_plugins/example_searchpath_plugin/__init__.py

@@ -0,0 +1,4 @@
+# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved
+from .example_searchpath_plugin import ExampleSearchPathPlugin
+
+__all__ = ["ExampleSearchPathPlugin"]

plugins/examples/example_searchpath_plugin/hydra_plugins/example_searchpath_plugin/example_searchpath_plugin.py

@@ -0,0 +1,16 @@
+# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved
+from hydra.core.config_search_path import ConfigSearchPath
+from hydra.plugins import SearchPathPlugin
+
+
+class ExampleSearchPathPlugin(SearchPathPlugin):
+    def manipulate_search_path(self, search_path: ConfigSearchPath) -> None:
+        # Appends the search path for this plugin to the end of the search path
+        # Note that foobar/conf is outside of the example plugin module.
+        # There is no requirement for it to be packaged with the plugin, it just needs
+        # be available in a package.
+        # Remember to verify the config is packaged properly (build sdist and look inside,
+        # and verify MANIFEST.in is correct).
+        search_path.append(
+            provider="example-searchpath-plugin", path="pkg://arbitrary_package/conf",
+        )

plugins/examples/example_searchpath_plugin/setup.py

@@ -0,0 +1,25 @@
+# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved
+from setuptools import find_packages, setup
+
+with open("README.md", "r") as fh:
+    LONG_DESC = fh.read()
+    setup(
+        name="hydra-example-searchpath-plugin",
+        version="0.1.0",
+        author="Omry Yadan",
+        author_email="omry@fb.com",
+        description="Example Hydra SearchPath plugin",
+        long_description=LONG_DESC,
+        long_description_content_type="text/markdown",
+        url="https://github.com/facebookresearch/hydra/",
+        packages=find_packages(exclude=["tests"]),
+        classifiers=[
+            "License :: OSI Approved :: MIT License",
+            "Programming Language :: Python :: 3.6",
+            "Programming Language :: Python :: 3.7",
+            "Programming Language :: Python :: 3.8",
+            "Operating System :: OS Independent",
+        ],
+        install_requires=["hydra-core>=0.12.0rc0"],
+        include_package_data=True,
+    )

plugins/examples/example_searchpath_plugin/tests/__init__.py

@@ -0,0 +1 @@
+# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved

plugins/examples/example_searchpath_plugin/tests/test_example_search_path_plugin.py

@@ -0,0 +1,28 @@
+# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved
+from hydra.core.global_hydra import GlobalHydra
+from hydra.core.plugins import Plugins
+from hydra.plugins import SearchPathPlugin
+
+# noinspection PyUnresolvedReferences
+from hydra.test_utils.test_utils import (  # noqa: F401
+    TGlobalHydraContext,
+    hydra_global_context,
+)
+from hydra_plugins.example_searchpath_plugin import ExampleSearchPathPlugin
+
+
+def test_discovery() -> None:
+    # Tests that this plugin can be discovered via the plugins subsystem when looking at all Plugins
+    assert ExampleSearchPathPlugin.__name__ in [
+        x.__name__ for x in Plugins.discover(SearchPathPlugin)
+    ]
+
+
+def test_config_installed(
+    hydra_global_context: TGlobalHydraContext,  # noqa: F811
+) -> None:
+    with hydra_global_context():
+        config_loader = GlobalHydra.instance().config_loader()
+        assert "my_default_output_dir" in config_loader.get_group_options(
+            "hydra/output"
+        )

tests/test_completion.py

@@ -195,7 +195,9 @@ def create_files(in_files: List[str]) -> None:
         os.chdir(pwd)
 
 
-@pytest.mark.parametrize("prefix", ["", " ", "\t", "/foo/bar", " /foo/bar/"])  # type: ignore
+@pytest.mark.parametrize(  # type: ignore
+    "prefix", ["", " ", "\t", "/foo/bar", " /foo/bar/"],
+)
 @pytest.mark.parametrize(  # type: ignore
     "app_prefix",
     [

tests/test_tutorials.py

@@ -353,3 +353,33 @@ def test_advanced_ad_hoc_composition(
     ]
     result = subprocess.check_output(cmd)
     assert OmegaConf.create(str(result.decode("utf-8"))) == OmegaConf.create(expected)
+
+
+def test_examples_configure_hydra_job_name_no_config_override(tmpdir: Path) -> None:
+    cmd = [
+        sys.executable,
+        "examples/configure_hydra/job_name/no_config_file_override.py",
+        "hydra.run.dir=" + str(tmpdir),
+    ]
+    result = subprocess.check_output(cmd)
+    assert result.decode("utf-8").rstrip() == "no_config_file_override"
+
+
+def test_examples_configure_hydra_job_name_with_config_override(tmpdir: Path) -> None:
+    cmd = [
+        sys.executable,
+        "examples/configure_hydra/job_name/with_config_file_override.py",
+        "hydra.run.dir=" + str(tmpdir),
+    ]
+    result = subprocess.check_output(cmd)
+    assert result.decode("utf-8").rstrip() == "with_config_file_override"
+
+
+def test_examples_configure_hydra_logging(tmpdir: Path) -> None:
+    cmd = [
+        sys.executable,
+        "examples/configure_hydra/logging/my_app.py",
+        "hydra.run.dir=" + str(tmpdir),
+    ]
+    result = subprocess.check_output(cmd)
+    assert result.decode("utf-8").rstrip() == "[INFO] - Info level message"

website/docs/tutorial/1_simple_cli_app.md

@@ -8,7 +8,7 @@ This is a simple Hydra application that prints your configuration.
 The `my_app` function is a place holder 
 for your code. We will slowly evolve this example to show-case more Hydra features.
 
-The examples in this tutorial are available [here](https://github.com/facebookresearch/hydra/tree/master/examples).
+The examples in this tutorial are available [here](https://github.com/facebookresearch/hydra/tree/master/examples/tutorial).
 
 Python file: `my_app.py`
 ```python

website/versioned_docs/version-0.11/advanced/hydra_compose.md

@@ -7,7 +7,7 @@ sidebar_label: Experimental compose API
 Hydra 0.11.0 introduces a new experimental API for composing configuration via the `hydra.experimental.compose()` function.
 Prior to calling compose(), you have to initialize Hydra: This can be done by using the standard `@hydra.main()` or by calling `hydra.experimental.initialize()`.
 
-Here is an [example Jupyter notebook utilizing this API](https://github.com/facebookresearch/hydra/tree/master/examples/notebook).
+Here is an [example Jupyter notebook utilizing this API](https://github.com/facebookresearch/hydra/tree/0.11_branch/examples/notebook).
 
 ### `hydra.experimental.compose()` example
 ```python

website/versioned_docs/version-0.11/advanced/packaging.md

@@ -5,7 +5,7 @@ sidebar_label: Application packaging
 ---
 
 You can package your Hydra application along with its configuration.
-There is a working example [here](https://github.com/facebookresearch/hydra/tree/master/examples/advanced/hydra_app_example).
+There is a working example [here](https://github.com/facebookresearch/hydra/tree/0.11_branch/examples/advanced/hydra_app_example).
 
 You can run it with: