From c325d4cb9ac050301fcabcc28f2acd9e5136424d Mon Sep 17 00:00:00 2001 From: Tyler Johnson Date: Fri, 24 Jul 2020 05:11:44 -0600 Subject: [PATCH] Switched to new sub_state return method --- changelog/57969.added | 2 + changelog/57993.added | 2 +- doc/ref/modules/all/index.rst | 1 + doc/ref/modules/all/salt.modules.idem.rst | 5 + doc/ref/states/all/index.rst | 1 + doc/ref/states/all/salt.states.idem.rst | 5 + salt/modules/idem.py | 72 ++++++++++ salt/states/idem.py | 159 ++++++++++++++++++++++ salt/utils/idem.py | 58 ++++++++ tests/integration/modules/test_idem.py | 17 +++ tests/integration/states/test_idem.py | 49 +++++++ tests/integration/utils/test_idem.py | 50 +++++++ 12 files changed, 420 insertions(+), 1 deletion(-) create mode 100644 changelog/57969.added create mode 100644 doc/ref/modules/all/salt.modules.idem.rst create mode 100644 doc/ref/states/all/salt.states.idem.rst create mode 100644 salt/modules/idem.py create mode 100644 salt/states/idem.py create mode 100644 salt/utils/idem.py create mode 100644 tests/integration/modules/test_idem.py create mode 100644 tests/integration/states/test_idem.py create mode 100644 tests/integration/utils/test_idem.py diff --git a/changelog/57969.added b/changelog/57969.added new file mode 100644 index 000000000000..cc94cda75344 --- /dev/null +++ b/changelog/57969.added @@ -0,0 +1,2 @@ +- Added an execution module for running idem exec modules +- Added a state module for running idem states diff --git a/changelog/57993.added b/changelog/57993.added index bb7cf45281b8..56cc2af7e76c 100644 --- a/changelog/57993.added +++ b/changelog/57993.added @@ -1 +1 @@ -- Added the ability for states to return `sub_state_run`s -- results frome external state engines \ No newline at end of file +- Added the ability for states to return `sub_state_run`s -- results from external state engines \ No newline at end of file diff --git a/doc/ref/modules/all/index.rst b/doc/ref/modules/all/index.rst index 4b0140a6d822..158f7ae60a78 100644 --- a/doc/ref/modules/all/index.rst +++ b/doc/ref/modules/all/index.rst @@ -194,6 +194,7 @@ execution modules hosts http icinga2 + idem ifttt ilo incron diff --git a/doc/ref/modules/all/salt.modules.idem.rst b/doc/ref/modules/all/salt.modules.idem.rst new file mode 100644 index 000000000000..7c4b40bd82e8 --- /dev/null +++ b/doc/ref/modules/all/salt.modules.idem.rst @@ -0,0 +1,5 @@ +salt.modules.idem module +======================== + +.. automodule:: salt.modules.idem + :members: diff --git a/doc/ref/states/all/index.rst b/doc/ref/states/all/index.rst index f9e4c5551b7f..2664b4ce458b 100644 --- a/doc/ref/states/all/index.rst +++ b/doc/ref/states/all/index.rst @@ -131,6 +131,7 @@ state modules host http icinga2 + idem ifttt incron influxdb08_database diff --git a/doc/ref/states/all/salt.states.idem.rst b/doc/ref/states/all/salt.states.idem.rst new file mode 100644 index 000000000000..92eda4b217ae --- /dev/null +++ b/doc/ref/states/all/salt.states.idem.rst @@ -0,0 +1,5 @@ +salt.states.idem +================ + +.. automodule:: salt.states.idem + :members: diff --git a/salt/modules/idem.py b/salt/modules/idem.py new file mode 100644 index 000000000000..c117954c3812 --- /dev/null +++ b/salt/modules/idem.py @@ -0,0 +1,72 @@ +# -*- coding: utf-8 -*- +# +# Author: Tyler Johnson +# + +""" +Idem Support +============ + +This module provides access to idem execution modules + +.. versionadded:: Magnesium +""" +# Function alias to make sure not to shadow built-in's +__func_alias__ = {"exec_": "exec"} +__virtualname__ = "idem" + + +def __virtual__(): + if "idem.hub" in __utils__: + return __virtualname__ + else: + return False, "idem is not available" + + +def exec_(path, acct_file=None, acct_key=None, acct_profile=None, *args, **kwargs): + """ + Call an idem execution module + + path + The idem path of the idem execution module to run + + acct_file + Path to the acct file used in generating idem ctx parameters. + Defaults to the value in the ACCT_FILE environment variable. + + acct_key + Key used to decrypt the acct file. + Defaults to the value in the ACCT_KEY environment variable. + + acct_profile + Name of the profile to add to idem's ctx.acct parameter. + Defaults to the value in the ACCT_PROFILE environment variable. + + args + Any positional arguments to pass to the idem exec function + + kwargs + Any keyword arguments to pass to the idem exec function + + CLI Example: + + .. code-block:: bash + + salt '*' idem.exec test.ping + + :maturity: new + :depends: acct, pop, pop-config, idem + :platform: all + """ + hub = __utils__["idem.hub"]() + + coro = hub.idem.ex.run( + path, + args, + {k: v for k, v in kwargs.items() if not k.startswith("__")}, + acct_file=acct_file or hub.OPT.acct.acct_file, + acct_key=acct_key or hub.OPT.acct.acct_key, + acct_profile=acct_profile or hub.OPT.acct.acct_profile or "default", + ) + + return hub.pop.Loop.run_until_complete(coro) diff --git a/salt/states/idem.py b/salt/states/idem.py new file mode 100644 index 000000000000..3e59b885a140 --- /dev/null +++ b/salt/states/idem.py @@ -0,0 +1,159 @@ +# -*- coding: utf-8 -*- +# +# Author: Tyler Johnson +# + +""" +Idem Support +============ + +This state provides access to idem states + +.. versionadded:: Magnesium +""" +import pathlib +import re + +__virtualname__ = "idem" + + +def __virtual__(): + if "idem.hub" in __utils__: + return __virtualname__ + else: + return False, "idem is not available" + + +def _get_refs(sources, tree): + """ + Determine where the sls sources are + """ + sls_sources = [] + SLSs = [] + if tree: + sls_sources.append("file://{}".format(tree)) + for sls in sources: + path = pathlib.Path(sls) + if path.is_file(): + ref = str(path.stem if path.suffix == ".sls" else path.name) + SLSs.append(ref) + implied = "file://{}".format(path.parent) + if implied not in sls_sources: + sls_sources.append(implied) + else: + SLSs.append(sls) + return sls_sources, SLSs + + +def _get_low_data(low_data): + """ + Get salt-style low data from an idem state name + """ + # state_|-id_|-name_|-function + match = re.match(r"(\w+)_\|-(\w+)\|-(\w+)_\|-(\w+)", low_data) + return { + "state": match.group(1), + "__id__": match.group(2), + "name": match.group(3), + "fun": match.group(4), + } + + +def state( + name, + sls, + acct_file=None, + acct_key=None, + acct_profile=None, + cache_dir=None, + render=None, + runtime=None, + source_dir=None, + test=False, +): + """ + Call an idem state through a salt state + + sls + A list of idem sls files or sources + + acct_file + Path to the acct file used in generating idem ctx parameters. + Defaults to the value in the ACCT_FILE environment variable. + + acct_key + Key used to decrypt the acct file. + Defaults to the value in the ACCT_KEY environment variable. + + acct_profile + Name of the profile to add to idem's ctx.acct parameter + Defaults to the value in the ACCT_PROFILE environment variable. + + cache_dir + The location to use for the cache directory + + render + The render pipe to use, this allows for the language to be specified (jinja|yaml) + + runtime + Select which execution runtime to use (serial|parallel) + + source_dir + The directory containing sls files + + .. code-block:: yaml + + cheese: + idem.state: + - runtime: parallel + - sls: + - idem_state.sls + - sls_source + + :maturity: new + :depends: acct, pop, pop-config, idem + :platform: all + """ + hub = __utils__["idem.hub"]() + + if isinstance(sls, str): + sls = [sls] + + sls_sources, SLSs = _get_refs(sls, source_dir or hub.OPT.idem.tree) + + coro = hub.idem.state.apply( + name=name, + sls_sources=sls_sources, + render=render or hub.OPT.idem.render, + runtime=runtime or hub.OPT.idem.runtime, + subs=["states"], + cache_dir=cache_dir or hub.OPT.idem.cache_dir, + sls=SLSs, + test=test, + acct_file=acct_file or hub.OPT.acct.acct_file, + acct_key=acct_key or hub.OPT.acct.acct_key, + acct_profile=acct_profile or hub.OPT.acct.acct_profile or "default", + ) + hub.pop.Loop.run_until_complete(coro) + + errors = hub.idem.RUNS[name]["errors"] + success = not errors + + running = [] + for idem_name, idem_return in hub.idem.RUNS[name]["running"].items(): + standardized_idem_return = { + "name": idem_return["name"], + "changes": idem_return["changes"], + "result": idem_return["result"], + "comment": idem_return.get("comment"), + "low": _get_low_data(idem_name), + } + running.append(standardized_idem_return) + + return { + "name": name, + "result": success, + "comment": "Ran {} idem states".format(len(running)) if success else errors, + "changes": {}, + "sub_state_run": running, + } diff --git a/salt/utils/idem.py b/salt/utils/idem.py new file mode 100644 index 000000000000..6712113dd282 --- /dev/null +++ b/salt/utils/idem.py @@ -0,0 +1,58 @@ +# -*- coding: utf-8 -*- +""" +Idem Support +============ + +This util provides access to an idem-ready hub + +.. versionadded:: Magnesium +""" +import logging +import sys + +try: + import pop.hub + + HAS_POP = True, None +except ImportError as e: + HAS_POP = False, str(e) + +log = logging.getLogger(__name__) + +__virtualname__ = "idem" + + +def __virtual__(): + if sys.version_info < (3, 6): + return False, "idem only works on python3.6 and later" + if not HAS_POP[0]: + return HAS_POP + return __virtualname__ + + +def hub(): + """ + Create a hub with idem ready to go and completely loaded + """ + if "idem.hub" not in __context__: + log.debug("Creating the POP hub") + hub = pop.hub.Hub() + + log.debug("Initializing the loop") + hub.pop.loop.create() + + log.debug("Loading subs onto hub") + hub.pop.sub.add(dyne_name="acct") + hub.pop.sub.add(dyne_name="config") + # We aren't collecting grains at all but some exec modules depend on the sub being on the hub + hub.pop.sub.add(dyne_name="grains") + hub.pop.sub.add(dyne_name="idem") + hub.pop.sub.add(dyne_name="exec") + hub.pop.sub.add(dyne_name="states") + + log.debug("Reading idem config options") + hub.config.integrate.load(["acct", "idem"], "idem", parse_cli=False, logs=False) + + __context__["idem.hub"] = hub + + return __context__["idem.hub"] diff --git a/tests/integration/modules/test_idem.py b/tests/integration/modules/test_idem.py new file mode 100644 index 000000000000..248187bdfdf6 --- /dev/null +++ b/tests/integration/modules/test_idem.py @@ -0,0 +1,17 @@ +# -*- coding: utf-8 -*- +""" +Integration tests for the idem execution module +""" +from contextlib import contextmanager + +import pytest +import salt.utils.idem as idem +import salt.utils.path + + +@pytest.mark.skipif(not idem.HAS_POP[0], reason=idem.HAS_POP[1]) +@pytest.mark.skipif(not salt.utils.path.which("idem"), reason="idem is not installed") +@contextmanager +def test_exec(salt_call_cli): + ret = salt_call_cli.run("--local", "idem.exec", "test.ping") + assert ret.json is True diff --git a/tests/integration/states/test_idem.py b/tests/integration/states/test_idem.py new file mode 100644 index 000000000000..fab38d22c441 --- /dev/null +++ b/tests/integration/states/test_idem.py @@ -0,0 +1,49 @@ +# -*- coding: utf-8 -*- +""" +Tests for the idem state +""" +import tempfile +from contextlib import contextmanager + +import pytest +import salt.utils.idem as idem +import salt.utils.path + +SLS_SUCCEED_WITHOUT_CHANGES = """ +state_name: + test.succeed_without_changes: + - name: idem_test + - foo: bar +""" + + +@pytest.mark.skipif(not idem.HAS_POP[0], reason=idem.HAS_POP[1]) +@pytest.mark.skipif(not salt.utils.path.which("idem"), reason="idem is not installed") +@contextmanager +def test_state(salt_call_cli): + with tempfile.NamedTemporaryFile(suffix=".sls", delete=True, mode="w+") as fh: + fh.write(SLS_SUCCEED_WITHOUT_CHANGES) + fh.flush() + ret = salt_call_cli.run( + "--local", "state.single", "idem.state", sls=fh.name, name="idem_test" + ) + + parent = ret.json["idem_|-idem_test_|-idem_test_|-state"] + assert parent["result"] is True, parent["comment"] + sub_state_ret = parent["sub_state_run"][0] + assert sub_state_ret["result"] is True + assert sub_state_ret["name"] == "idem_test" + assert "Success!" in sub_state_ret["comment"] + + +def test_bad_state(salt_call_cli): + bad_sls = "non-existant-file.sls" + + ret = salt_call_cli.run( + "--local", "state.single", "idem.state", sls=bad_sls, name="idem_bad_test" + ) + parent = ret.json["idem_|-idem_bad_test_|-idem_bad_test_|-state"] + + assert parent["result"] is False + assert "SLS ref {} did not resolve to a file".format(bad_sls) == parent["comment"] + assert not parent["sub_state_run"] diff --git a/tests/integration/utils/test_idem.py b/tests/integration/utils/test_idem.py new file mode 100644 index 000000000000..761edf1c3486 --- /dev/null +++ b/tests/integration/utils/test_idem.py @@ -0,0 +1,50 @@ +# -*- coding: utf-8 -*- +""" +Test utility methods that the idem module and state share +""" +from contextlib import contextmanager + +import salt.utils.idem as idem +import salt.utils.path +from tests.support.case import TestCase +from tests.support.unit import skipIf + +HAS_IDEM = not salt.utils.path.which("idem") + + +@skipIf(not idem.HAS_POP[0], str(idem.HAS_POP[1])) +@contextmanager +class TestIdem(TestCase): + @classmethod + def setUpClass(cls): + cls.hub = idem.hub() + + def test_loop(self): + assert hasattr(self.hub.pop, "Loop") + + def test_subs(self): + for sub in ("acct", "config", "idem", "exec", "states"): + with self.subTest(sub=sub): + assert hasattr(self.hub, sub) + + @skipIf(not HAS_IDEM, "idem is not installed") + def test_idem_ex(self): + assert hasattr(self.hub.idem, "ex") + + @skipIf(not HAS_IDEM, "idem is not installed") + def test_idem_state_apply(self): + assert hasattr(self.hub.idem.state, "apply") + + @skipIf(not HAS_IDEM, "idem is not installed") + def test_idem_exec(self): + # self.hub.exec.test.ping() causes a pylint error because of "exec" in the namespace + assert getattr(self.hub, "exec").test.ping() + + @skipIf(not HAS_IDEM, "idem is not installed") + def test_idem_state(self): + ret = self.hub.states.test.succeed_without_changes({}, "test_state") + assert ret["result"] is True + + def test_config(self): + assert self.hub.OPT.acct + assert self.hub.OPT.idem