diff --git a/changelog/57969.added b/changelog/57969.added new file mode 100644 index 000000000000..cc94cda75344 --- /dev/null +++ b/changelog/57969.added @@ -0,0 +1,2 @@ +- Added an execution module for running idem exec modules +- Added a state module for running idem states diff --git a/changelog/57993.added b/changelog/57993.added new file mode 100644 index 000000000000..56cc2af7e76c --- /dev/null +++ b/changelog/57993.added @@ -0,0 +1 @@ +- Added the ability for states to return `sub_state_run`s -- results from external state engines \ No newline at end of file diff --git a/doc/ref/modules/all/index.rst b/doc/ref/modules/all/index.rst index 4b0140a6d822..158f7ae60a78 100644 --- a/doc/ref/modules/all/index.rst +++ b/doc/ref/modules/all/index.rst @@ -194,6 +194,7 @@ execution modules hosts http icinga2 + idem ifttt ilo incron diff --git a/doc/ref/modules/all/salt.modules.idem.rst b/doc/ref/modules/all/salt.modules.idem.rst new file mode 100644 index 000000000000..7c4b40bd82e8 --- /dev/null +++ b/doc/ref/modules/all/salt.modules.idem.rst @@ -0,0 +1,5 @@ +salt.modules.idem module +======================== + +.. automodule:: salt.modules.idem + :members: diff --git a/doc/ref/states/all/index.rst b/doc/ref/states/all/index.rst index f9e4c5551b7f..2664b4ce458b 100644 --- a/doc/ref/states/all/index.rst +++ b/doc/ref/states/all/index.rst @@ -131,6 +131,7 @@ state modules host http icinga2 + idem ifttt incron influxdb08_database diff --git a/doc/ref/states/all/salt.states.idem.rst b/doc/ref/states/all/salt.states.idem.rst new file mode 100644 index 000000000000..92eda4b217ae --- /dev/null +++ b/doc/ref/states/all/salt.states.idem.rst @@ -0,0 +1,5 @@ +salt.states.idem +================ + +.. automodule:: salt.states.idem + :members: diff --git a/doc/ref/states/writing.rst b/doc/ref/states/writing.rst index c28ff61b08ca..e824825153b8 100644 --- a/doc/ref/states/writing.rst +++ b/doc/ref/states/writing.rst @@ -259,6 +259,45 @@ A State Module must return a dict containing the following keys/values: States should not return data which cannot be serialized such as frozensets. +Sub State Runs +-------------- + +Some states can return multiple state runs from an external engine. +State modules that extend tools like Puppet, Chef, Ansible, and idem can run multiple external +states and then return their results individually in the "sub_state_run" portion of their return +as long as their individual state runs are formatted like salt states with low and high data. + +For example, the idem state module can execute multiple idem states +via it's runtime and report the status of all those runs by attaching them to "sub_state_run" in it's state return. +These sub_state_runs will be formatted and printed alongside other salt states. + +Example: + +.. code-block:: python + + state_return = { + "name": None, # The parent state name + "result": None, # The overall status of the external state engine run + "comment": None, # Comments on the overall external state engine run + "changes": {}, # An empty dictionary, each sub state run has it's own changes to report + "sub_state_run": [ + { + "changes": {}, # A dictionary describing the changes made in the external state run + "result": None, # The external state run name + "comment": None, # Comment on the external state run + "duration": None, # Optional, the duration in seconds of the external state run + "start_time": None, # Optional, the timestamp of the external state run's start time + "low": { + "name": None, # The name of the state from the external state run + "state": None, # Name of the external state run + "__id__": None, # ID of the external state run + "fun": None, # The Function name from the external state run + }, + } + ], + } + + Test State ========== diff --git a/salt/modules/idem.py b/salt/modules/idem.py new file mode 100644 index 000000000000..c117954c3812 --- /dev/null +++ b/salt/modules/idem.py @@ -0,0 +1,72 @@ +# -*- coding: utf-8 -*- +# +# Author: Tyler Johnson +# + +""" +Idem Support +============ + +This module provides access to idem execution modules + +.. versionadded:: Magnesium +""" +# Function alias to make sure not to shadow built-in's +__func_alias__ = {"exec_": "exec"} +__virtualname__ = "idem" + + +def __virtual__(): + if "idem.hub" in __utils__: + return __virtualname__ + else: + return False, "idem is not available" + + +def exec_(path, acct_file=None, acct_key=None, acct_profile=None, *args, **kwargs): + """ + Call an idem execution module + + path + The idem path of the idem execution module to run + + acct_file + Path to the acct file used in generating idem ctx parameters. + Defaults to the value in the ACCT_FILE environment variable. + + acct_key + Key used to decrypt the acct file. + Defaults to the value in the ACCT_KEY environment variable. + + acct_profile + Name of the profile to add to idem's ctx.acct parameter. + Defaults to the value in the ACCT_PROFILE environment variable. + + args + Any positional arguments to pass to the idem exec function + + kwargs + Any keyword arguments to pass to the idem exec function + + CLI Example: + + .. code-block:: bash + + salt '*' idem.exec test.ping + + :maturity: new + :depends: acct, pop, pop-config, idem + :platform: all + """ + hub = __utils__["idem.hub"]() + + coro = hub.idem.ex.run( + path, + args, + {k: v for k, v in kwargs.items() if not k.startswith("__")}, + acct_file=acct_file or hub.OPT.acct.acct_file, + acct_key=acct_key or hub.OPT.acct.acct_key, + acct_profile=acct_profile or hub.OPT.acct.acct_profile or "default", + ) + + return hub.pop.Loop.run_until_complete(coro) diff --git a/salt/state.py b/salt/state.py index 10b5d47b98c7..f69e8fbcbc47 100644 --- a/salt/state.py +++ b/salt/state.py @@ -3082,6 +3082,22 @@ def call_chunk(self, low, running, chunks): running[tag] = self.call(low, chunks, running) if tag in running: self.event(running[tag], len(chunks), fire_event=low.get("fire_event")) + + for sub_state_data in running[tag].pop("sub_state_run", ()): + self.__run_num += 1 + sub_tag = _gen_tag(sub_state_data["low"]) + running[sub_tag] = { + "name": sub_state_data["low"]["name"], + "changes": sub_state_data["changes"], + "result": sub_state_data["result"], + "duration": sub_state_data.get("duration"), + "start_time": sub_state_data.get("start_time"), + "comment": sub_state_data.get("comment"), + "__state_ran__": True, + "__run_num__": self.__run_num, + "__sls__": low["__sls__"], + } + return running def call_listen(self, chunks, running): diff --git a/salt/states/idem.py b/salt/states/idem.py new file mode 100644 index 000000000000..3e59b885a140 --- /dev/null +++ b/salt/states/idem.py @@ -0,0 +1,159 @@ +# -*- coding: utf-8 -*- +# +# Author: Tyler Johnson +# + +""" +Idem Support +============ + +This state provides access to idem states + +.. versionadded:: Magnesium +""" +import pathlib +import re + +__virtualname__ = "idem" + + +def __virtual__(): + if "idem.hub" in __utils__: + return __virtualname__ + else: + return False, "idem is not available" + + +def _get_refs(sources, tree): + """ + Determine where the sls sources are + """ + sls_sources = [] + SLSs = [] + if tree: + sls_sources.append("file://{}".format(tree)) + for sls in sources: + path = pathlib.Path(sls) + if path.is_file(): + ref = str(path.stem if path.suffix == ".sls" else path.name) + SLSs.append(ref) + implied = "file://{}".format(path.parent) + if implied not in sls_sources: + sls_sources.append(implied) + else: + SLSs.append(sls) + return sls_sources, SLSs + + +def _get_low_data(low_data): + """ + Get salt-style low data from an idem state name + """ + # state_|-id_|-name_|-function + match = re.match(r"(\w+)_\|-(\w+)\|-(\w+)_\|-(\w+)", low_data) + return { + "state": match.group(1), + "__id__": match.group(2), + "name": match.group(3), + "fun": match.group(4), + } + + +def state( + name, + sls, + acct_file=None, + acct_key=None, + acct_profile=None, + cache_dir=None, + render=None, + runtime=None, + source_dir=None, + test=False, +): + """ + Call an idem state through a salt state + + sls + A list of idem sls files or sources + + acct_file + Path to the acct file used in generating idem ctx parameters. + Defaults to the value in the ACCT_FILE environment variable. + + acct_key + Key used to decrypt the acct file. + Defaults to the value in the ACCT_KEY environment variable. + + acct_profile + Name of the profile to add to idem's ctx.acct parameter + Defaults to the value in the ACCT_PROFILE environment variable. + + cache_dir + The location to use for the cache directory + + render + The render pipe to use, this allows for the language to be specified (jinja|yaml) + + runtime + Select which execution runtime to use (serial|parallel) + + source_dir + The directory containing sls files + + .. code-block:: yaml + + cheese: + idem.state: + - runtime: parallel + - sls: + - idem_state.sls + - sls_source + + :maturity: new + :depends: acct, pop, pop-config, idem + :platform: all + """ + hub = __utils__["idem.hub"]() + + if isinstance(sls, str): + sls = [sls] + + sls_sources, SLSs = _get_refs(sls, source_dir or hub.OPT.idem.tree) + + coro = hub.idem.state.apply( + name=name, + sls_sources=sls_sources, + render=render or hub.OPT.idem.render, + runtime=runtime or hub.OPT.idem.runtime, + subs=["states"], + cache_dir=cache_dir or hub.OPT.idem.cache_dir, + sls=SLSs, + test=test, + acct_file=acct_file or hub.OPT.acct.acct_file, + acct_key=acct_key or hub.OPT.acct.acct_key, + acct_profile=acct_profile or hub.OPT.acct.acct_profile or "default", + ) + hub.pop.Loop.run_until_complete(coro) + + errors = hub.idem.RUNS[name]["errors"] + success = not errors + + running = [] + for idem_name, idem_return in hub.idem.RUNS[name]["running"].items(): + standardized_idem_return = { + "name": idem_return["name"], + "changes": idem_return["changes"], + "result": idem_return["result"], + "comment": idem_return.get("comment"), + "low": _get_low_data(idem_name), + } + running.append(standardized_idem_return) + + return { + "name": name, + "result": success, + "comment": "Ran {} idem states".format(len(running)) if success else errors, + "changes": {}, + "sub_state_run": running, + } diff --git a/salt/utils/decorators/state.py b/salt/utils/decorators/state.py index 752b0b56c839..df25d9ad0ed0 100644 --- a/salt/utils/decorators/state.py +++ b/salt/utils/decorators/state.py @@ -26,26 +26,36 @@ def __init__(self, *policies): else: self.policies.append(getattr(self, pls)) + def _run_policies(self, data): + for pls in self.policies: + try: + data = pls(data) + except Exception as exc: # pylint: disable=broad-except + log.debug( + "An exception occurred in this state: %s", + exc, + exc_info_on_loglevel=logging.DEBUG, + ) + data = { + "result": False, + "name": "later", + "changes": {}, + "comment": "An exception occurred in this state: {0}".format(exc), + } + return data + def __call__(self, func): def _func(*args, **kwargs): result = func(*args, **kwargs) - for pls in self.policies: - try: - result = pls(result) - except Exception as exc: # pylint: disable=broad-except - log.debug( - "An exception occurred in this state: %s", - exc, - exc_info_on_loglevel=logging.DEBUG, - ) - result = { - "result": False, - "name": "later", - "changes": {}, - "comment": "An exception occurred in this state: {0}".format( - exc - ), - } + sub_state_run = None + if isinstance(result, dict): + sub_state_run = result.get("sub_state_run", ()) + result = self._run_policies(result) + if sub_state_run: + result["sub_state_run"] = [ + self._run_policies(sub_state_data) + for sub_state_data in sub_state_run + ] return result return _func @@ -77,6 +87,9 @@ def content_check(self, result): if err_msg: raise SaltException(err_msg) + for sub_state in result.get("sub_state_run", ()): + self.content_check(sub_state) + return result def unify(self, result): diff --git a/salt/utils/idem.py b/salt/utils/idem.py new file mode 100644 index 000000000000..6712113dd282 --- /dev/null +++ b/salt/utils/idem.py @@ -0,0 +1,58 @@ +# -*- coding: utf-8 -*- +""" +Idem Support +============ + +This util provides access to an idem-ready hub + +.. versionadded:: Magnesium +""" +import logging +import sys + +try: + import pop.hub + + HAS_POP = True, None +except ImportError as e: + HAS_POP = False, str(e) + +log = logging.getLogger(__name__) + +__virtualname__ = "idem" + + +def __virtual__(): + if sys.version_info < (3, 6): + return False, "idem only works on python3.6 and later" + if not HAS_POP[0]: + return HAS_POP + return __virtualname__ + + +def hub(): + """ + Create a hub with idem ready to go and completely loaded + """ + if "idem.hub" not in __context__: + log.debug("Creating the POP hub") + hub = pop.hub.Hub() + + log.debug("Initializing the loop") + hub.pop.loop.create() + + log.debug("Loading subs onto hub") + hub.pop.sub.add(dyne_name="acct") + hub.pop.sub.add(dyne_name="config") + # We aren't collecting grains at all but some exec modules depend on the sub being on the hub + hub.pop.sub.add(dyne_name="grains") + hub.pop.sub.add(dyne_name="idem") + hub.pop.sub.add(dyne_name="exec") + hub.pop.sub.add(dyne_name="states") + + log.debug("Reading idem config options") + hub.config.integrate.load(["acct", "idem"], "idem", parse_cli=False, logs=False) + + __context__["idem.hub"] = hub + + return __context__["idem.hub"] diff --git a/tests/integration/modules/test_idem.py b/tests/integration/modules/test_idem.py new file mode 100644 index 000000000000..248187bdfdf6 --- /dev/null +++ b/tests/integration/modules/test_idem.py @@ -0,0 +1,17 @@ +# -*- coding: utf-8 -*- +""" +Integration tests for the idem execution module +""" +from contextlib import contextmanager + +import pytest +import salt.utils.idem as idem +import salt.utils.path + + +@pytest.mark.skipif(not idem.HAS_POP[0], reason=idem.HAS_POP[1]) +@pytest.mark.skipif(not salt.utils.path.which("idem"), reason="idem is not installed") +@contextmanager +def test_exec(salt_call_cli): + ret = salt_call_cli.run("--local", "idem.exec", "test.ping") + assert ret.json is True diff --git a/tests/integration/states/test_idem.py b/tests/integration/states/test_idem.py new file mode 100644 index 000000000000..fab38d22c441 --- /dev/null +++ b/tests/integration/states/test_idem.py @@ -0,0 +1,49 @@ +# -*- coding: utf-8 -*- +""" +Tests for the idem state +""" +import tempfile +from contextlib import contextmanager + +import pytest +import salt.utils.idem as idem +import salt.utils.path + +SLS_SUCCEED_WITHOUT_CHANGES = """ +state_name: + test.succeed_without_changes: + - name: idem_test + - foo: bar +""" + + +@pytest.mark.skipif(not idem.HAS_POP[0], reason=idem.HAS_POP[1]) +@pytest.mark.skipif(not salt.utils.path.which("idem"), reason="idem is not installed") +@contextmanager +def test_state(salt_call_cli): + with tempfile.NamedTemporaryFile(suffix=".sls", delete=True, mode="w+") as fh: + fh.write(SLS_SUCCEED_WITHOUT_CHANGES) + fh.flush() + ret = salt_call_cli.run( + "--local", "state.single", "idem.state", sls=fh.name, name="idem_test" + ) + + parent = ret.json["idem_|-idem_test_|-idem_test_|-state"] + assert parent["result"] is True, parent["comment"] + sub_state_ret = parent["sub_state_run"][0] + assert sub_state_ret["result"] is True + assert sub_state_ret["name"] == "idem_test" + assert "Success!" in sub_state_ret["comment"] + + +def test_bad_state(salt_call_cli): + bad_sls = "non-existant-file.sls" + + ret = salt_call_cli.run( + "--local", "state.single", "idem.state", sls=bad_sls, name="idem_bad_test" + ) + parent = ret.json["idem_|-idem_bad_test_|-idem_bad_test_|-state"] + + assert parent["result"] is False + assert "SLS ref {} did not resolve to a file".format(bad_sls) == parent["comment"] + assert not parent["sub_state_run"] diff --git a/tests/integration/utils/test_idem.py b/tests/integration/utils/test_idem.py new file mode 100644 index 000000000000..761edf1c3486 --- /dev/null +++ b/tests/integration/utils/test_idem.py @@ -0,0 +1,50 @@ +# -*- coding: utf-8 -*- +""" +Test utility methods that the idem module and state share +""" +from contextlib import contextmanager + +import salt.utils.idem as idem +import salt.utils.path +from tests.support.case import TestCase +from tests.support.unit import skipIf + +HAS_IDEM = not salt.utils.path.which("idem") + + +@skipIf(not idem.HAS_POP[0], str(idem.HAS_POP[1])) +@contextmanager +class TestIdem(TestCase): + @classmethod + def setUpClass(cls): + cls.hub = idem.hub() + + def test_loop(self): + assert hasattr(self.hub.pop, "Loop") + + def test_subs(self): + for sub in ("acct", "config", "idem", "exec", "states"): + with self.subTest(sub=sub): + assert hasattr(self.hub, sub) + + @skipIf(not HAS_IDEM, "idem is not installed") + def test_idem_ex(self): + assert hasattr(self.hub.idem, "ex") + + @skipIf(not HAS_IDEM, "idem is not installed") + def test_idem_state_apply(self): + assert hasattr(self.hub.idem.state, "apply") + + @skipIf(not HAS_IDEM, "idem is not installed") + def test_idem_exec(self): + # self.hub.exec.test.ping() causes a pylint error because of "exec" in the namespace + assert getattr(self.hub, "exec").test.ping() + + @skipIf(not HAS_IDEM, "idem is not installed") + def test_idem_state(self): + ret = self.hub.states.test.succeed_without_changes({}, "test_state") + assert ret["result"] is True + + def test_config(self): + assert self.hub.OPT.acct + assert self.hub.OPT.idem diff --git a/tests/unit/test_state.py b/tests/unit/test_state.py index 5233c13a16fd..bab32c24ff25 100644 --- a/tests/unit/test_state.py +++ b/tests/unit/test_state.py @@ -505,6 +505,49 @@ def test_render_requisite_require_in_disabled(self): ] self.assertEqual(run_num, 0) + def test_call_chunk_sub_state_run(self): + """ + Test running a batch of states with an external runner + that returns sub_state_run + """ + low_data = { + "state": "external", + "name": "external_state_name", + "__id__": "do_a_thing", + "__sls__": "external", + "order": 10000, + "fun": "state", + } + mock_call_return = { + "__run_num__": 0, + "sub_state_run": [ + { + "changes": {}, + "result": True, + "comment": "", + "low": { + "name": "external_state_name", + "__id__": "external_state_id", + "state": "external_state", + "fun": "external_function", + }, + } + ], + } + expected_sub_state_tag = "external_state_|-external_state_id_|-external_state_name_|-external_function" + with patch("salt.state.State._gather_pillar") as state_patch: + with patch("salt.state.State.call", return_value=mock_call_return): + minion_opts = self.get_temp_config("minion") + minion_opts["disabled_requisites"] = ["require"] + state_obj = salt.state.State(minion_opts) + ret = state_obj.call_chunk(low_data, {}, {}) + sub_state = ret.get(expected_sub_state_tag) + assert sub_state + self.assertEqual(sub_state["__run_num__"], 1) + self.assertEqual(sub_state["name"], "external_state_name") + self.assertEqual(sub_state["__state_ran__"], True) + self.assertEqual(sub_state["__sls__"], "external") + class HighStateTestCase(TestCase, AdaptedConfigurationTestCaseMixin): def setUp(self): @@ -775,6 +818,138 @@ def test_state_output_unifier_result_converted_to_false(self): assert statedecorators.OutputUnifier("unify")(lambda: data)()["result"] is False +@skipIf(pytest is None, "PyTest is missing") +class SubStateReturnsTestCase(TestCase): + """ + TestCase for code handling state returns. + """ + + def test_sub_state_output_check_changes_is_dict(self): + """ + Test that changes key contains a dictionary. + :return: + """ + data = {"changes": {}, "sub_state_run": [{"changes": []}]} + out = statedecorators.OutputUnifier("content_check")(lambda: data)() + assert "'Changes' should be a dictionary" in out["sub_state_run"][0]["comment"] + assert not out["sub_state_run"][0]["result"] + + def test_sub_state_output_check_return_is_dict(self): + """ + Test for the entire return is a dictionary + :return: + """ + data = {"sub_state_run": [["whatever"]]} + out = statedecorators.OutputUnifier("content_check")(lambda: data)() + assert ( + "Malformed state return. Data must be a dictionary type" + in out["sub_state_run"][0]["comment"] + ) + assert not out["sub_state_run"][0]["result"] + + def test_sub_state_output_check_return_has_nrc(self): + """ + Test for name/result/comment keys are inside the return. + :return: + """ + data = {"sub_state_run": [{"arbitrary": "data", "changes": {}}]} + out = statedecorators.OutputUnifier("content_check")(lambda: data)() + assert ( + " The following keys were not present in the state return: name, result, comment" + in out["sub_state_run"][0]["comment"] + ) + assert not out["sub_state_run"][0]["result"] + + def test_sub_state_output_unifier_comment_is_not_list(self): + """ + Test for output is unified so the comment is converted to a multi-line string + :return: + """ + data = { + "sub_state_run": [ + { + "comment": ["data", "in", "the", "list"], + "changes": {}, + "name": None, + "result": "fantastic!", + } + ] + } + expected = { + "sub_state_run": [ + { + "comment": "data\nin\nthe\nlist", + "changes": {}, + "name": None, + "result": True, + } + ] + } + assert statedecorators.OutputUnifier("unify")(lambda: data)() == expected + + data = { + "sub_state_run": [ + { + "comment": ["data", "in", "the", "list"], + "changes": {}, + "name": None, + "result": None, + } + ] + } + expected = "data\nin\nthe\nlist" + assert ( + statedecorators.OutputUnifier("unify")(lambda: data)()["sub_state_run"][0][ + "comment" + ] + == expected + ) + + def test_sub_state_output_unifier_result_converted_to_true(self): + """ + Test for output is unified so the result is converted to True + :return: + """ + data = { + "sub_state_run": [ + { + "comment": ["data", "in", "the", "list"], + "changes": {}, + "name": None, + "result": "Fantastic", + } + ] + } + assert ( + statedecorators.OutputUnifier("unify")(lambda: data)()["sub_state_run"][0][ + "result" + ] + is True + ) + + def test_sub_state_output_unifier_result_converted_to_false(self): + """ + Test for output is unified so the result is converted to False + :return: + """ + data = { + "sub_state_run": [ + { + "comment": ["data", "in", "the", "list"], + "changes": {}, + "name": None, + "result": "", + } + ] + } + assert ( + statedecorators.OutputUnifier("unify")(lambda: data)()["sub_state_run"][0][ + "result" + ] + is False + ) + + class StateFormatSlotsTestCase(TestCase, AdaptedConfigurationTestCaseMixin): """ TestCase for code handling slots