From a54d4e6b9b4cfe1d70a27e669cc0e9fa23ba9dcc Mon Sep 17 00:00:00 2001 From: David Murphy < dmurphy@saltstack.com> Date: Thu, 4 Jan 2024 15:11:54 -0700 Subject: [PATCH] Added code to handle masterless sync of salt-minion, tests work-in-progress --- salt/fileclient.py | 7 +- salt/minion.py | 1 + salt/utils/extmods.py | 6 +- .../pytests/integration/cli/test_salt_call.py | 131 ++++++++++++++++++ 4 files changed, 142 insertions(+), 3 deletions(-) diff --git a/salt/fileclient.py b/salt/fileclient.py index 32c5cd0d948d..4c60d93fcab3 100644 --- a/salt/fileclient.py +++ b/salt/fileclient.py @@ -46,12 +46,15 @@ MAX_FILENAME_LENGTH = 255 -def get_file_client(opts, pillar=False): +def get_file_client(opts, pillar=False, force_local=False): """ Read in the ``file_client`` option and return the correct type of file server """ - client = opts.get("file_client", "remote") + if force_local: + client = "local" + else: + client = opts.get("file_client", "remote") if pillar and client == "local": client = "pillar" diff --git a/salt/minion.py b/salt/minion.py index 95588496c996..04787ad93798 100644 --- a/salt/minion.py +++ b/salt/minion.py @@ -938,6 +938,7 @@ def __init__(self, opts, context=None): # Late setup of the opts grains, so we can log from the grains module import salt.loader + _sync_grains(opts) opts["grains"] = salt.loader.grains(opts) super().__init__(opts) diff --git a/salt/utils/extmods.py b/salt/utils/extmods.py index f1b8a8264483..e46ce13611cf 100644 --- a/salt/utils/extmods.py +++ b/salt/utils/extmods.py @@ -39,6 +39,7 @@ def sync( saltenv=None, extmod_whitelist=None, extmod_blacklist=None, + force_local=False, ): """ Sync custom modules into the extension_modules directory @@ -82,7 +83,10 @@ def sync( "Cannot create cache module directory %s. Check permissions.", mod_dir, ) - with salt.fileclient.get_file_client(opts) as fileclient: + ## DGM with salt.fileclient.get_file_client(opts) as fileclient: + with salt.fileclient.get_file_client( + opts, pillar=False, force_local=force_local + ) as fileclient: for sub_env in saltenv: log.info("Syncing %s for environment '%s'", form, sub_env) cache = [] diff --git a/tests/pytests/integration/cli/test_salt_call.py b/tests/pytests/integration/cli/test_salt_call.py index 1d770c0ffbed..ce3b4ad5d0b8 100644 --- a/tests/pytests/integration/cli/test_salt_call.py +++ b/tests/pytests/integration/cli/test_salt_call.py @@ -429,3 +429,134 @@ def test_local_salt_call_no_function_no_retcode(salt_call_cli): assert "test" in ret.data assert ret.data["test"] == "'test' is not available." assert "test.echo" in ret.data + + +def test_state_highstate_custom_grains_masterless_mode( + salt_master, salt_minion_factory +): + """ + This test ensure that custom grains in salt://_grains are loaded before pillar compilation + to ensure that any use of custom grains in pillar files are available when in masterless mode, + this implies that a sync of grains occurs before loading the regular + /etc/salt/grains or configuration file grains, as well as the usual grains. + Note: cannot use salt_minion and salt_call_cli, since these will be loaded before + the pillar and custom_grains files are written, hence using salt_minion_factory. + """ + pillar_top_sls = """ + base: + '*': + - defaults + """ + + pillar_defaults_sls = """ + mypillar: "{{ grains['custom_grain'] }}" + """ + + salt_top_sls = """ + base: + '*': + - test + """ + + salt_test_sls = """ + "donothing": + test.nop: [] + """ + + salt_custom_grains_py = """ + def main(): + return {'custom_grain': 'test_value'} + """ + + ## DGM TBD need to get masterless mode + assert salt_master.is_running() + with salt_minion_factory.started(): + salt_minion = salt_minion_factory + salt_call_cli = salt_minion_factory.salt_call_cli() + with salt_minion.pillar_tree.base.temp_file( + "top.sls", pillar_top_sls + ), salt_minion.pillar_tree.base.temp_file( + "defaults.sls", pillar_defaults_sls + ), salt_minion.state_tree.base.temp_file( + "top.sls", salt_top_sls + ), salt_minion.state_tree.base.temp_file( + "test.sls", salt_test_sls + ), salt_minion.state_tree.base.temp_file( + "_grains/custom_grain.py", salt_custom_grains_py + ): + ret = salt_call_cli.run("state.highstate") + assert ret.returncode == 0 + ret = salt_call_cli.run("pillar.items") + assert ret.returncode == 0 + assert ret.data + pillar_items = ret.data + assert "mypillar" in pillar_items + assert pillar_items["mypillar"] == "test_value" + + +def test_state_highstate_custom_grains_master_mode(salt_master, salt_minion_factory): + """ + This test ensure that custom grains in salt://_grains are loaded before pillar compilation + to ensure that any use of custom grains in pillar files are unaffected by changes for + sync custom grains for the salt-minion in masterless mode, + """ + pillar_top_sls = """ + base: + '*': + - defaults + """ + + pillar_defaults_sls = """ + mypillar: "{{ grains['custom_grain'] }}" + """ + + salt_top_sls = """ + base: + '*': + - test + """ + + salt_test_sls = """ + "donothing": + test.nop: [] + """ + + salt_custom_grains_py = """ + def main(): + return {'custom_grain': 'test_value'} + """ + + ## DGM TBD need to ensure master mode + assert salt_master.is_running() + with salt_minion_factory.started(): + salt_minion = salt_minion_factory + salt_call_cli = salt_minion_factory.salt_call_cli() + with salt_minion.pillar_tree.base.temp_file( + "top.sls", pillar_top_sls + ), salt_minion.pillar_tree.base.temp_file( + "defaults.sls", pillar_defaults_sls + ), salt_minion.state_tree.base.temp_file( + "top.sls", salt_top_sls + ), salt_minion.state_tree.base.temp_file( + "test.sls", salt_test_sls + ), salt_minion.state_tree.base.temp_file( + "_grains/custom_grain.py", salt_custom_grains_py + ): + ret = salt_call_cli.run("state.highstate") + assert ret.returncode == 0 + ret = salt_call_cli.run("pillar.items") + assert ret.returncode == 0 + assert ret.data + pillar_items = ret.data + assert "mypillar" not in pillar_items + + +def test_salt_call_versions(salt_call_cli, caplog): + """ + Call test.versions without '--local' to test grains + are sync'd without any missing keys in opts + """ + with caplog.at_level(logging.DEBUG): + ret = salt_call_cli.run("test.versions") + assert ret.returncode == 0 + assert "Failed to sync grains module: 'master_uri'" not in caplog.messages