From 9790e4f1314e0e824ab4ef9d054e913c1b0cfc01 Mon Sep 17 00:00:00 2001 From: Anthony Martinet Date: Wed, 10 Nov 2021 11:07:46 +0100 Subject: [PATCH 1/2] feat(http_pillar): added kwargs to handle extra option for http.query --- changelog/36138.fixed | 1 + salt/pillar/http_json.py | 22 +++++++++++++++++++--- salt/pillar/http_yaml.py | 22 +++++++++++++++++++--- 3 files changed, 39 insertions(+), 6 deletions(-) create mode 100644 changelog/36138.fixed diff --git a/changelog/36138.fixed b/changelog/36138.fixed new file mode 100644 index 000000000000..78b2e97cc44f --- /dev/null +++ b/changelog/36138.fixed @@ -0,0 +1 @@ +Add kwargs to handle extra parameters for http.query diff --git a/salt/pillar/http_json.py b/salt/pillar/http_json.py index 98c8a853232e..a302d13f0a3b 100644 --- a/salt/pillar/http_json.py +++ b/salt/pillar/http_json.py @@ -12,9 +12,15 @@ ext_pillar: - http_json: url: http://example.com/api/minion_id - ::TODO:: username: username password: password + header_dict: None + auth: None + +You can pass additional parameters, they will be added to the http.query call +:py:func:`utils.http.query function `: + +.. autofunction:: salt.utils.http.query If the with_grains parameter is set, grain keys wrapped in can be provided (wrapped in <> brackets) in the url in order to populate pillar data based on the grain value. @@ -52,16 +58,26 @@ def __virtual__(): return True -def ext_pillar(minion_id, pillar, url, with_grains=False): # pylint: disable=W0613 +def ext_pillar( + minion_id, pillar, url, with_grains=False, **kwargs +): # pylint: disable=W0613 """ Read pillar data from HTTP response. :param str url: Url to request. :param bool with_grains: Whether to substitute strings in the url with their grain values. + :param dict header_dict: Extra headers to send + :param str username: username for auth + :param str pasword: password for auth + :param auth: special auth if needed :return: A dictionary of the pillar data to add. :rtype: dict """ + # As we are dealing with kwargs, clean args that are hardcoded in this function + for arg in ["url", "decode", "decode_type"]: + if arg in kwargs: + del kwargs[arg] url = url.replace("%s", urllib.parse.quote(minion_id)) @@ -82,7 +98,7 @@ def ext_pillar(minion_id, pillar, url, with_grains=False): # pylint: disable=W0 url = re.sub("<{}>".format(grain_name), grain_value, url) log.debug("Getting url: %s", url) - data = __salt__["http.query"](url=url, decode=True, decode_type="json") + data = __salt__["http.query"](url=url, decode=True, decode_type="json", **kwargs) if "dict" in data: return data["dict"] diff --git a/salt/pillar/http_yaml.py b/salt/pillar/http_yaml.py index 72a7910382c7..e9dd518cbb95 100644 --- a/salt/pillar/http_yaml.py +++ b/salt/pillar/http_yaml.py @@ -12,9 +12,15 @@ ext_pillar: - http_yaml: url: http://example.com/api/minion_id - ::TODO:: username: username password: password + header_dict: None + auth: None + +You can pass additional parameters, they will be added to the http.query call +:py:func:`utils.http.query function `: + +.. autofunction:: salt.utils.http.query If the with_grains parameter is set, grain keys wrapped in can be provided (wrapped in <> brackets) in the url in order to populate pillar data based on the grain value. @@ -50,16 +56,26 @@ def __virtual__(): return True -def ext_pillar(minion_id, pillar, url, with_grains=False): # pylint: disable=W0613 +def ext_pillar( + minion_id, pillar, url, with_grains=False, **kwargs +): # pylint: disable=W0613 """ Read pillar data from HTTP response. :param str url: Url to request. :param bool with_grains: Whether to substitute strings in the url with their grain values. + :param dict header_dict: Extra headers to send + :param str username: username for auth + :param str pasword: password for auth + :param auth: special auth if needed :return: A dictionary of the pillar data to add. :rtype: dict """ + # As we are dealing with kwargs, clean args that are hardcoded in this function + for arg in ["url", "decode", "decode_type"]: + if arg in kwargs: + del kwargs[arg] url = url.replace("%s", urllib.parse.quote(minion_id)) @@ -80,7 +96,7 @@ def ext_pillar(minion_id, pillar, url, with_grains=False): # pylint: disable=W0 url = re.sub("<{}>".format(grain_name), grain_value, url) log.debug("Getting url: %s", url) - data = __salt__["http.query"](url=url, decode=True, decode_type="yaml") + data = __salt__["http.query"](url=url, decode=True, decode_type="yaml", **kwargs) if "dict" in data: return data["dict"] From 8d1a228c7c10247fa040bfce2aabf3f90d135753 Mon Sep 17 00:00:00 2001 From: Alkivi Date: Mon, 30 May 2022 10:35:24 +0200 Subject: [PATCH 2/2] fix(all) --- .../unit/pillar/test_http_json_pillar.py | 40 +++++++++++++++++++ .../unit/pillar/test_http_yaml_pillar.py | 40 +++++++++++++++++++ 2 files changed, 80 insertions(+) create mode 100644 tests/pytests/unit/pillar/test_http_json_pillar.py create mode 100644 tests/pytests/unit/pillar/test_http_yaml_pillar.py diff --git a/tests/pytests/unit/pillar/test_http_json_pillar.py b/tests/pytests/unit/pillar/test_http_json_pillar.py new file mode 100644 index 000000000000..8ecf7104176f --- /dev/null +++ b/tests/pytests/unit/pillar/test_http_json_pillar.py @@ -0,0 +1,40 @@ +import pytest +import salt.utils.json +from salt.modules import http +from salt.pillar import http_json + + +@pytest.fixture +def configure_loader_modules(): + return { + http_json: { + "__salt__": { + "http.query": http.query, + }, + }, + http: { + "__opts__": {}, + }, + } + + +@pytest.mark.requires_network +@pytest.mark.parametrize("backend", ["requests", "tornado", "urllib2"]) +def test_ext_pillar_can_take_http_query_kwargs(backend, httpserver): + response = { + "dict": { + "backend": backend, + "pillar_type": "http_json", + }, + } + header_dict = {"custom-backend-header": backend} + + # If the headers in header_dict are not in the request, httpserver will return an empty dictionary, so we know it will fail + httpserver.expect_request( + "/http_json_pillar/{}".format(backend), + headers={"custom-backend-header": backend}, + ).respond_with_data(salt.utils.json.dumps(response), content_type="text/plain") + url = httpserver.url_for("/http_json_pillar/{}".format(backend)) + + actual = http_json.ext_pillar("test-minion-id", {}, url, header_dict=header_dict) + assert actual == response diff --git a/tests/pytests/unit/pillar/test_http_yaml_pillar.py b/tests/pytests/unit/pillar/test_http_yaml_pillar.py new file mode 100644 index 000000000000..0e8f01bccf3d --- /dev/null +++ b/tests/pytests/unit/pillar/test_http_yaml_pillar.py @@ -0,0 +1,40 @@ +import pytest +import salt.utils.json +from salt.modules import http +from salt.pillar import http_yaml + + +@pytest.fixture +def configure_loader_modules(): + return { + http_yaml: { + "__salt__": { + "http.query": http.query, + }, + }, + http: { + "__opts__": {}, + }, + } + + +@pytest.mark.requires_network +@pytest.mark.parametrize("backend", ["requests", "tornado", "urllib2"]) +def test_ext_pillar_can_take_http_query_kwargs(backend, httpserver): + response = { + "dict": { + "backend": backend, + "pillar_type": "http_yaml", + }, + } + header_dict = {"custom-backend-header": backend} + + # If the headers in header_dict are not in the request, httpserver will return an empty dictionary, so we know it will fail + httpserver.expect_request( + "/http_yaml_pillar/{}".format(backend), + headers={"custom-backend-header": backend}, + ).respond_with_data(salt.utils.json.dumps(response), content_type="text/plain") + url = httpserver.url_for("/http_yaml_pillar/{}".format(backend)) + + actual = http_yaml.ext_pillar("test-minion-id", {}, url, header_dict=header_dict) + assert actual == response