Hotfix/fix pillar http json

changelog/36138.fixed

@@ -0,0 +1 @@
+Add kwargs to handle extra parameters for http.query

salt/pillar/http_json.py

@@ -12,9 +12,13 @@
   ext_pillar:
     - http_json:
         url: http://example.com/api/minion_id
-        ::TODO::
         username: username
         password: password
+        header_dict: None
+        auth: None
+
+You can pass additional parameters, they will be added to the http.query call
+:py:func:`utils.http.query function <salt.utils.http.query>`:
 
 If the with_grains parameter is set, grain keys wrapped in can be provided (wrapped
 in <> brackets) in the url in order to populate pillar data based on the grain value.
@@ -52,17 +56,29 @@ def __virtual__():
     return True
 
 
-def ext_pillar(minion_id, pillar, url, with_grains=False):  # pylint: disable=W0613
+def ext_pillar(
+    minion_id,
+    pillar,
+    url,
+    with_grains=False,
+    header_dict=None,
+    auth=None,
+    username=None,
+    password=None,
+):  # pylint: disable=W0613
     """
     Read pillar data from HTTP response.
 
     :param str url: Url to request.
     :param bool with_grains: Whether to substitute strings in the url with their grain values.
+    :param dict header_dict: Extra headers to send
+    :param auth: special auth if needed
+    :param str username: username for auth
+    :param str pasword: password for auth
 
     :return: A dictionary of the pillar data to add.
     :rtype: dict
     """
-
     url = url.replace("%s", urllib.parse.quote(minion_id))
 
     grain_pattern = r"<(?P<grain_name>.*?)>"
@@ -82,7 +98,15 @@ def ext_pillar(minion_id, pillar, url, with_grains=False):  # pylint: disable=W0
             url = re.sub("<{}>".format(grain_name), grain_value, url)
 
     log.debug("Getting url: %s", url)
-    data = __salt__["http.query"](url=url, decode=True, decode_type="json")
+    data = __salt__["http.query"](
+        url=url,
+        decode=True,
+        decode_type="json",
+        header_dict=header_dict,
+        auth=auth,
+        username=username,
+        password=password,
+    )
 
     if "dict" in data:
         return data["dict"]

salt/pillar/http_yaml.py

@@ -12,9 +12,13 @@
   ext_pillar:
     - http_yaml:
         url: http://example.com/api/minion_id
-        ::TODO::
         username: username
         password: password
+        header_dict: None
+        auth: None
+
+You can pass additional parameters, they will be added to the http.query call
+:py:func:`utils.http.query function <salt.utils.http.query>`:
 
 If the with_grains parameter is set, grain keys wrapped in can be provided (wrapped
 in <> brackets) in the url in order to populate pillar data based on the grain value.
@@ -50,17 +54,30 @@ def __virtual__():
     return True
 
 
-def ext_pillar(minion_id, pillar, url, with_grains=False):  # pylint: disable=W0613
+def ext_pillar(
+    minion_id,
+    pillar,
+    url,
+    with_grains=False,
+    header_dict=None,
+    auth=None,
+    username=None,
+    password=None,
+):  # pylint: disable=W0613
     """
     Read pillar data from HTTP response.
 
     :param str url: Url to request.
     :param bool with_grains: Whether to substitute strings in the url with their grain values.
+    :param dict header_dict: Extra headers to send
+    :param str username: username for auth
+    :param str pasword: password for auth
+    :param auth: special auth if needed
 
     :return: A dictionary of the pillar data to add.
     :rtype: dict
     """
-
+    # As we are dealing with kwargs, clean args that are hardcoded in this function
     url = url.replace("%s", urllib.parse.quote(minion_id))
 
     grain_pattern = r"<(?P<grain_name>.*?)>"
@@ -80,7 +97,15 @@ def ext_pillar(minion_id, pillar, url, with_grains=False):  # pylint: disable=W0
             url = re.sub("<{}>".format(grain_name), grain_value, url)
 
     log.debug("Getting url: %s", url)
-    data = __salt__["http.query"](url=url, decode=True, decode_type="yaml")
+    data = __salt__["http.query"](
+        url=url,
+        decode=True,
+        decode_type="yaml",
+        header_dict=header_dict,
+        auth=auth,
+        username=username,
+        password=password,
+    )
 
     if "dict" in data:
         return data["dict"]

tests/pytests/unit/pillar/test_http_json_pillar.py

@@ -0,0 +1,41 @@
+import pytest
+
+import salt.utils.json
+from salt.modules import http
+from salt.pillar import http_json
+
+
+@pytest.fixture
+def configure_loader_modules():
+    return {
+        http_json: {
+            "__salt__": {
+                "http.query": http.query,
+            },
+        },
+        http: {
+            "__opts__": {},
+        },
+    }
+
+
+@pytest.mark.requires_network
+@pytest.mark.parametrize("backend", ["requests", "tornado", "urllib2"])
+def test_ext_pillar_can_take_http_query_kwargs(backend, httpserver):
+    response = {
+        "dict": {
+            "backend": backend,
+            "pillar_type": "http_json",
+        },
+    }
+    header_dict = {"custom-backend-header": backend}
+
+    # If the headers in header_dict are not in the request, httpserver will return an empty dictionary, so we know it will fail
+    httpserver.expect_request(
+        "/http_json_pillar/{}".format(backend),
+        headers={"custom-backend-header": backend},
+    ).respond_with_data(salt.utils.json.dumps(response), content_type="text/plain")
+    url = httpserver.url_for("/http_json_pillar/{}".format(backend))
+
+    actual = http_json.ext_pillar("test-minion-id", {}, url, header_dict=header_dict)
+    assert actual == response

tests/pytests/unit/pillar/test_http_yaml_pillar.py

@@ -0,0 +1,41 @@
+import pytest
+
+import salt.utils.json
+from salt.modules import http
+from salt.pillar import http_yaml
+
+
+@pytest.fixture
+def configure_loader_modules():
+    return {
+        http_yaml: {
+            "__salt__": {
+                "http.query": http.query,
+            },
+        },
+        http: {
+            "__opts__": {},
+        },
+    }
+
+
+@pytest.mark.requires_network
+@pytest.mark.parametrize("backend", ["requests", "tornado", "urllib2"])
+def test_ext_pillar_can_take_http_query_kwargs(backend, httpserver):
+    response = {
+        "dict": {
+            "backend": backend,
+            "pillar_type": "http_yaml",
+        },
+    }
+    header_dict = {"custom-backend-header": backend}
+
+    # If the headers in header_dict are not in the request, httpserver will return an empty dictionary, so we know it will fail
+    httpserver.expect_request(
+        "/http_yaml_pillar/{}".format(backend),
+        headers={"custom-backend-header": backend},
+    ).respond_with_data(salt.utils.json.dumps(response), content_type="text/plain")
+    url = httpserver.url_for("/http_yaml_pillar/{}".format(backend))
+
+    actual = http_yaml.ext_pillar("test-minion-id", {}, url, header_dict=header_dict)
+    assert actual == response