Add authentication support to URL catalog connector (#2804)

Co-authored-by: Martha Cryan <Martha.Cryan@ibm.com>
elyra-ai · Jun 29, 2022 · a8aceaa · a8aceaa
1 parent 8ab787a
commit a8aceaa
Show file tree

Hide file tree

Showing 3 changed files with 61 additions and 7 deletions.
diff --git a/docs/source/user_guide/pipeline-components.md b/docs/source/user_guide/pipeline-components.md
@@ -394,8 +394,9 @@ Examples (CLI):
 #### URL component catalog
 
 The URL component catalog connector provides access to components that are stored on the web:
-- The specified URL must be retrievable using an anonymous HTTP `GET` request.
-- You can specify one or more URLs.
+- You can specify one or more URL resources.
+- The specified URLs must be retrievable using an HTTP `GET` request.
+- If the resources are secured, provide credentials, such as a user id and password or API key.
 
 Examples (GUI):
  - `https://raw.githubusercontent.com/elyra-ai/examples/main/component-catalog-connectors/kfp-example-components-connector/kfp_examples_connector/resources/filter_text_using_shell_and_grep.yaml`

diff --git a/elyra/metadata/schemas/url-catalog.json b/elyra/metadata/schemas/url-catalog.json
@@ -63,7 +63,29 @@
             "format": "uri"
           },
           "uihints": {
-            "category": "Configuration"
+            "category": "Configuration",
+            "items": {
+              "ui:placeholder": "https://host:port/path/component_file"
+            }
+          }
+        },
+        "auth_id": {
+          "title": "User Id",
+          "description": "User id that has read access for the specified URL resources",
+          "type": "string",
+          "minLength": 1,
+          "uihints": {
+            "category": "Source credentials"
+          }
+        },
+        "auth_password": {
+          "title": "Password",
+          "description": "Password or API key for the specified user id",
+          "type": "string",
+          "minLength": 1,
+          "uihints": {
+            "ui:field": "password",
+            "category": "Source credentials"
           }
         }
       },

diff --git a/elyra/pipeline/catalog_connector.py b/elyra/pipeline/catalog_connector.py
@@ -29,7 +29,8 @@
 
 from deprecation import deprecated
 from jupyter_core.paths import ENV_JUPYTER_PATH
-import requests
+from requests import get
+from requests.auth import HTTPBasicAuth
 from traitlets.config import LoggingConfigurable
 from traitlets.traitlets import default
 from traitlets.traitlets import Integer
@@ -607,6 +608,8 @@ class UrlComponentCatalogConnector(ComponentCatalogConnector):
     Read a singular component definition from a url
     """
 
+    REQUEST_TIMEOUT = 30
+
     def get_catalog_entries(self, catalog_metadata: Dict[str, Any]) -> List[Dict[str, Any]]:
         """
         Returns a list of catalog_entry_data dictionary instances, one per entry in the given catalog.
@@ -633,13 +636,41 @@ def get_entry_data(
             individual catalog entries
         """
         url = catalog_entry_data.get("url")
+
+        # determine whether authentication needs to be performed
+        auth_id = catalog_metadata.get("auth_id")
+        auth_password = catalog_metadata.get("auth_password")
+        if auth_id and auth_password:
+            auth = HTTPBasicAuth(auth_id, auth_password)
+        elif auth_id or auth_password:
+            self.log.error(
+                f"Error. URL catalog connector '{catalog_metadata.get('display_name')}' "
+                "is not configured properly. "
+                "Authentication requires a user id and password or API key."
+            )
+            return None
+        else:
+            auth = None
+
         try:
-            res = requests.get(url)
+            res = get(
+                url,
+                timeout=UrlComponentCatalogConnector.REQUEST_TIMEOUT,
+                allow_redirects=True,
+                auth=auth,
+            )
         except Exception as e:
-            self.log.warning(f"Failed to connect to URL for component: {url}: {e}")
+            self.log.error(
+                f"Error. The URL catalog connector '{catalog_metadata.get('display_name')}' "
+                f"encountered an issue downloading '{url}': {e} "
+            )
         else:
             if res.status_code != HTTPStatus.OK:
-                self.log.warning(f"Invalid location for component: {url} (HTTP code {res.status_code})")
+                self.log.error(
+                    f"Error. The URL catalog connector '{catalog_metadata.get('display_name')}' "
+                    f"encountered an issue downloading '{url}'. "
+                    f"HTTP response code: {res.status_code}"
+                )
             else:
                 return EntryData(definition=res.text)