Fixes and adds configurability to test_spark_sql_s3_with_privileges.py regtest (apache#1060) (apache#23)

* Test fix and ability to pass REG_TEST_TOKEN

* Ability to configure bucket prefix

* Add reg_test env var to docker compose

* PR Comments

* Add path prefix

* Nit Updates

* More updates

* PR Feedback

---------

Co-authored-by: Travis Bowen <122238243+travis-bowen@users.noreply.github.com>

Author: sfc-gh-tbowen

regtests/run.sh

@@ -67,23 +67,26 @@ NUM_SUCCESSES=0
 export AWS_ACCESS_KEY_ID=''
 export AWS_SECRET_ACCESS_KEY=''
 
-if ! output=$(curl -X POST -H "Polaris-Realm: POLARIS" "http://${POLARIS_HOST:-localhost}:8181/api/catalog/v1/oauth/tokens" \
-  -d "grant_type=client_credentials" \
-  -d "client_id=root" \
-  -d "client_secret=secret" \
-  -d "scope=PRINCIPAL_ROLE:ALL"); then
-  logred "Error: Failed to retrieve bearer token"
-  exit 1
-fi
+# Allow bearer token to be provided if desired
+if [[ -z "$REGTEST_ROOT_BEARER_TOKEN" ]]; then
+  if ! output=$(curl -X POST -H "Polaris-Realm: POLARIS" "http://${POLARIS_HOST:-localhost}:8181/api/catalog/v1/oauth/tokens" \
+    -d "grant_type=client_credentials" \
+    -d "client_id=root" \
+    -d "client_secret=secret" \
+    -d "scope=PRINCIPAL_ROLE:ALL"); then
+    logred "Error: Failed to retrieve bearer token"
+    exit 1
+  fi
 
-token=$(echo "$output" | awk -F\" '{print $4}')
+  token=$(echo "$output" | awk -F\" '{print $4}')
 
-if [ "$token" == "unauthorized_client" ]; then
-  logred "Error: Failed to retrieve bearer token"
-  exit 1
-fi
+  if [ "$token" == "unauthorized_client" ]; then
+    logred "Error: Failed to retrieve bearer token"
+    exit 1
+  fi
 
-export REGTEST_ROOT_BEARER_TOKEN=$token
+  export REGTEST_ROOT_BEARER_TOKEN=$token
+fi
 
 echo "Root bearer token: ${REGTEST_ROOT_BEARER_TOKEN}"
 

regtests/t_pyspark/src/conftest.py

@@ -19,6 +19,7 @@
 
 import codecs
 import os
+import uuid
 from typing import List
 
 import pytest
@@ -39,15 +40,33 @@ def polaris_host():
 def polaris_port():
   return int(os.getenv('POLARIS_PORT', '8181'))
 
+@pytest.fixture
+def polaris_path_prefix():
+  """
+  Used to provide a path prefix between the port number and the standard polaris endpoint paths.
+  No leading or trailing /
+  :return:
+  """
+  return os.getenv('POLARIS_PATH_PREFIX', '')
 
 @pytest.fixture
-def polaris_url(polaris_host, polaris_port):
-  return f"http://{polaris_host}:{polaris_port}/api/management/v1"
+def polaris_url_scheme():
+  """
+  The URL Schema - either http or https - no : or trailing /
+  :return:
+  """
+  return os.getenv('POLARIS_URL_SCHEME', 'http')
+
+@pytest.fixture
+def polaris_url(polaris_url_scheme, polaris_host, polaris_port, polaris_path_prefix):
+  polaris_path_prefix = polaris_path_prefix if len(polaris_path_prefix) == 0 else '/' + polaris_path_prefix
+  return f"{polaris_url_scheme}://{polaris_host}:{polaris_port}{polaris_path_prefix}/api/management/v1"
 
 
 @pytest.fixture
-def polaris_catalog_url(polaris_host, polaris_port):
-  return f"http://{polaris_host}:{polaris_port}/api/catalog"
+def polaris_catalog_url(polaris_url_scheme, polaris_host, polaris_port, polaris_path_prefix):
+  polaris_path_prefix = polaris_path_prefix if len(polaris_path_prefix) == 0 else '/' + polaris_path_prefix
+  return f"{polaris_url_scheme}://{polaris_host}:{polaris_port}{polaris_path_prefix}/api/catalog"
 
 @pytest.fixture
 def test_bucket():
@@ -57,6 +76,16 @@ def test_bucket():
 def aws_role_arn():
   return os.getenv('AWS_ROLE_ARN')
 
+@pytest.fixture
+def aws_bucket_base_location_prefix():
+  """
+  :return: Base location prefix for tests, excluding leading and trailing '/'
+    Provides a default if null or empty
+  """
+  default_val = 'polaris_test'
+  bucket_prefix = os.getenv('AWS_BUCKET_BASE_LOCATION_PREFIX', default_val)
+  return default_val if bucket_prefix == '' else bucket_prefix
+
 @pytest.fixture
 def catalog_client(polaris_catalog_url):
   """
@@ -72,13 +101,13 @@ def catalog_client(polaris_catalog_url):
 
 
 @pytest.fixture
-def snowflake_catalog(root_client, catalog_client, test_bucket, aws_role_arn):
+def snowflake_catalog(root_client, catalog_client, test_bucket, aws_role_arn, aws_bucket_base_location_prefix):
   storage_conf = AwsStorageConfigInfo(storage_type="S3",
-                                      allowed_locations=[f"s3://{test_bucket}/polaris_test/"],
+                                      allowed_locations=[f"s3://{test_bucket}/{aws_bucket_base_location_prefix}/"],
                                       role_arn=aws_role_arn)
-  catalog_name = 'snowflake'
+  catalog_name = f'snowflake_{str(uuid.uuid4())[-10:]}'
   catalog = Catalog(name=catalog_name, type='INTERNAL', properties={
-    "default-base-location": f"s3://{test_bucket}/polaris_test/snowflake_catalog",
+    "default-base-location": f"s3://{test_bucket}/{aws_bucket_base_location_prefix}/snowflake_catalog",
     "client.credentials-provider": "software.amazon.awssdk.auth.credentials.SystemPropertyCredentialsProvider"
   },
                     storage_config_info=storage_conf)