Merge branch 'master' into chore/heal-workspaces-test

files/squid_whitelist/web_whitelist

@@ -92,6 +92,7 @@ kubecost.github.io
 kubernetes.github.io
 kubernetes-sigs.github.io
 lib.stat.cmu.edu
+localstack.github.io
 login.mathworks.com
 login.microsoftonline.com
 maven.restlet.org
@@ -174,4 +175,4 @@ vpodc.org
 yahoo.com
 idp.stage.qdr.org
 stage.qdr.org
-data.qdr.syr.edu
+data.qdr.syr.edu

gen3/bin/api.sh

@@ -68,9 +68,7 @@ gen3_access_token() {
   if [ "$skip_cache" != "true" ]; then
     gen3_access_token_from_cache "$username" && return 0
   fi
-  # Adding a fallback to `poetry run fence-create` to cater to fence containers with amazon linux.
-  g3kubectl exec -c fence $(gen3 pod fence) -- fence-create token-create --scopes openid,user,fence,data,credentials,google_service_account --type access_token --exp ${exp} --username ${username} | tail -1 | gen3_access_token_to_cache "$username" || \
-    g3kubectl exec -c fence $(gen3 pod fence) -- poetry run fence-create token-create --scopes openid,user,fence,data,credentials,google_service_account --type access_token --exp ${exp} --username ${username} | tail -1 | gen3_access_token_to_cache "$username"
+  g3kubectl exec -c fence $(gen3 pod fence) -- fence-create token-create --scopes openid,user,fence,data,credentials,google_service_account --type access_token --exp ${exp} --username ${username} | tail -1 | gen3_access_token_to_cache "$username"
 }
 
 #

gen3/bin/dcf.sh

@@ -218,14 +218,9 @@ create_gs_bucket() {
   echo "Start creating gs bucket ...."
 
   if [[ $public == "controlled" ]]; then
-    # Adding a fallback to `poetry run fence-create` to cater to fence containers with amazon linux.
-    g3kubectl exec -c fence $(get_pod fence) -- fence-create google-bucket-create --unique-name $bucket_name --storage-class MULTI_REGIONAL --public False --project-auth-id $phsid --access-logs-bucket dcf-logs || \
-    g3kubectl exec -c fence $(get_pod fence) -- poetry run fence-create google-bucket-create --unique-name $bucket_name --storage-class MULTI_REGIONAL --public False --project-auth-id $phsid --access-logs-bucket dcf-logs
-
+    g3kubectl exec -c fence $(get_pod fence) -- fence-create google-bucket-create --unique-name $bucket_name --storage-class MULTI_REGIONAL --public False --project-auth-id $phsid --access-logs-bucket dcf-logs
   elif [[ $public == "public" ]]; then
-    # Adding a fallback to `poetry run fence-create` to cater to fence containers with amazon linux.
-    g3kubectl exec -c fence $(get_pod fence) -- fence-create google-bucket-create --unique-name $bucket_name --storage-class MULTI_REGIONAL --public True --access-logs-bucket dcf-logs || \
-    g3kubectl exec -c fence $(get_pod fence) -- poetry run fence-create google-bucket-create --unique-name $bucket_name --storage-class MULTI_REGIONAL --public True --access-logs-bucket dcf-logs
+    g3kubectl exec -c fence $(get_pod fence) -- fence-create google-bucket-create --unique-name $bucket_name --storage-class MULTI_REGIONAL --public True --access-logs-bucket dcf-logs
   else
     echo "Can not create the bucket. $public is not supported"
     exit 1

gen3/bin/kube-setup-apache-guacamole.sh

@@ -11,21 +11,12 @@ export namespace=$(gen3 api namespace)
 new_client() {
   gen3_log_info "kube-setup-apache-guacamole" "creating fence oidc client for Apache Guacamole"
   local fence_client="guacamole"
-  # Adding a fallback to `poetry run fence-create` to cater to fence containers with amazon linux.
-
-  local secrets=$(
-    (g3kubectl exec -c fence $(gen3 pod fence) -- fence-create client-create --client $fence_client --urls https://${hostname}/guac/guacamole/#/ --username guacamole --auto-approve --public --external --allowed-scopes openid profile email user | tail -1) 2>/dev/null || \
-      g3kubectl exec -c fence $(gen3 pod fence) -- poetry run fence-create client-create --client $fence_client --urls https://${hostname}/guac/guacamole/#/ --username guacamole --auto-approve --public --external --allowed-scopes openid profile email user | tail -1
-  )
+  local secrets=$(g3kubectl exec -c fence $(gen3 pod fence) -- fence-create client-create --client $fence_client --urls https://${hostname}/guac/guacamole/#/ --username guacamole --auto-approve --public --external --allowed-scopes openid profile email user | tail -1)
   # secrets looks like ('CLIENT_ID', 'CLIENT_SECRET')
   if [[ ! $secrets =~ (\'(.*)\', None) ]]; then
       # try delete client
-      g3kubectl exec -c fence $(gen3 pod fence) -- fence-create client-delete --client $fence_client > /dev/null 2>&1 || \
-        g3kubectl exec -c fence $(gen3 pod fence) -- poetry run fence-create client-delete --client $fence_client > /dev/null 2>&1
-      secrets=$(
-        (g3kubectl exec -c fence $(gen3 pod fence) -- fence-create client-create --client $fence_client --urls https://${hostname}/guac/guacamole/#/ --username guacamole --auto-approve --public --external --allowed-scopes openid profile email user | tail -1) 2>/dev/null || \
-        g3kubectl exec -c fence $(gen3 pod fence) -- poetry run fence-create client-create --client $fence_client --urls https://${hostname}/guac/guacamole/#/ --username guacamole --auto-approve --public --external --allowed-scopes openid profile email user | tail -1
-      )
+      g3kubectl exec -c fence $(gen3 pod fence) -- fence-create client-delete --client $fence_client > /dev/null 2>&1
+      secrets=$(g3kubectl exec -c fence $(gen3 pod fence) -- fence-create client-create --client $fence_client --urls https://${hostname}/guac/guacamole/#/ --username guacamole --auto-approve --public --external --allowed-scopes openid profile email user | tail -1)
       if [[ ! $secrets =~ (\'(.*)\', None) ]]; then
           gen3_log_err "kube-setup-apache-guacamole" "Failed generating oidc client for guacamole: $secrets"
           return 1

gen3/bin/kube-setup-cedar-wrapper.sh

@@ -5,15 +5,9 @@ create_client_and_secret() {
     local hostname=$(gen3 api hostname)
     local client_name="cedar_ingest_client"
     gen3_log_info "kube-setup-cedar-wrapper" "creating fence ${client_name} for $hostname"
-
-# Adding a fallback to `poetry run fence-create` to cater to fence containers with amazon linux.
     # delete any existing fence cedar clients
-    g3kubectl exec -c fence $(gen3 pod fence) -- fence-create client-delete --client ${client_name} > /dev/null 2>&1 || \
-        g3kubectl exec -c fence $(gen3 pod fence) -- poetry run fence-create client-delete --client ${client_name} > /dev/null 2>&1
-    local secrets=$(
-        (g3kubectl exec -c fence $(gen3 pod fence) -- fence-create client-create --client ${client_name} --grant-types client_credentials | tail -1) 2>/dev/null || \
-            g3kubectl exec -c fence $(gen3 pod fence) -- poetry run fence-create client-create --client ${client_name} --grant-types client_credentials | tail -1
-    )
+    g3kubectl exec -c fence $(gen3 pod fence) -- fence-create client-delete --client ${client_name} > /dev/null 2>&1
+    local secrets=$(g3kubectl exec -c fence $(gen3 pod fence) -- fence-create client-create --client ${client_name} --grant-types client_credentials | tail -1)
     # secrets looks like ('CLIENT_ID', 'CLIENT_SECRET')
     if [[ ! $secrets =~ (\'(.*)\', \'(.*)\') ]]; then
         gen3_log_err "kube-setup-cedar-wrapper" "Failed generating ${client_name}"
@@ -42,10 +36,7 @@ setup_creds() {
     fi
 
     local client_name="cedar_ingest_client"
-    local client_list=$(
-        (g3kubectl exec -c fence $(gen3 pod fence) -- fence-create client-list) 2>/dev/null || \
-            g3kubectl exec -c fence $(gen3 pod fence) -- poetry run fence-create client-list
-    )
+    local client_list=$(g3kubectl exec -c fence $(gen3 pod fence) -- fence-create client-list)
     local client_count=$(echo "$client_list=" | grep -cE "'name':.*'${client_name}'")
     gen3_log_info "CEDAR client count = ${client_count}"
 

gen3/bin/kube-setup-metadata-delete-expired-objects-cronjob.sh

@@ -12,22 +12,12 @@ setup_config() {
   if [[ ! -f "$secretsFolder/config.json" ]]; then
     local hostname=$(gen3 api hostname)
     gen3_log_info "kube-setup-metadata-delete-expired-objects-job" "creating fence oidc client for $hostname"
-    # Adding a fallback to `poetry run fence-create` to cater to fence containers with amazon linux.
-    local secrets=$(
-      (g3kubectl exec -c fence $(gen3 pod fence) -- fence-create client-create --client metadata-delete-expired-objects-job --grant-types client_credentials | tail -1) 2>/dev/null || \
-        g3kubectl exec -c fence $(gen3 pod fence) -- poetry run fence-create client-create --client metadata-delete-expired-objects-job --grant-types client_credentials | tail -1
-      
-    )
+    local secrets=$(g3kubectl exec -c fence $(gen3 pod fence) -- fence-create client-create --client metadata-delete-expired-objects-job --grant-types client_credentials | tail -1)
     # secrets looks like ('CLIENT_ID', 'CLIENT_SECRET')
     if [[ ! $secrets =~ (\'(.*)\', \'(.*)\') ]]; then
         # try delete client
-        g3kubectl exec -c fence $(gen3 pod fence) -- fence-create client-delete --client metadata-delete-expired-objects-job > /dev/null 2>&1 || \
-          g3kubectl exec -c fence $(gen3 pod fence) -- poetry run fence-create client-delete --client metadata-delete-expired-objects-job > /dev/null 2>&1
-        secrets=$(
-          (g3kubectl exec -c fence $(gen3 pod fence) -- fence-create client-create --client metadata-delete-expired-objects-job --grant-types client_credentials | tail -1) 2>/dev/null || \
-            g3kubectl exec -c fence $(gen3 pod fence) -- poetry run fence-create client-create --client metadata-delete-expired-objects-job --grant-types client_credentials | tail -1
-          
-        )
+        g3kubectl exec -c fence $(gen3 pod fence) -- fence-create client-delete --client metadata-delete-expired-objects-job > /dev/null 2>&1
+        secrets=$(g3kubectl exec -c fence $(gen3 pod fence) -- fence-create client-create --client metadata-delete-expired-objects-job --grant-types client_credentials | tail -1)
         if [[ ! $secrets =~ (\'(.*)\', \'(.*)\') ]]; then
             gen3_log_err "kube-setup-metadata-delete-expired-objects-job" "Failed generating oidc client: $secrets"
             return 1

gen3/bin/kube-setup-ohdsi.sh

@@ -11,13 +11,7 @@ export namespace=$(gen3 api namespace)
 new_client() {
   atlas_hostname="atlas.${hostname}"
   gen3_log_info "kube-setup-ohdsi" "creating fence oidc client for $atlas_hostname"
-
-  # Adding a fallback to `poetry run fence-create` to cater to fence containers with amazon linux.
-  local secrets=$(
-    (g3kubectl exec -c fence $(gen3 pod fence) -- fence-create client-create --client atlas --urls https://${atlas_hostname}/WebAPI/user/oauth/callback?client_name=OidcClient --username atlas --allowed-scopes openid profile email user | tail -1) 2>/dev/null || \
-      g3kubectl exec -c fence $(gen3 pod fence) -- poetry run fence-create client-create --client atlas --urls https://${atlas_hostname}/WebAPI/user/oauth/callback?client_name=OidcClient --username atlas --allowed-scopes openid profile email user | tail -1
-
-  )
+  local secrets=$(g3kubectl exec -c fence $(gen3 pod fence) -- fence-create client-create --client atlas --urls https://${atlas_hostname}/WebAPI/user/oauth/callback?client_name=OidcClient --username atlas --allowed-scopes openid profile email user | tail -1)
   # secrets looks like ('CLIENT_ID', 'CLIENT_SECRET')
   if [[ ! $secrets =~ (\'(.*)\', \'(.*)\') ]]; then
     gen3_log_err "kube-setup-ohdsi" "Failed generating oidc client for atlas: $secrets"

gen3/bin/kube-setup-pelicanjob.sh

@@ -28,20 +28,12 @@ if ! g3kubectl describe secret pelicanservice-g3auto | grep config.json > /dev/n
     # setup fence OIDC client with client_credentials grant for access to MDS API
     hostname=$(gen3 api hostname)
     gen3_log_info "kube-setup-sower-jobs" "creating fence oidc client for $hostname"
-    # Adding a fallback to `poetry run fence-create` to cater to fence containers with amazon linux.
-    secrets=$(
-      (g3kubectl exec -c fence $(gen3 pod fence) -- fence-create client-create --client pelican-export-job --grant-types client_credentials | tail -1) 2>/dev/null || \
-        g3kubectl exec -c fence $(gen3 pod fence) -- poetry run fence-create client-create --client pelican-export-job --grant-types client_credentials | tail -1
-    )
+    secrets=$(g3kubectl exec -c fence $(gen3 pod fence) -- fence-create client-create --client pelican-export-job --grant-types client_credentials | tail -1)
     # secrets looks like ('CLIENT_ID', 'CLIENT_SECRET')
     if [[ ! $secrets =~ (\'(.*)\', \'(.*)\') ]]; then
         # try delete client
-        g3kubectl exec -c fence $(gen3 pod fence) -- fence-create client-delete --client pelican-export-job > /dev/null 2>&1 || \
-        g3kubectl exec -c fence $(gen3 pod fence) -- poetry run fence-create client-delete --client pelican-export-job > /dev/null 2>&1
-        secrets=$(
-          (g3kubectl exec -c fence $(gen3 pod fence) -- fence-create client-create --client pelican-export-job --grant-types client_credentials | tail -1) 2>/dev/null || \
-                g3kubectl exec -c fence $(gen3 pod fence) -- poetry run fence-create client-create --client pelican-export-job --grant-types client_credentials | tail -1
-        )
+        g3kubectl exec -c fence $(gen3 pod fence) -- fence-create client-delete --client pelican-export-job > /dev/null 2>&1
+        secrets=$(g3kubectl exec -c fence $(gen3 pod fence) -- fence-create client-create --client pelican-export-job --grant-types client_credentials | tail -1)
         if [[ ! $secrets =~ (\'(.*)\', \'(.*)\') ]]; then
             gen3_log_err "kube-setup-sower-jobs" "Failed generating oidc client: $secrets"
             return 1

gen3/bin/kube-setup-superset.sh

@@ -8,20 +8,12 @@ new_client() {
   local hostname=$(gen3 api hostname)
   superset_hostname="superset.${hostname}"
   gen3_log_info "kube-setup-superset" "creating fence oidc client for $superset_hostname"
-  # Adding a fallback to `poetry run fence-create` to cater to fence containers with amazon linux.
-  local secrets=$(
-    (g3kubectl exec -c fence $(gen3 pod fence) -- fence-create client-create --client superset --urls https://${superset_hostname}/oauth-authorized/fence --username superset | tail -1) 2>/dev/null || \
-        g3kubectl exec -c fence $(gen3 pod fence) -- poetry run fence-create client-create --client superset --urls https://${superset_hostname}/oauth-authorized/fence --username superset | tail -1
-  )
+  local secrets=$(g3kubectl exec -c fence $(gen3 pod fence) -- fence-create client-create --client superset --urls https://${superset_hostname}/oauth-authorized/fence --username superset | tail -1)
   # secrets looks like ('CLIENT_ID', 'CLIENT_SECRET')
   if [[ ! $secrets =~ (\'(.*)\', \'(.*)\') ]]; then
       # try delete client
-      g3kubectl exec -c fence $(gen3 pod fence) -- fence-create client-delete --client superset > /dev/null 2>&1 || \
-        g3kubectl exec -c fence $(gen3 pod fence) -- poetry run fence-create client-delete --client superset > /dev/null 2>&1
-      secrets=$(
-        (g3kubectl exec -c fence $(gen3 pod fence) -- fence-create client-create --client superset --urls https://${superset_hostname}/oauth-authorized/fence --username superset | tail -1) 2>/dev/null || \
-            g3kubectl exec -c fence $(gen3 pod fence) -- poetry run fence-create client-create --client superset --urls https://${superset_hostname}/oauth-authorized/fence --username superset | tail -1
-      )
+      g3kubectl exec -c fence $(gen3 pod fence) -- fence-create client-delete --client superset > /dev/null 2>&1
+      secrets=$(g3kubectl exec -c fence $(gen3 pod fence) -- fence-create client-create --client superset --urls https://${superset_hostname}/oauth-authorized/fence --username superset | tail -1)
       if [[ ! $secrets =~ (\'(.*)\', \'(.*)\') ]]; then
           gen3_log_err "kube-setup-superset" "Failed generating oidc client for superset: $secrets"
           return 1

gen3/bin/kube-setup-wts.sh

@@ -14,20 +14,12 @@ gen3_load "gen3/lib/kube-setup-init"
 new_client() {
   local hostname=$(gen3 api hostname)
   gen3_log_info "kube-setup-wts" "creating fence oidc client for $hostname"
-  # Adding a fallback to `poetry run fence-create` to cater to fence containers with amazon linux.
-  local secrets=$(
-    (g3kubectl exec -c fence $(gen3 pod fence) -- fence-create client-create --client wts --urls "https://${hostname}/wts/oauth2/authorize" --username wts --auto-approve | tail -1) 1>/dev/null || \
-      g3kubectl exec -c fence $(gen3 pod fence) -- poetry run fence-create client-create --client wts --urls "https://${hostname}/wts/oauth2/authorize" --username wts --auto-approve | tail -1
-  )
+  local secrets=$(g3kubectl exec -c fence $(gen3 pod fence) -- fence-create client-create --client wts --urls "https://${hostname}/wts/oauth2/authorize" --username wts --auto-approve | tail -1)
   # secrets looks like ('CLIENT_ID', 'CLIENT_SECRET')
   if [[ ! $secrets =~ (\'(.*)\', \'(.*)\') ]]; then
       # try delete client
-      g3kubectl exec -c fence $(gen3 pod fence) -- fence-create client-delete --client wts > /dev/null 2>&1 || \
-        g3kubectl exec -c fence $(gen3 pod fence) -- poetry run  fence-create client-delete --client wts > /dev/null 2>&1
-      secrets=$(
-        (g3kubectl exec -c fence $(gen3 pod fence) -- fence-create client-create --client wts --urls "https://${hostname}/wts/oauth2/authorize" --username wts --auto-approve | tail -1) 1>/dev/null || \
-              g3kubectl exec -c fence $(gen3 pod fence) -- poetry run fence-create client-create --client wts --urls "https://${hostname}/wts/oauth2/authorize" --username wts --auto-approve | tail -1
-      )
+      g3kubectl exec -c fence $(gen3 pod fence) -- fence-create client-delete --client wts > /dev/null 2>&1
+      secrets=$(g3kubectl exec -c fence $(gen3 pod fence) -- fence-create client-create --client wts --urls "https://${hostname}/wts/oauth2/authorize" --username wts --auto-approve | tail -1)
       if [[ ! $secrets =~ (\'(.*)\', \'(.*)\') ]]; then
           gen3_log_err "kube-setup-wts" "Failed generating oidc client for workspace token service: $secrets"
           return 1

gen3/lib/testData/default/expectedFenceResult.yaml

@@ -219,7 +219,7 @@ spec:
           - "-c"
           - |
             echo "${FENCE_PUBLIC_CONFIG:-""}" > "/var/www/fence/fence-config-public.yaml"
-            poetry run python /var/www/fence/yaml_merge.py /var/www/fence/fence-config-public.yaml /var/www/fence/fence-config-secret.yaml > /var/www/fence/fence-config.yaml || python /var/www/fence/yaml_merge.py /var/www/fence/fence-config-public.yaml /var/www/fence/fence-config-secret.yaml > /var/www/fence/fence-config.yaml
+            python /var/www/fence/yaml_merge.py /var/www/fence/fence-config-public.yaml /var/www/fence/fence-config-secret.yaml > /var/www/fence/fence-config.yaml
             bash /fence/dockerrun.bash && if [[ -f /dockerrun.sh ]]; then /dockerrun.sh; fi
       initContainers:
       - name: fence-init
@@ -256,11 +256,11 @@ spec:
           - "-c"
           - |
             echo "${FENCE_PUBLIC_CONFIG:-""}" > "/var/www/fence/fence-config-public.yaml"
-            poetry run python /var/www/fence/yaml_merge.py /var/www/fence/fence-config-public.yaml /var/www/fence/fence-config-secret.yaml > /var/www/fence/fence-config.yaml || python /var/www/fence/yaml_merge.py /var/www/fence/fence-config-public.yaml /var/www/fence/fence-config-secret.yaml > /var/www/fence/fence-config.yaml
-            if (fence-create migrate --help || poetry run fence-create migrate --help) > /dev/null 2>&1; then
+            python /var/www/fence/yaml_merge.py /var/www/fence/fence-config-public.yaml /var/www/fence/fence-config-secret.yaml > /var/www/fence/fence-config.yaml
+            if fence-create migrate --help > /dev/null 2>&1; then
               if ! grep -E 'ENABLE_DB_MIGRATION"?: *false' /var/www/fence/fence-config.yaml; then
                 echo "Running db migration: fence-create migrate"
-                fence-create migrate || poetry run fence-create migrate
+                fence-create migrate
               else
                 echo "Db migration disabled in fence-config"
               fi