Merge remote-tracking branch 'upstream/master' into feat/s3-delete-files

.github/workflows/buildpipeline.yaml → .github/workflows/ci.yaml

@@ -1,24 +1,27 @@
 name: CI
-
 on:
-    push:
-    pull_request:
-      types: [opened, reopened]
-
+  push:
+  pull_request:
+    types: [opened, reopened]
 concurrency:
   group: '${{ github.workflow }} @ ${{ github.event.pull_request.head.label || github.head_ref || github.ref }}'
   cancel-in-progress: true
-
-
 jobs:
   Security:
     name: Security Pipeline
     uses: uc-cdis/.github/.github/workflows/securitypipeline.yaml@master
     secrets: inherit
 
+  UnitTest:
+    name: Python Unit Test with Postgres
+    uses: uc-cdis/.github/.github/workflows/python_unit_test.yaml@master
+    with:
+       python-version: '3.9'
+       test-script: 'tests/ci_commands_script.sh'
+       run-coveralls: true
   ci:
     name: Build Image and Push
-    # TODO Add this line back once we update to Python 3.9 from 3.6
+    # TODO Uncomment after PXP-9212
     # needs: Security
     uses: uc-cdis/.github/.github/workflows/image_build_push.yaml@master
     secrets:

.secrets.baseline

@@ -115,13 +115,13 @@
     }
   ],
   "results": {
-    ".github/workflows/buildpipeline.yaml": [
+    ".github/workflows/ci.yaml": [
       {
         "type": "Secret Keyword",
-        "filename": ".github/workflows/buildpipeline.yaml",
+        "filename": ".github/workflows/ci.yaml",
         "hashed_secret": "3e26d6750975d678acb8fa35a0f69237881576b0",
         "is_verified": false,
-        "line_number": 17
+        "line_number": 13
       }
     ],
     "deployment/scripts/postgresql/postgresql_init.sql": [
@@ -210,13 +210,22 @@
         "line_number": 137
       }
     ],
+    "fence/resources/storage/storageclient/cleversafe.py": [
+      {
+        "type": "Secret Keyword",
+        "filename": "fence/resources/storage/storageclient/cleversafe.py",
+        "hashed_secret": "7cb6efb98ba5972a9b5090dc2e517fe14d12cb04",
+        "is_verified": false,
+        "line_number": 274
+      }
+    ],
     "fence/utils.py": [
       {
         "type": "Secret Keyword",
         "filename": "fence/utils.py",
         "hashed_secret": "8318df9ecda039deac9868adf1944a29a95c7114",
         "is_verified": false,
-        "line_number": 128
+        "line_number": 129
       }
     ],
     "migrations/versions/a04a70296688_non_unique_client_name.py": [
@@ -259,14 +268,14 @@
         "filename": "tests/conftest.py",
         "hashed_secret": "1348b145fa1a555461c1b790a2f66614781091e9",
         "is_verified": false,
-        "line_number": 1559
+        "line_number": 1561
       },
       {
         "type": "Base64 High Entropy String",
         "filename": "tests/conftest.py",
         "hashed_secret": "227dea087477346785aefd575f91dd13ab86c108",
         "is_verified": false,
-        "line_number": 1582
+        "line_number": 1583
       }
     ],
     "tests/credentials/google/test_credentials.py": [
@@ -385,6 +394,24 @@
         "line_number": 300
       }
     ],
+    "tests/storageclient/storage_client_mock.py": [
+      {
+        "type": "Secret Keyword",
+        "filename": "tests/storageclient/storage_client_mock.py",
+        "hashed_secret": "37bbea9557f9efd1eeadb25dda9ab6514f08fde9",
+        "is_verified": false,
+        "line_number": 158
+      }
+    ],
+    "tests/storageclient/test_cleversafe_api_client.py": [
+      {
+        "type": "Secret Keyword",
+        "filename": "tests/storageclient/test_cleversafe_api_client.py",
+        "hashed_secret": "f683c485d521c2e45830146dd570111770baea29",
+        "is_verified": false,
+        "line_number": 130
+      }
+    ],
     "tests/test-fence-config.yaml": [
       {
         "type": "Basic Auth Credentials",
@@ -395,5 +422,5 @@
       }
     ]
   },
-  "generated_at": "2023-11-16T21:15:57Z"
+  "generated_at": "2024-03-16T00:09:27Z"
 }

bin/fence_create.py

@@ -329,6 +329,9 @@ def parse_arguments():
         help='scopes to include in the token (e.g. "user" or "data")',
     )
     token_create.add_argument("--exp", help="time in seconds until token expiration")
+    token_create.add_argument(
+        "--client_id", help="Client Id, required to generate refresh token"
+    )
 
     force_link_google = subparsers.add_parser("force-link-google")
     force_link_google.add_argument(
@@ -581,6 +584,7 @@ def main():
             username=args.username,
             scopes=args.scopes,
             expires_in=args.exp,
+            client_id=args.client_id,
         )
         token_type = str(args.type).strip().lower()
         if token_type == "access_token" or token_type == "access":

docs/base_user.yaml

@@ -78,14 +78,16 @@ authz:
       - /programs
   - id: open_data_reader
     role_ids:
-      - reader
-      - storage_reader
+      - peregrine_reader
+      - guppy_reader
+      - fence_storage_reader
     resource_paths:
     - /open
   - id: all_programs_reader
     role_ids:
-    - reader
-    - storage_reader
+    - peregrine_reader
+    - guppy_reader
+    - fence_storage_reader
     resource_paths:
     - /programs
   - id: MyFirstProject_submitter
@@ -168,6 +170,24 @@ authz:
         action:
           service: '*'
           method: read-storage
+  - id: peregrine_reader
+    permissions:
+    - id: peregrine_reader
+      action:
+        method: read
+        service: peregrine
+  - id: guppy_reader
+    permissions:
+    - id: guppy_reader
+      action:
+        method: read
+        service: guppy
+  - id: fence_storage_reader
+    permissions:
+    - id: fence_storage_reader
+      action:
+        method: read-storage
+        service: fence
 
 clients:
   wts:

docs/google_architecture.md

@@ -17,7 +17,7 @@ We'll talk about each one of those in-depth here (and even delve into the intern
 
 ### Fence -> cirrus -> Google: A library wrapping Google's API
 
-We have a library that wraps Google's public API called [cirrus](https://github.com/uc-cdis/cirrus). Our design is such that fence does not hit Google's API directly, but goes through cirrus. For all of cirrus's features to work, a very specific setup is required, which is detailed in cirrus's README.
+We have a library that wraps Google's public API called [cirrus](https://github.com/uc-cdis/cirrus). Our design is such that fence does not hit Google's API directly, but goes through gen3cirrus. For all of cirrus's features to work, a very specific setup is required, which is detailed in cirrus's README.
 
 Essentially, cirrus requires a Google Cloud Identity account (for group management) and
 Google Cloud Platform project(s). In order to automate group management in Google Cloud Identity with cirrus, you must go through a manual process of allowing API access and delegating a specific service account from a Google Cloud Platform project to have group management authority. Details can be found in cirrus's README.

fence/__init__.py

@@ -5,7 +5,7 @@
 import flask
 from flask_cors import CORS
 from sqlalchemy.orm import scoped_session
-from flask import _app_ctx_stack, current_app
+from flask import current_app
 from werkzeug.local import LocalProxy
 
 from authutils.oauth2.client import OAuthClient
@@ -364,7 +364,6 @@ def app_config(
     _setup_audit_service_client(app)
     _setup_data_endpoint_and_boto(app)
     _load_keys(app, root_dir)
-    _set_authlib_cfgs(app)
 
     app.prometheus_counters = {}
     if config["ENABLE_PROMETHEUS_METRICS"]:
@@ -407,24 +406,6 @@ def _load_keys(app, root_dir):
     }
 
 
-def _set_authlib_cfgs(app):
-    # authlib OIDC settings
-    # key will need to be added
-    settings = {"OAUTH2_JWT_KEY": keys.default_private_key(app)}
-    app.config.update(settings)
-    config.update(settings)
-
-    # only add the following if not already provided
-    config.setdefault("OAUTH2_JWT_ENABLED", True)
-    config.setdefault("OAUTH2_JWT_ALG", "RS256")
-    config.setdefault("OAUTH2_JWT_ISS", app.config["BASE_URL"])
-    config.setdefault("OAUTH2_PROVIDER_ERROR_URI", "/api/oauth2/errors")
-    app.config.setdefault("OAUTH2_JWT_ENABLED", True)
-    app.config.setdefault("OAUTH2_JWT_ALG", "RS256")
-    app.config.setdefault("OAUTH2_JWT_ISS", app.config["BASE_URL"])
-    app.config.setdefault("OAUTH2_PROVIDER_ERROR_URI", "/api/oauth2/errors")
-
-
 def _setup_oidc_clients(app):
     configured_idps = config.get("OPENID_CONNECT", {})
 
@@ -482,7 +463,10 @@ def _setup_oidc_clients(app):
                 logger=logger,
             )
         elif idp == "fence":
-            app.fence_client = OAuthClient(**settings)
+            # https://docs.authlib.org/en/latest/client/flask.html#flask-client
+            app.fence_client = OAuthClient(app)
+            # https://docs.authlib.org/en/latest/client/frameworks.html
+            app.fence_client.register(**settings)
         else:  # generic OIDC implementation
             client = Oauth2ClientBase(
                 settings=settings,

fence/auth.py

@@ -35,7 +35,10 @@ def get_jwt():
     try:
         bearer, token = header.split(" ")
     except ValueError:
-        raise Unauthorized("authorization header not in expected format")
+        msg = "authorization header not in expected format"
+        logger.debug(f"{msg}. Received header: {header}")
+        logger.error(f"{msg}.")
+        raise Unauthorized(msg)
     if bearer.lower() != "bearer":
         raise Unauthorized("expected bearer token in auth header")
     return token