DRIVERS-1746 Add native support for AWS IAM Roles for service accounts, EKS in particular

.evergreen/auth_aws/activate_venv.sh

@@ -12,7 +12,7 @@ fi
 # create venv on first run
 if [ ! -d authawsvenv ]; then
   venvcreate "$PYTHON_BINARY" authawsvenv
-  pip install --upgrade boto3
+  pip install --upgrade boto3 pyop
 else
   venvactivate authawsvenv
 fi

.evergreen/auth_aws/aws_e2e_web_identity.js

@@ -0,0 +1,94 @@
+/**
+ * Verify the AWS IAM EC2 hosted auth works
+ */
+load("lib/aws_e2e_lib.js");
+
+(function() {
+"use strict";
+
+const ASSUMED_ROLE = "arn:aws:sts::857654397073:assumed-role/webIdentityTestRole/*"
+
+function unAssignInstanceProfile() {
+    const config = readSetupJson();
+
+    const env = {
+        AWS_ACCESS_KEY_ID: config["iam_auth_ec2_instance_account"],
+        AWS_SECRET_ACCESS_KEY: config["iam_auth_ec2_instance_secret_access_key"],
+    };
+    const python_command = getPython3Binary() +
+        " -u lib/aws_unassign_instance_profile.py";
+
+    const ret = runShellCmdWithEnv(python_command, env);
+    if (ret == 2) {
+        print("WARNING: Request limit exceeded for AWS API");
+        return false;
+    }
+
+    assert.eq(ret, 0, "Failed to assign an instance profile to the current machine");
+    return true;
+}
+
+unAssignInstanceProfile();
+
+
+function writeWebTokenFile() {
+    const config = readSetupJson();
+
+    const env = {
+        IDP_ISSUER: config["iam_web_identity_issuer"],
+        IDP_JWKS_URI: config["iam_web_identity_jwks_uri"],
+        IDP_RSA_KEY: config["iam_web_identity_rsa_key"],
+        AWS_WEB_IDENTITY_TOKEN_FILE: config['iam_web_identity_token_file']
+    };
+
+    const python_command = getPython3Binary() +
+        " -u lib/aws_handle_oidc_creds.py token > /dev/null"
+
+    const ret = runShellCmdWithEnv(python_command, env);
+    assert.eq(ret, 0, "Failed to write the web token");
+    return true;
+}
+
+writeWebTokenFile();
+
+
+function getWebIdentityCredentials() {
+    const config = readSetupJson();
+
+    const env = {
+        AWS_WEB_IDENTITY_TOKEN_FILE: config['iam_web_identity_token_file'],
+        AWS_ROLE_ARN: config["iam_auth_assume_web_role_name"]
+    };
+
+    const python_command = getPython3Binary() +
+        ` -u lib/aws_assume_web_role.py > creds.json`;
+
+    const ret = runShellCmdWithEnv(python_command, env);
+    assert.eq(ret, 0, "Failed to assume role on the current machine");
+
+    const result = cat("creds.json");
+    try {
+        return JSON.parse(result);
+    } catch (e) {
+        jsTestLog("Failed to parse: " + result);
+        throw e;
+    }
+}
+
+const credentials = getWebIdentityCredentials();
+const admin = Mongo().getDB("admin");
+const external = admin.getMongo().getDB("$external");
+
+assert(admin.auth("bob", "pwd123"));
+const config = readSetupJson();
+assert.commandWorked(external.runCommand({createUser: ASSUMED_ROLE, roles:[{role: 'read', db: "aws"}]}));
+
+const testConn = new Mongo();
+const testExternal = testConn.getDB('$external');
+assert(testExternal.auth({
+    user: credentials["AccessKeyId"],
+    pwd: credentials["SecretAccessKey"],
+    awsIamSessionToken: credentials["SessionToken"],
+    mechanism: 'MONGODB-AWS'
+}));
+}());

.evergreen/auth_aws/lib/aws_assume_web_role.py

@@ -0,0 +1,55 @@
+#!/usr/bin/env python3
+"""
+Script for assuming an aws role using AssumeRoleWithWebIdentity.
+"""
+
+import argparse
+import os
+import uuid
+import logging
+
+import boto3
+
+LOGGER = logging.getLogger(__name__)
+
+def _assume_role_with_web_identity():
+    sts_client = boto3.client("sts")
+
+    token_file = os.environ['AWS_WEB_IDENTITY_TOKEN_FILE']
+    with open(token_file) as fid:
+        token = fid.read()
+    role_name = os.environ['AWS_ROLE_ARN']
+
+    response = sts_client.assume_role_with_web_identity(RoleArn=role_name, RoleSessionName=str(uuid.uuid4()), WebIdentityToken=token, DurationSeconds=900)
+
+    creds = response["Credentials"]
+
+
+    print(f"""{{
+  "AccessKeyId" : "{creds["AccessKeyId"]}",
+  "SecretAccessKey" : "{creds["SecretAccessKey"]}",
+  "SessionToken" : "{creds["SessionToken"]}",
+  "Expiration" : "{str(creds["Expiration"])}"
+}}""")
+
+
+def main() -> None:
+    """Execute Main entry point."""
+
+    parser = argparse.ArgumentParser(description='Assume Role frontend.')
+
+    parser.add_argument('-v', "--verbose", action='store_true', help="Enable verbose logging")
+    parser.add_argument('-d', "--debug", action='store_true', help="Enable debug logging")
+
+    args = parser.parse_args()
+
+    if args.debug:
+        logging.basicConfig(level=logging.DEBUG)
+    elif args.verbose:
+        logging.basicConfig(level=logging.INFO)
+
+    _assume_role_with_web_identity()
+
+
+if __name__ == "__main__":
+    main()

.evergreen/auth_aws/lib/aws_handle_oidc_creds.py

@@ -0,0 +1,111 @@
+#!/usr/bin/env python3
+"""
+Script for handling OIDC credentials.
+"""
+import argparse
+import base64
+import os
+import time
+import uuid
+
+from jwkest.jwk import RSAKey, import_rsa_key
+from pyop.authz_state import AuthorizationState
+from pyop.provider import Provider
+from pyop.subject_identifier import HashBasedSubjectIdentifierFactory
+from pyop.userinfo import Userinfo
+
+
+HERE = os.path.abspath(os.path.dirname(__file__))
+ISSUER = os.environ['IDP_ISSUER']
+JWKS_URI = os.environ['IDP_JWKS_URI']
+RSA_KEY = os.environ["IDP_RSA_KEY"]
+if RSA_KEY.endswith('='):
+    RSA_KEY = base64.urlsafe_b64decode(RSA_KEY).decode('utf-8')
+AUDIENCE = 'sts.amazonaws.com'
+
+
+def get_provider(client_id=None, client_secret=None):
+    """Get a configured OIDC provider."""
+    configuration_information = {
+        'issuer': ISSUER,
+        'authorization_endpoint': "https://example.com",
+        'jwks_uri': JWKS_URI,
+        'token_endpoint': "https://example.com",
+        'userinfo_endpoint': "https://example.com",
+        'registration_endpoint': "https://example.com",
+        'end_session_endpoint': "https://example.com",
+        'scopes_supported': ['openid', 'profile'],
+        'response_types_supported': ['code', 'code id_token', 'code token', 'code id_token token'],  # code and hybrid
+        'response_modes_supported': ['query', 'fragment'],
+        'grant_types_supported': ['authorization_code', 'implicit'],
+        'subject_types_supported': ['pairwise'],
+        'token_endpoint_auth_methods_supported': ['client_secret_basic'],
+        'claims_parameter_supported': True
+    }
+
+    userinfo_db = Userinfo({'test_user': {}})
+    signing_key = RSAKey(key=import_rsa_key(RSA_KEY), alg='RS256')
+
+    if client_id:
+        client_info = {
+            'client_id': client_id,
+            'client_id_issued_at': int(time.time()),
+            'client_secret': client_secret,
+            'redirect_uris': ['https://example.com'],
+            'response_types': ['code'],
+            'client_secret_expires_at': 0  # never expires
+        }
+        clients = {client_id: client_info}
+    else:
+        clients = {}
+    auth_state = AuthorizationState(HashBasedSubjectIdentifierFactory('salt'))
+    return Provider(signing_key, configuration_information,
+                    auth_state, clients, userinfo_db)
+
+
+def get_id_token():
+    """Get a valid ID token."""
+    client_id = AUDIENCE
+    client_secret = uuid.uuid4().hex
+    provider = get_provider(client_id, client_secret)
+    response = provider.parse_authentication_request(f'response_type=code&client_id={client_id}&scope=openid&redirect_uri=https://example.com')
+    resp = provider.authorize(response, 'test_user')
+    code = resp.to_dict()["code"]
+    creds = f'{client_id}:{client_secret}'
+    creds = base64.urlsafe_b64encode(creds.encode('utf-8')).decode('utf-8')
+    headers = dict(Authorization=f'Basic {creds}')
+    response = provider.handle_token_request(f'grant_type=authorization_code&code={code}&redirect_uri=https://example.com', headers)
+    token = response["id_token"]
+    if 'AWS_WEB_IDENTITY_TOKEN_FILE' in os.environ:
+        with open(os.environ['AWS_WEB_IDENTITY_TOKEN_FILE'], 'w') as fid:
+            fid.write(token)
+    return token
+
+
+def get_jwks_data():
+    """Get the jkws data for the jwks lambda endpoint."""
+    jwks = get_provider().jwks
+    jwks['keys'][0]['use'] = 'sig'
+    jwks['keys'][0]['kid'] = '1549e0aef574d1c7bdd136c202b8d290580b165c'
+    return jwks
+
+
+def get_config_data():
+    """Get the config data for the openid config lambda endpoint."""
+    return get_provider().provider_configuration.to_dict()
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument(dest='command', help="The command to run (config, jwks, token)")
+
+    # Parse and print the results
+    args = parser.parse_args()
+    if args.command == 'jwks':
+        print(get_jwks_data())
+    elif args.command == 'config':
+        print(get_config_data())
+    elif args.command == 'token':
+        print(get_id_token())
+    else:
+        raise ValueError('Command must be one of: (config, jwks, token)')

.evergreen/auth_aws/lib/aws_unassign_instance_profile.py

@@ -0,0 +1,99 @@
+#!/usr/bin/env python3
+"""
+Script for unassigning an instance policy from the current machine.
+"""
+
+import argparse
+import urllib.error
+import urllib.request
+import logging
+import sys
+import time
+
+import boto3
+import botocore
+
+LOGGER = logging.getLogger(__name__)
+
+def _get_local_instance_id():
+    return urllib.request.urlopen('http://169.254.169.254/latest/meta-data/instance-id', timeout=5).read().decode()
+
+def _has_instance_profile():
+    base_url = "http://169.254.169.254/latest/meta-data/iam/security-credentials/"
+    try:
+        print("Reading: " + base_url)
+        iam_role = urllib.request.urlopen(base_url).read().decode()
+    except urllib.error.HTTPError as e:
+        print(e)
+        if e.code == 404:
+            return False
+        raise e
+
+    try:
+        url = base_url + iam_role
+        print("Reading: " + url)
+        req = urllib.request.urlopen(url)
+    except urllib.error.HTTPError as e:
+        print(e)
+        if e.code == 404:
+            return False
+        raise e
+
+    return True
+
+def _wait_no_instance_profile():
+    retry = 60
+    while _has_instance_profile() and retry:
+        time.sleep(5)
+        retry -= 1
+
+    if retry == 0:
+        raise ValueError("Timeout on waiting for no instance profile")
+
+def _unassign_instance_policy():
+
+    try:
+        instance_id = _get_local_instance_id()
+    except urllib.error.URLError as e:
+        print(e)
+        sys.exit(0)
+
+    ec2_client = boto3.client("ec2", 'us-east-1')
+
+    #https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/ec2.html#EC2.Client.describe_iam_instance_profile_associations
+    try:
+        response = ec2_client.describe_iam_instance_profile_associations(Filters=[{"Name":"instance-id","Values": [instance_id]}])
+        associations = response['IamInstanceProfileAssociations']
+        if associations:
+            print('disassociating')
+            ec2_client.disassociate_iam_instance_profile(AssociationId=associations[0]['AssociationId'])
+
+        # Wait for the instance profile to be assigned by polling the local instance metadata service
+        _wait_no_instance_profile()
+
+    except botocore.exceptions.ClientError as ce:
+        if ce.response["Error"]["Code"] == "RequestLimitExceeded":
+            print("WARNING: RequestLimitExceeded, exiting with error code 2")
+            sys.exit(2)
+        raise
+
+def main() -> None:
+    """Execute Main entry point."""
+
+    parser = argparse.ArgumentParser(description='IAM UnAssign Instance frontend.')
+
+    parser.add_argument('-v', "--verbose", action='store_true', help="Enable verbose logging")
+    parser.add_argument('-d', "--debug", action='store_true', help="Enable debug logging")
+
+    args = parser.parse_args()
+
+    if args.debug:
+        logging.basicConfig(level=logging.DEBUG)
+    elif args.verbose:
+        logging.basicConfig(level=logging.INFO)
+
+    _unassign_instance_policy()
+
+
+if __name__ == "__main__":
+    main()

.gitignore

@@ -2,3 +2,6 @@
 evergreen_config_generator/build
 evergreen_config_generator/dist
 *.egg-info
+# The credentials files should not be checked in to version control.
+.evergreen/auth_aws/aws_e2e_setup.json
+.evergreen/auth_aws/creds.json