From b6721edc7a924da72c158973bb5eba1a0441284b Mon Sep 17 00:00:00 2001 From: JD Davis Date: Wed, 7 Aug 2024 16:52:25 +0000 Subject: [PATCH 1/4] use dns suffix based on region --- src/mount_efs/__init__.py | 31 ++++++++++++++++++++++++++----- 1 file changed, 26 insertions(+), 5 deletions(-) diff --git a/src/mount_efs/__init__.py b/src/mount_efs/__init__.py index 083e3fa1..29889dbd 100755 --- a/src/mount_efs/__init__.py +++ b/src/mount_efs/__init__.py @@ -195,7 +195,7 @@ CREDENTIALS_KEYS = ["AccessKeyId", "SecretAccessKey", "Token"] ECS_TASK_METADATA_API = "http://169.254.170.2" -STS_ENDPOINT_URL_FORMAT = "https://sts.{}.amazonaws.com/" +STS_ENDPOINT_URL_FORMAT = "https://sts.{}.{}/" INSTANCE_METADATA_TOKEN_URL = "http://169.254.169.254/latest/api/token" INSTANCE_METADATA_SERVICE_URL = ( "http://169.254.169.254/latest/dynamic/instance-identity/document/" @@ -404,6 +404,22 @@ def _fatal_error(message): _fatal_error(metadata_exception) +def get_target_domain_suffix(config): + def _fatal_error(): + fatal_error( + 'Error retrieving region. Please set the "dns_name_suffix" parameter ' + "in the efs-utils configuration file." + ) + region = get_target_region(config) + config_section = get_config_section(config, region) + + try: + return config.get(config_section, "dns_name_suffix") + except NoOptionError: + pass + + _fatal_error() + def get_target_az(config, options): if "az" in options: @@ -686,6 +702,7 @@ def get_aws_security_credentials( config, use_iam, region, + dns_name_suffix, awsprofile=None, aws_creds_uri=None, jwt_path=None, @@ -730,6 +747,7 @@ def get_aws_security_credentials( role_arn, jwt_path, region, + dns_name_suffix, False, ) if credentials and credentials_source: @@ -744,6 +762,7 @@ def get_aws_security_credentials( os.environ[WEB_IDENTITY_ROLE_ARN_ENV], os.environ[WEB_IDENTITY_TOKEN_FILE_ENV], region, + dns_name_suffix, False, ) if credentials and credentials_source: @@ -817,7 +836,7 @@ def get_aws_security_credentials_from_ecs(config, aws_creds_uri, is_fatal=False) def get_aws_security_credentials_from_webidentity( - config, role_arn, token_file, region, is_fatal=False + config, role_arn, token_file, region, dns_name_suffix, is_fatal=False ): try: with open(token_file, "r") as f: @@ -829,7 +848,7 @@ def get_aws_security_credentials_from_webidentity( else: return None, None - STS_ENDPOINT_URL = STS_ENDPOINT_URL_FORMAT.format(region) + STS_ENDPOINT_URL = STS_ENDPOINT_URL_FORMAT.format(region,dns_name_suffix) webidentity_url = ( STS_ENDPOINT_URL + "?" @@ -1748,6 +1767,7 @@ def bootstrap_proxy( security_credentials = None client_info = get_client_info(config) region = get_target_region(config) + dns_name_suffix = get_target_domain_suffix(config) if tls_enabled(options): cert_details = {} @@ -1764,7 +1784,7 @@ def bootstrap_proxy( kwargs = {"awsprofile": get_aws_profile(options, use_iam)} security_credentials, credentials_source = get_aws_security_credentials( - config, use_iam, region, **kwargs + config, use_iam, region, dns_name_suffix, **kwargs ) if credentials_source: @@ -2663,7 +2683,8 @@ def _validate_replacement_field_count(format_str, expected_ct): try: az_id = get_az_id_from_instance_metadata(config, options) region = get_target_region(config) - dns_name = "%s.%s.efs.%s.amazonaws.com" % (az_id, fs_id, region) + dns_name_suffix = get_target_domain_suffix(config) + dns_name = "%s.%s.efs.%s.%s" % (az_id, fs_id, region, dns_name_suffix) except RuntimeError: err_msg = "Cannot retrieve AZ-ID from metadata service. This is required for the crossaccount mount option." fatal_error(err_msg) From 769637c637b40816edbda87fa6e7c1304f67b9fd Mon Sep 17 00:00:00 2001 From: JD Davis Date: Thu, 8 Aug 2024 19:59:54 +0000 Subject: [PATCH 2/4] adding dns name suffixes for new regions --- dist/efs-utils.conf | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/dist/efs-utils.conf b/dist/efs-utils.conf index 1b6d849e..82bf90ec 100644 --- a/dist/efs-utils.conf +++ b/dist/efs-utils.conf @@ -57,11 +57,9 @@ retry_nfs_mount_command_timeout_sec = 15 [mount.cn-north-1] dns_name_suffix = amazonaws.com.cn - [mount.cn-northwest-1] dns_name_suffix = amazonaws.com.cn - [mount.us-iso-east-1] dns_name_suffix = c2s.ic.gov stunnel_cafile = /etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem @@ -74,6 +72,22 @@ stunnel_cafile = /etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem dns_name_suffix = sc2s.sgov.gov stunnel_cafile = /etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem +[mount.us-isob-west-1] +dns_name_suffix = sc2s.sgov.gov +stunnel_cafile = /etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem + +[mount.us-isof-east-1] +dns_name_suffix = csp.hci.ic.gov +stunnel_cafile = /etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem + +[mount.us-isof-south-1] +dns_name_suffix = csp.hci.ic.gov +stunnel_cafile = /etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem + +[mount.eu-isoe-west-1] +dns_name_suffix = cloud.adc-e.uk +stunnel_cafile = /etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem + [mount-watchdog] enabled = true poll_interval_sec = 1 From 5addd5051d8a9eda20e583ce8a00bb2d9b59596a Mon Sep 17 00:00:00 2001 From: JD Davis Date: Fri, 9 Aug 2024 21:57:39 +0000 Subject: [PATCH 3/4] use region-specific dns suffix for sts --- src/watchdog/__init__.py | 29 +++++++++++++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/src/watchdog/__init__.py b/src/watchdog/__init__.py index 3247a380..56811073 100755 --- a/src/watchdog/__init__.py +++ b/src/watchdog/__init__.py @@ -146,7 +146,7 @@ AP_ID_RE = re.compile("^fsap-[0-9a-f]{17}$") ECS_TASK_METADATA_API = "http://169.254.170.2" -STS_ENDPOINT_URL_FORMAT = "https://sts.{}.amazonaws.com/" +STS_ENDPOINT_URL_FORMAT = "https://sts.{}.{}/" INSTANCE_IAM_URL = "http://169.254.169.254/latest/meta-data/iam/security-credentials/" INSTANCE_METADATA_TOKEN_URL = "http://169.254.169.254/latest/api/token" SECURITY_CREDS_ECS_URI_HELP_URL = ( @@ -383,8 +383,10 @@ def get_aws_security_credentials_from_webidentity(config, role_arn, token_file, except Exception as e: logging.error("Error reading token file %s: %s", token_file, e) return None + + dns_name_suffix = get_target_domain_suffix(config, region) - STS_ENDPOINT_URL = STS_ENDPOINT_URL_FORMAT.format(region) + STS_ENDPOINT_URL = STS_ENDPOINT_URL_FORMAT.format(region, dns_name_suffix) webidentity_url = ( STS_ENDPOINT_URL + "?" @@ -497,6 +499,29 @@ def credentials_file_helper(file_path, awsprofile): return credentials +def get_target_domain_suffix(config, region): + def _fatal_error(): + fatal_error( + 'Error retrieving DNS domain suffix for region. Please set the "dns_name_suffix" parameter ' + "in the efs-utils configuration file." + ) + + config_section = get_config_section(config, region) + + try: + return config.get(config_section, "dns_name_suffix") + except NoOptionError: + pass + + _fatal_error() + +def get_config_section(config, region): + region_specific_config_section = "%s.%s" % (MOUNT_CONFIG_SECTION, region) + if config.has_section(region_specific_config_section): + config_section = region_specific_config_section + else: + config_section = MOUNT_CONFIG_SECTION + return config_section def is_instance_metadata_url(url): return url.startswith("http://169.254.169.254") From 2911904e3d1a2e6a34669d7cf09e730218ed58d8 Mon Sep 17 00:00:00 2001 From: JD Davis Date: Tue, 17 Sep 2024 13:37:10 +0000 Subject: [PATCH 4/4] adding unit tests --- test/mount_efs_test/test_bootstrap_proxy.py | 4 ++++ .../test_get_aws_security_credentials.py | 23 ++++++++++--------- ...me_and_fallback_mount_target_ip_address.py | 7 +++++- 3 files changed, 22 insertions(+), 12 deletions(-) diff --git a/test/mount_efs_test/test_bootstrap_proxy.py b/test/mount_efs_test/test_bootstrap_proxy.py index d56f3e53..739c416d 100644 --- a/test/mount_efs_test/test_bootstrap_proxy.py +++ b/test/mount_efs_test/test_bootstrap_proxy.py @@ -16,6 +16,7 @@ DNS_NAME = "%s.efs.us-east-1.amazonaws.com" % FS_ID MOUNT_POINT = "/mnt" REGION = "us-east-1" +DOMAIN_SUFFIX = "amazonaws.com" DEFAULT_TLS_PORT = 20049 @@ -40,6 +41,7 @@ def setup_mocks(mocker): return_value=(DNS_NAME, None), ) mocker.patch("mount_efs.get_target_region", return_value=REGION) + mocker.patch("mount_efs.get_target_domain_suffix", return_value=DOMAIN_SUFFIX) mocker.patch("mount_efs.write_tunnel_state_file", return_value="~mocktempfile") mocker.patch("mount_efs.create_certificate") mocker.patch("os.rename") @@ -139,6 +141,7 @@ def test_bootstrap_proxy_cert_created_tls_mount(mocker, tmpdir): setup_mocks_without_popen(mocker) mocker.patch("mount_efs.get_mount_specific_filename", return_value=DNS_NAME) mocker.patch("mount_efs.get_target_region", return_value=REGION) + mocker.patch("mount_efs.get_target_domain_suffix", return_value=DOMAIN_SUFFIX) state_file_dir = str(tmpdir) tls_dict = mount_efs.tls_paths_dictionary(DNS_NAME + "+", state_file_dir) mocker.patch("mount_efs.is_ocsp_enabled", return_value=False) @@ -184,6 +187,7 @@ def test_bootstrap_proxy_cert_not_created_non_tls_mount(mocker, tmpdir): setup_mocks_without_popen(mocker) mocker.patch("mount_efs.get_mount_specific_filename", return_value=DNS_NAME) mocker.patch("mount_efs.get_target_region", return_value=REGION) + mocker.patch("mount_efs.get_target_domain_suffix", return_value=DOMAIN_SUFFIX) state_file_dir = str(tmpdir) tls_dict = mount_efs.tls_paths_dictionary(DNS_NAME + "+", state_file_dir) diff --git a/test/mount_efs_test/test_get_aws_security_credentials.py b/test/mount_efs_test/test_get_aws_security_credentials.py index e931f32e..a0798dc7 100644 --- a/test/mount_efs_test/test_get_aws_security_credentials.py +++ b/test/mount_efs_test/test_get_aws_security_credentials.py @@ -102,7 +102,7 @@ def test_get_aws_security_credentials_config_or_creds_file_found_creds_found_wit mocker.patch("mount_efs.credentials_file_helper", return_value=file_helper_resp) credentials, credentials_source = mount_efs.get_aws_security_credentials( - config, True, "us-east-1", "test_profile" + config, True, "us-east-1", "amazonaws.com", "test_profile" ) assert credentials["AccessKeyId"] == ACCESS_KEY_ID_VAL @@ -126,7 +126,7 @@ def test_get_aws_security_credentials_config_or_creds_file_found_creds_found_wit mocker.patch("mount_efs.credentials_file_helper", return_value=file_helper_resp) credentials, credentials_source = mount_efs.get_aws_security_credentials( - config, True, "us-east-1", "test_profile" + config, True, "us-east-1", "amazonaws.com", "test_profile" ) assert credentials["AccessKeyId"] == ACCESS_KEY_ID_VAL @@ -138,7 +138,7 @@ def test_get_aws_security_credentials_config_or_creds_file_found_creds_found_wit def test_get_aws_security_credentials_do_not_use_iam(): config = get_fake_config() credentials, credentials_source = mount_efs.get_aws_security_credentials( - config, False, "us-east-1", "test_profile" + config, False, "us-east-1", "amazonaws.com", "test_profile" ) assert not credentials @@ -165,7 +165,7 @@ def _test_get_aws_security_credentials_get_ecs_from_env_url(mocker): mocker.patch("mount_efs.urlopen", return_value=MockUrlLibResponse(data=response)) credentials, credentials_source = mount_efs.get_aws_security_credentials( - config, True, "us-east-1", None + config, True, "us-east-1", "amazonaws.com", None ) assert credentials["AccessKeyId"] == ACCESS_KEY_ID_VAL @@ -187,7 +187,7 @@ def test_get_aws_security_credentials_get_ecs_from_option_url(mocker): ) mocker.patch("mount_efs.urlopen", return_value=MockUrlLibResponse(data=response)) credentials, credentials_source = mount_efs.get_aws_security_credentials( - config, True, "us-east-1", None, AWSCREDSURI + config, True, "us-east-1", "amazonaws.com", None, AWSCREDSURI ) assert credentials["AccessKeyId"] == ACCESS_KEY_ID_VAL @@ -279,7 +279,7 @@ def _test_get_aws_security_credentials_get_instance_metadata_role_name( mocker.patch("mount_efs.urlopen", side_effect=side_effects) credentials, credentials_source = mount_efs.get_aws_security_credentials( - config, True, "us-east-1", None + config, True, "us-east-1", "amazonaws.com", None ) assert credentials["AccessKeyId"] == ACCESS_KEY_ID_VAL @@ -295,7 +295,7 @@ def test_get_aws_security_credentials_no_credentials_found(mocker, capsys): mocker.patch("mount_efs.urlopen") with pytest.raises(SystemExit) as ex: - mount_efs.get_aws_security_credentials(config, True, "us-east-1", None) + mount_efs.get_aws_security_credentials(config, True, "us-east-1", "amazonaws.com", None) assert 0 != ex.value.code @@ -320,7 +320,7 @@ def test_get_aws_security_credentials_credentials_not_found_in_files_and_botocor mount_efs.BOTOCORE_PRESENT = False with pytest.raises(SystemExit) as ex: - mount_efs.get_aws_security_credentials(config, True, "us-east-1", "default") + mount_efs.get_aws_security_credentials(config, True, "us-east-1", "amazonaws.com", "default") assert 0 != ex.value.code @@ -348,7 +348,7 @@ def test_get_aws_security_credentials_botocore_present_get_assumed_profile_crede ) credentials, credentials_source = mount_efs.get_aws_security_credentials( - config, True, "us-east-1", awsprofile="test-profile" + config, True, "us-east-1", "amazonaws.com", awsprofile="test-profile" ) assert credentials["AccessKeyId"] == ACCESS_KEY_ID_VAL assert credentials["SecretAccessKey"] == SECRET_ACCESS_KEY_VAL @@ -365,7 +365,7 @@ def test_get_aws_security_credentials_credentials_not_found_in_aws_creds_uri( with pytest.raises(SystemExit) as ex: mount_efs.get_aws_security_credentials( - config, True, "us-east-1", "default", AWSCREDSURI + config, True, "us-east-1", "amazonaws.com", "default", AWSCREDSURI ) assert 0 != ex.value.code @@ -474,6 +474,7 @@ def test_get_aws_security_credentials_from_webidentity_passed_in_both_params(moc config, True, "us-east-1", + "amazonaws.com", jwt_path=WEB_IDENTITY_TOKEN_FILE, role_arn=WEB_IDENTITY_ROLE_ARN, ) @@ -506,7 +507,7 @@ def test_get_aws_security_credentials_from_webidentity_passed_in_one_param( with pytest.raises(SystemExit) as ex: mount_efs.get_aws_security_credentials( - config, True, "us-east-1", jwt_path=WEB_IDENTITY_TOKEN_FILE + config, True, "us-east-1", "amazonaws.com", jwt_path=WEB_IDENTITY_TOKEN_FILE ) assert 0 != ex.value.code diff --git a/test/mount_efs_test/test_get_dns_name_and_fallback_mount_target_ip_address.py b/test/mount_efs_test/test_get_dns_name_and_fallback_mount_target_ip_address.py index 59a185e7..7de5e02b 100644 --- a/test/mount_efs_test/test_get_dns_name_and_fallback_mount_target_ip_address.py +++ b/test/mount_efs_test/test_get_dns_name_and_fallback_mount_target_ip_address.py @@ -28,9 +28,14 @@ "cn-north-1": "amazonaws.com.cn", "cn-northwest-1": "amazonaws.com.cn", "us-iso-east-1": "c2s.ic.gov", + "us-iso-west-1": "c2s.ic.gov", "us-isob-east-1": "sc2s.sgov.gov", + "us-isob-west-1": "sc2s.sgov.gov", + "us-isof-south-1": "csp.hci.ic.gov", + "us-isof-east-1": "csp.hci.ic.gov", + "eu-isoe-west-1": "cloud.adc-e.uk" } -SPECIAL_REGIONS = ["cn-north-1", "cn-northwest-1", "us-iso-east-1", "us-isob-east-1"] +SPECIAL_REGIONS = ["cn-north-1", "cn-northwest-1", "us-iso-east-1", "us-iso-west-1", "us-isob-east-1", "us-isob-west-1", "us-isof-south-1", "us-isof-east-1", "eu-isoe-west-1"] DEFAULT_NFS_OPTIONS = {} OPTIONS_WITH_AZ = {"az": DEFAULT_AZ} OPTIONS_WITH_IP = {"mounttargetip": IP_ADDRESS}