Adding region-specifc domain suffix for sts endpoints and adding new regions and domain suffixes

dist/efs-utils.conf

@@ -57,11 +57,9 @@ retry_nfs_mount_command_timeout_sec = 15
 [mount.cn-north-1]
 dns_name_suffix = amazonaws.com.cn
 
-
 [mount.cn-northwest-1]
 dns_name_suffix = amazonaws.com.cn
 
-
 [mount.us-iso-east-1]
 dns_name_suffix = c2s.ic.gov
 stunnel_cafile = /etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem

src/mount_efs/__init__.py

@@ -195,7 +195,7 @@
 
 CREDENTIALS_KEYS = ["AccessKeyId", "SecretAccessKey", "Token"]
 ECS_TASK_METADATA_API = "http://169.254.170.2"
-STS_ENDPOINT_URL_FORMAT = "https://sts.{}.amazonaws.com/"
+STS_ENDPOINT_URL_FORMAT = "https://sts.{}.{}/"
 INSTANCE_METADATA_TOKEN_URL = "http://169.254.169.254/latest/api/token"
 INSTANCE_METADATA_SERVICE_URL = (
     "http://169.254.169.254/latest/dynamic/instance-identity/document/"
@@ -409,6 +409,22 @@ def _fatal_error(message):
 
     _fatal_error(metadata_exception)
 
+def get_target_domain_suffix(config):
+    def _fatal_error():
+        fatal_error(
+            'Error retrieving region. Please set the "dns_name_suffix" parameter '
+            "in the efs-utils configuration file."
+        )
+    region = get_target_region(config)
+    config_section = get_config_section(config, region)
+
+    try:
+        return config.get(config_section, "dns_name_suffix")
+    except NoOptionError:
+        pass
+
+    _fatal_error()
+
 
 def get_target_az(config, options):
     if "az" in options:
@@ -691,6 +707,7 @@ def get_aws_security_credentials(
     config,
     use_iam,
     region,
+    dns_name_suffix,
     awsprofile=None,
     aws_creds_uri=None,
     jwt_path=None,
@@ -735,6 +752,7 @@ def get_aws_security_credentials(
             role_arn,
             jwt_path,
             region,
+            dns_name_suffix,
             False,
         )
         if credentials and credentials_source:
@@ -749,6 +767,7 @@ def get_aws_security_credentials(
             os.environ[WEB_IDENTITY_ROLE_ARN_ENV],
             os.environ[WEB_IDENTITY_TOKEN_FILE_ENV],
             region,
+            dns_name_suffix,
             False,
         )
         if credentials and credentials_source:
@@ -822,7 +841,7 @@ def get_aws_security_credentials_from_ecs(config, aws_creds_uri, is_fatal=False)
 
 
 def get_aws_security_credentials_from_webidentity(
-    config, role_arn, token_file, region, is_fatal=False
+    config, role_arn, token_file, region, dns_name_suffix, is_fatal=False
 ):
     try:
         with open(token_file, "r") as f:
@@ -834,7 +853,7 @@ def get_aws_security_credentials_from_webidentity(
         else:
             return None, None
 
-    STS_ENDPOINT_URL = STS_ENDPOINT_URL_FORMAT.format(region)
+    STS_ENDPOINT_URL = STS_ENDPOINT_URL_FORMAT.format(region,dns_name_suffix)
     webidentity_url = (
         STS_ENDPOINT_URL
         + "?"
@@ -1753,6 +1772,10 @@ def bootstrap_proxy(
         security_credentials = None
         client_info = get_client_info(config)
         region = get_target_region(config, options)
+<<<<<<< HEAD
+        dns_name_suffix = get_target_domain_suffix(config)
+=======
+>>>>>>> upstream/master
 
         if tls_enabled(options):
             cert_details = {}
@@ -1769,7 +1792,7 @@ def bootstrap_proxy(
                     kwargs = {"awsprofile": get_aws_profile(options, use_iam)}
 
                 security_credentials, credentials_source = get_aws_security_credentials(
-                    config, use_iam, region, **kwargs
+                    config, use_iam, region, dns_name_suffix, **kwargs
                 )
 
                 if credentials_source:
@@ -2667,8 +2690,14 @@ def _validate_replacement_field_count(format_str, expected_ct):
     if options and "crossaccount" in options:
         try:
             az_id = get_az_id_from_instance_metadata(config, options)
+<<<<<<< HEAD
+            region = get_target_region(config)
+            dns_name_suffix = get_target_domain_suffix(config)
+            dns_name = "%s.%s.efs.%s.%s" % (az_id, fs_id, region, dns_name_suffix)
+=======
             region = get_target_region(config, options)
             dns_name = "%s.%s.efs.%s.amazonaws.com" % (az_id, fs_id, region)
+>>>>>>> upstream/master
         except RuntimeError:
             err_msg = "Cannot retrieve AZ-ID from metadata service. This is required for the crossaccount mount option."
             fatal_error(err_msg)

src/watchdog/__init__.py

@@ -146,7 +146,7 @@
 AP_ID_RE = re.compile("^fsap-[0-9a-f]{17}$")
 
 ECS_TASK_METADATA_API = "http://169.254.170.2"
-STS_ENDPOINT_URL_FORMAT = "https://sts.{}.amazonaws.com/"
+STS_ENDPOINT_URL_FORMAT = "https://sts.{}.{}/"
 INSTANCE_IAM_URL = "http://169.254.169.254/latest/meta-data/iam/security-credentials/"
 INSTANCE_METADATA_TOKEN_URL = "http://169.254.169.254/latest/api/token"
 SECURITY_CREDS_ECS_URI_HELP_URL = (
@@ -383,8 +383,10 @@ def get_aws_security_credentials_from_webidentity(config, role_arn, token_file,
     except Exception as e:
         logging.error("Error reading token file %s: %s", token_file, e)
         return None
+
+    dns_name_suffix = get_target_domain_suffix(config, region)
 
-    STS_ENDPOINT_URL = STS_ENDPOINT_URL_FORMAT.format(region)
+    STS_ENDPOINT_URL = STS_ENDPOINT_URL_FORMAT.format(region, dns_name_suffix)
     webidentity_url = (
         STS_ENDPOINT_URL
         + "?"
@@ -497,6 +499,29 @@ def credentials_file_helper(file_path, awsprofile):
 
     return credentials
 
+def get_target_domain_suffix(config, region):
+    def _fatal_error():
+        fatal_error(
+            'Error retrieving DNS domain suffix for region. Please set the "dns_name_suffix" parameter '
+            "in the efs-utils configuration file."
+        )
+
+    config_section = get_config_section(config, region)
+
+    try:
+        return config.get(config_section, "dns_name_suffix")
+    except NoOptionError:
+        pass
+
+    _fatal_error()
+
+def get_config_section(config, region):
+    region_specific_config_section = "%s.%s" % (MOUNT_CONFIG_SECTION, region)
+    if config.has_section(region_specific_config_section):
+        config_section = region_specific_config_section
+    else:
+        config_section = MOUNT_CONFIG_SECTION
+    return config_section
 
 def is_instance_metadata_url(url):
     return url.startswith("http://169.254.169.254")

test/mount_efs_test/test_bootstrap_proxy.py

@@ -16,6 +16,7 @@
 DNS_NAME = "%s.efs.us-east-1.amazonaws.com" % FS_ID
 MOUNT_POINT = "/mnt"
 REGION = "us-east-1"
+DOMAIN_SUFFIX = "amazonaws.com"
 
 
 DEFAULT_TLS_PORT = 20049
@@ -40,6 +41,7 @@ def setup_mocks(mocker):
         return_value=(DNS_NAME, None),
     )
     mocker.patch("mount_efs.get_target_region", return_value=REGION)
+    mocker.patch("mount_efs.get_target_domain_suffix", return_value=DOMAIN_SUFFIX)
     mocker.patch("mount_efs.write_tunnel_state_file", return_value="~mocktempfile")
     mocker.patch("mount_efs.create_certificate")
     mocker.patch("os.rename")
@@ -139,6 +141,7 @@ def test_bootstrap_proxy_cert_created_tls_mount(mocker, tmpdir):
     setup_mocks_without_popen(mocker)
     mocker.patch("mount_efs.get_mount_specific_filename", return_value=DNS_NAME)
     mocker.patch("mount_efs.get_target_region", return_value=REGION)
+    mocker.patch("mount_efs.get_target_domain_suffix", return_value=DOMAIN_SUFFIX)
     state_file_dir = str(tmpdir)
     tls_dict = mount_efs.tls_paths_dictionary(DNS_NAME + "+", state_file_dir)
     mocker.patch("mount_efs.is_ocsp_enabled", return_value=False)
@@ -184,6 +187,7 @@ def test_bootstrap_proxy_cert_not_created_non_tls_mount(mocker, tmpdir):
     setup_mocks_without_popen(mocker)
     mocker.patch("mount_efs.get_mount_specific_filename", return_value=DNS_NAME)
     mocker.patch("mount_efs.get_target_region", return_value=REGION)
+    mocker.patch("mount_efs.get_target_domain_suffix", return_value=DOMAIN_SUFFIX)
     state_file_dir = str(tmpdir)
     tls_dict = mount_efs.tls_paths_dictionary(DNS_NAME + "+", state_file_dir)
 

test/mount_efs_test/test_get_aws_security_credentials.py

@@ -102,7 +102,7 @@ def test_get_aws_security_credentials_config_or_creds_file_found_creds_found_wit
     mocker.patch("mount_efs.credentials_file_helper", return_value=file_helper_resp)
 
     credentials, credentials_source = mount_efs.get_aws_security_credentials(
-        config, True, "us-east-1", "test_profile"
+        config, True, "us-east-1", "amazonaws.com", "test_profile"
     )
 
     assert credentials["AccessKeyId"] == ACCESS_KEY_ID_VAL
@@ -126,7 +126,7 @@ def test_get_aws_security_credentials_config_or_creds_file_found_creds_found_wit
     mocker.patch("mount_efs.credentials_file_helper", return_value=file_helper_resp)
 
     credentials, credentials_source = mount_efs.get_aws_security_credentials(
-        config, True, "us-east-1", "test_profile"
+        config, True, "us-east-1", "amazonaws.com", "test_profile"
     )
 
     assert credentials["AccessKeyId"] == ACCESS_KEY_ID_VAL
@@ -138,7 +138,7 @@ def test_get_aws_security_credentials_config_or_creds_file_found_creds_found_wit
 def test_get_aws_security_credentials_do_not_use_iam():
     config = get_fake_config()
     credentials, credentials_source = mount_efs.get_aws_security_credentials(
-        config, False, "us-east-1", "test_profile"
+        config, False, "us-east-1", "amazonaws.com", "test_profile"
     )
 
     assert not credentials
@@ -165,7 +165,7 @@ def _test_get_aws_security_credentials_get_ecs_from_env_url(mocker):
     mocker.patch("mount_efs.urlopen", return_value=MockUrlLibResponse(data=response))
 
     credentials, credentials_source = mount_efs.get_aws_security_credentials(
-        config, True, "us-east-1", None
+        config, True, "us-east-1", "amazonaws.com", None
     )
 
     assert credentials["AccessKeyId"] == ACCESS_KEY_ID_VAL
@@ -187,7 +187,7 @@ def test_get_aws_security_credentials_get_ecs_from_option_url(mocker):
     )
     mocker.patch("mount_efs.urlopen", return_value=MockUrlLibResponse(data=response))
     credentials, credentials_source = mount_efs.get_aws_security_credentials(
-        config, True, "us-east-1", None, AWSCREDSURI
+        config, True, "us-east-1", "amazonaws.com", None, AWSCREDSURI
     )
 
     assert credentials["AccessKeyId"] == ACCESS_KEY_ID_VAL
@@ -279,7 +279,7 @@ def _test_get_aws_security_credentials_get_instance_metadata_role_name(
     mocker.patch("mount_efs.urlopen", side_effect=side_effects)
 
     credentials, credentials_source = mount_efs.get_aws_security_credentials(
-        config, True, "us-east-1", None
+        config, True, "us-east-1", "amazonaws.com", None
     )
 
     assert credentials["AccessKeyId"] == ACCESS_KEY_ID_VAL
@@ -295,7 +295,7 @@ def test_get_aws_security_credentials_no_credentials_found(mocker, capsys):
     mocker.patch("mount_efs.urlopen")
 
     with pytest.raises(SystemExit) as ex:
-        mount_efs.get_aws_security_credentials(config, True, "us-east-1", None)
+        mount_efs.get_aws_security_credentials(config, True, "us-east-1", "amazonaws.com", None)
 
     assert 0 != ex.value.code
 
@@ -320,7 +320,7 @@ def test_get_aws_security_credentials_credentials_not_found_in_files_and_botocor
     mount_efs.BOTOCORE_PRESENT = False
 
     with pytest.raises(SystemExit) as ex:
-        mount_efs.get_aws_security_credentials(config, True, "us-east-1", "default")
+        mount_efs.get_aws_security_credentials(config, True, "us-east-1", "amazonaws.com", "default")
 
     assert 0 != ex.value.code
 
@@ -348,7 +348,7 @@ def test_get_aws_security_credentials_botocore_present_get_assumed_profile_crede
     )
 
     credentials, credentials_source = mount_efs.get_aws_security_credentials(
-        config, True, "us-east-1", awsprofile="test-profile"
+        config, True, "us-east-1", "amazonaws.com", awsprofile="test-profile"
     )
     assert credentials["AccessKeyId"] == ACCESS_KEY_ID_VAL
     assert credentials["SecretAccessKey"] == SECRET_ACCESS_KEY_VAL
@@ -365,7 +365,7 @@ def test_get_aws_security_credentials_credentials_not_found_in_aws_creds_uri(
 
     with pytest.raises(SystemExit) as ex:
         mount_efs.get_aws_security_credentials(
-            config, True, "us-east-1", "default", AWSCREDSURI
+            config, True, "us-east-1", "amazonaws.com", "default", AWSCREDSURI
         )
 
     assert 0 != ex.value.code
@@ -474,6 +474,7 @@ def test_get_aws_security_credentials_from_webidentity_passed_in_both_params(moc
         config,
         True,
         "us-east-1",
+        "amazonaws.com",
         jwt_path=WEB_IDENTITY_TOKEN_FILE,
         role_arn=WEB_IDENTITY_ROLE_ARN,
     )
@@ -506,7 +507,7 @@ def test_get_aws_security_credentials_from_webidentity_passed_in_one_param(
 
     with pytest.raises(SystemExit) as ex:
         mount_efs.get_aws_security_credentials(
-            config, True, "us-east-1", jwt_path=WEB_IDENTITY_TOKEN_FILE
+            config, True, "us-east-1", "amazonaws.com", jwt_path=WEB_IDENTITY_TOKEN_FILE
         )
 
     assert 0 != ex.value.code

test/mount_efs_test/test_get_dns_name_and_fallback_mount_target_ip_address.py

@@ -28,9 +28,14 @@
     "cn-north-1": "amazonaws.com.cn",
     "cn-northwest-1": "amazonaws.com.cn",
     "us-iso-east-1": "c2s.ic.gov",
+    "us-iso-west-1": "c2s.ic.gov",
     "us-isob-east-1": "sc2s.sgov.gov",
+    "us-isob-west-1": "sc2s.sgov.gov",
+    "us-isof-south-1": "csp.hci.ic.gov",
+    "us-isof-east-1": "csp.hci.ic.gov",
+    "eu-isoe-west-1": "cloud.adc-e.uk"
 }
-SPECIAL_REGIONS = ["cn-north-1", "cn-northwest-1", "us-iso-east-1", "us-isob-east-1"]
+SPECIAL_REGIONS = ["cn-north-1", "cn-northwest-1", "us-iso-east-1", "us-iso-west-1", "us-isob-east-1", "us-isob-west-1", "us-isof-south-1", "us-isof-east-1", "eu-isoe-west-1"]
 DEFAULT_NFS_OPTIONS = {}
 OPTIONS_WITH_AZ = {"az": DEFAULT_AZ}
 OPTIONS_WITH_IP = {"mounttargetip": IP_ADDRESS}