Adding modeled endpoint variants support

botocore/args.py

@@ -153,13 +153,18 @@ def compute_client_args(self, service_model, client_config,
             endpoint_bridge=endpoint_bridge,
             s3_config=s3_config,
         )
+        endpoint_variant_tags = endpoint_config['metadata'].get('tags', [])
         # Create a new client config to be passed to the client based
         # on the final values. We do not want the user to be able
         # to try to modify an existing client with a client config.
         config_kwargs = dict(
             region_name=endpoint_config['region_name'],
             signature_version=endpoint_config['signature_version'],
             user_agent=user_agent)
+        if 'dualstack' in endpoint_variant_tags:
+            config_kwargs.update(use_dualstack_endpoint=True)
+        if 'fips' in endpoint_variant_tags:
+            config_kwargs.update(use_fips_endpoint=True)
         if client_config is not None:
             config_kwargs.update(
                 connect_timeout=client_config.connect_timeout,
@@ -173,6 +178,14 @@ def compute_client_args(self, service_model, client_config,
             )
         self._compute_retry_config(config_kwargs)
         s3_config = self.compute_s3_config(client_config)
+
+        is_s3_service = service_name in ['s3', 's3-control']
+
+        if is_s3_service and 'dualstack' in endpoint_variant_tags:
+            if s3_config is None:
+                s3_config = {}
+            s3_config['use_dualstack_endpoint'] = True
+
         return {
             'service_name': service_name,
             'parameter_validation': parameter_validation,
@@ -267,14 +280,18 @@ def _set_region_if_custom_s3_endpoint(self, endpoint_config,
     def _compute_sts_endpoint_config(self, **resolve_endpoint_kwargs):
         endpoint_config = self._resolve_endpoint(**resolve_endpoint_kwargs)
         if self._should_set_global_sts_endpoint(
-                resolve_endpoint_kwargs['region_name'],
-                resolve_endpoint_kwargs['endpoint_url']):
+            resolve_endpoint_kwargs['region_name'],
+            resolve_endpoint_kwargs['endpoint_url'],
+            endpoint_config
+        ):
             self._set_global_sts_endpoint(
                 endpoint_config, resolve_endpoint_kwargs['is_secure'])
         return endpoint_config
 
-    def _should_set_global_sts_endpoint(self, region_name, endpoint_url):
-        if endpoint_url:
+    def _should_set_global_sts_endpoint(self, region_name, endpoint_url,
+                                        endpoint_config):
+        endpoint_variant_tags = endpoint_config['metadata'].get('tags')
+        if endpoint_url or endpoint_variant_tags:
             return False
         return (
             self._get_sts_regional_endpoints_config() == 'legacy' and

botocore/client.py

@@ -20,7 +20,7 @@
 from botocore.docs.docstring import PaginatorDocstring
 from botocore.exceptions import (
     DataNotFoundError, OperationNotPageableError, UnknownSignatureVersionError,
-    InvalidEndpointDiscoveryConfigurationError, UnknownFIPSEndpointError,
+    InvalidEndpointDiscoveryConfigurationError
 )
 from botocore.hooks import first_non_none_response
 from botocore.model import ServiceModel
@@ -82,9 +82,12 @@ def create_client(self, service_name, region_name, is_secure=True,
         service_name = first_non_none_response(responses, default=service_name)
         service_model = self._load_service_model(service_name, api_version)
         cls = self._create_client_class(service_name, service_model)
+        region_name, client_config = self._normalize_fips_region(
+            region_name, client_config)
         endpoint_bridge = ClientEndpointBridge(
             self._endpoint_resolver, scoped_config, client_config,
-            service_signing_name=service_model.metadata.get('signingName'))
+            service_signing_name=service_model.metadata.get('signingName'),
+            config_store=self._config_store)
         client_args = self._get_client_args(
             service_model, region_name, is_secure, endpoint_url,
             verify, credentials, scoped_config, client_config, endpoint_bridge)
@@ -99,7 +102,6 @@ def create_client(self, service_name, region_name, is_secure=True,
         self._register_endpoint_discovery(
             service_client, endpoint_url, client_config
         )
-        self._register_lazy_block_unknown_fips_pseudo_regions(service_client)
         return service_client
 
     def create_client_class(self, service_name, api_version=None):
@@ -120,6 +122,30 @@ def _create_client_class(self, service_name, service_model):
         cls = type(str(class_name), tuple(bases), class_attributes)
         return cls
 
+    def _normalize_fips_region(self, region_name,
+                               client_config):
+        if region_name is not None:
+            normalized_region_name = region_name.replace(
+                'fips-', '').replace('-fips', '')
+            # If region has been transformed then set flag
+            if normalized_region_name != region_name:
+                config_use_fips_endpoint = Config(use_fips_endpoint=True)
+                if client_config:
+                    # Keeping endpoint setting client specific
+                    client_config = client_config.merge(
+                        config_use_fips_endpoint)
+                else:
+                    client_config = config_use_fips_endpoint
+                logger.warn(
+                    'transforming region from %s to %s and setting '
+                    'use_fips_endpoint to true. client should not '
+                    'be configured with a fips psuedo region.' % (
+                        region_name, normalized_region_name
+                    )
+                )
+                region_name = normalized_region_name
+        return region_name, client_config
+
     def _load_service_model(self, service_name, api_version=None):
         json_model = self._loader.load_service_model(service_name, 'service-2',
                                                      api_version=api_version)
@@ -235,45 +261,20 @@ def _requires_endpoint_discovery(self, client, enabled):
             return client.meta.service_model.endpoint_discovery_required
         return enabled
 
-    def _register_lazy_block_unknown_fips_pseudo_regions(self, client):
-        # This function blocks usage of FIPS pseudo-regions when the endpoint
-        # is not explicitly known to exist to the client to prevent accidental
-        # usage of incorrect or non-FIPS endpoints. This is done lazily by
-        # registering an exception on the before-sign event to allow for
-        # general client usage such as can_paginate, exceptions, etc.
-        region_name = client.meta.region_name
-        if not region_name or 'fips' not in region_name.lower():
-            return
-
-        partition = client.meta.partition
-        endpoint_prefix = client.meta.service_model.endpoint_prefix
-        known_regions = self._endpoint_resolver.get_available_endpoints(
-            endpoint_prefix,
-            partition,
-            allow_non_regional=True,
-        )
-
-        if region_name not in known_regions:
-            def _lazy_fips_exception(**kwargs):
-                service_name = client.meta.service_model.service_name
-                raise UnknownFIPSEndpointError(
-                    region_name=region_name,
-                    service_name=service_name,
-                )
-            client.meta.events.register('before-sign', _lazy_fips_exception)
-
     def _register_s3_events(self, client, endpoint_bridge, endpoint_url,
                             client_config, scoped_config):
         if client.meta.service_model.service_name != 's3':
             return
         S3RegionRedirector(endpoint_bridge, client).register()
         S3ArnParamHandler().register(client.meta.events)
+        use_fips_endpoint = client.meta.config.use_fips_endpoint
         S3EndpointSetter(
             endpoint_resolver=self._endpoint_resolver,
             region=client.meta.region_name,
             s3_config=client.meta.config.s3,
             endpoint_url=endpoint_url,
-            partition=client.meta.partition
+            partition=client.meta.partition,
+            use_fips_endpoint=use_fips_endpoint
         ).register(client.meta.events)
         self._set_s3_presign_signature_version(
             client.meta, client_config, scoped_config)
@@ -284,13 +285,15 @@ def _register_s3_control_events(
     ):
         if client.meta.service_model.service_name != 's3control':
             return
+        use_fips_endpoint = client.meta.config.use_fips_endpoint
         S3ControlArnParamHandler().register(client.meta.events)
         S3ControlEndpointSetter(
             endpoint_resolver=self._endpoint_resolver,
             region=client.meta.region_name,
             s3_config=client.meta.config.s3,
             endpoint_url=endpoint_url,
-            partition=client.meta.partition
+            partition=client.meta.partition,
+            use_fips_endpoint=use_fips_endpoint
         ).register(client.meta.events)
 
     def _set_s3_presign_signature_version(self, client_meta,
@@ -415,30 +418,42 @@ class ClientEndpointBridge(object):
     utilize "us-east-1" by default if no region can be resolved."""
 
     DEFAULT_ENDPOINT = '{service}.{region}.amazonaws.com'
-    _DUALSTACK_ENABLED_SERVICES = ['s3', 's3-control']
+    _DUALSTACK_CUSTOMIZED_SERVICES = ['s3', 's3-control']
 
     def __init__(self, endpoint_resolver, scoped_config=None,
                  client_config=None, default_endpoint=None,
-                 service_signing_name=None):
+                 service_signing_name=None, config_store=None):
         self.service_signing_name = service_signing_name
         self.endpoint_resolver = endpoint_resolver
         self.scoped_config = scoped_config
         self.client_config = client_config
         self.default_endpoint = default_endpoint or self.DEFAULT_ENDPOINT
+        self.config_store = config_store
 
     def resolve(self, service_name, region_name=None, endpoint_url=None,
                 is_secure=True):
         region_name = self._check_default_region(service_name, region_name)
+        use_dualstack_endpoint = self._resolve_use_dualstack_endpoint(
+            service_name)
+        use_fips_endpoint = self._resolve_endpoint_variant_config_var(
+            'use_fips_endpoint'
+        )
         resolved = self.endpoint_resolver.construct_endpoint(
-            service_name, region_name)
+            service_name, region_name,
+            use_dualstack_endpoint=use_dualstack_endpoint,
+            use_fips_endpoint=use_fips_endpoint,
+        )
 
         # If we can't resolve the region, we'll attempt to get a global
         # endpoint for non-regionalized services (iam, route53, etc)
         if not resolved:
             # TODO: fallback partition_name should be configurable in the
             # future for users to define as needed.
             resolved = self.endpoint_resolver.construct_endpoint(
-                service_name, region_name, partition_name='aws')
+                service_name, region_name, partition_name='aws',
+                use_dualstack_endpoint=use_dualstack_endpoint,
+                use_fips_endpoint=use_fips_endpoint,
+            )
 
         if resolved:
             return self._create_endpoint(
@@ -456,20 +471,13 @@ def _check_default_region(self, service_name, region_name):
 
     def _create_endpoint(self, resolved, service_name, region_name,
                          endpoint_url, is_secure):
-        explicit_region = region_name is not None
         region_name, signing_region = self._pick_region_values(
             resolved, region_name, endpoint_url)
         if endpoint_url is None:
-            if self._is_s3_dualstack_mode(service_name):
-                endpoint_url = self._create_dualstack_endpoint(
-                    service_name, region_name,
-                    resolved['dnsSuffix'], is_secure, explicit_region)
-            else:
-                # Use the sslCommonName over the hostname for Python 2.6 compat.
-                hostname = resolved.get('sslCommonName', resolved.get('hostname'))
-                endpoint_url = self._make_url(
-                    hostname, is_secure, resolved.get('protocols', [])
-                )
+            # Use the sslCommonName over the hostname for Python 2.6 compat.
+            hostname = resolved.get('sslCommonName', resolved.get('hostname'))
+            endpoint_url = self._make_url(hostname, is_secure,
+                                          resolved.get('protocols', []))
         signature_version = self._resolve_signature_version(
             service_name, resolved)
         signing_name = self._resolve_signing_name(service_name, resolved)
@@ -479,9 +487,28 @@ def _create_endpoint(self, resolved, service_name, region_name,
             endpoint_url=endpoint_url, metadata=resolved,
             signature_version=signature_version)
 
+    def _resolve_endpoint_variant_config_var(self, config_var):
+        client_config = self.client_config
+        config_val = False
+
+        # Client configuration arg has precedence
+        if client_config and getattr(client_config, config_var) is not None:
+            return getattr(client_config, config_var)
+        elif self.config_store is not None:
+            # Check config store
+            config_val = self.config_store.get_config_variable(config_var)
+        return config_val
+
+    def _resolve_use_dualstack_endpoint(self, service_name):
+        s3_dualstack_mode = self._is_s3_dualstack_mode(service_name)
+        if s3_dualstack_mode is not None:
+            return s3_dualstack_mode
+        return self._resolve_endpoint_variant_config_var(
+            'use_dualstack_endpoint')
+
     def _is_s3_dualstack_mode(self, service_name):
-        if service_name not in self._DUALSTACK_ENABLED_SERVICES:
-            return False
+        if service_name not in self._DUALSTACK_CUSTOMIZED_SERVICES:
+            return None
         # TODO: This normalization logic is duplicated from the
         # ClientArgsCreator class.  Consolidate everything to
         # ClientArgsCreator.  _resolve_signature_version also has similarly
@@ -491,27 +518,11 @@ def _is_s3_dualstack_mode(self, service_name):
                 'use_dualstack_endpoint' in client_config.s3:
             # Client config trumps scoped config.
             return client_config.s3['use_dualstack_endpoint']
-        if self.scoped_config is None:
-            return False
-        enabled = self.scoped_config.get('s3', {}).get(
-            'use_dualstack_endpoint', False)
-        if enabled in [True, 'True', 'true']:
-            return True
-        return False
-
-    def _create_dualstack_endpoint(self, service_name, region_name,
-                                   dns_suffix, is_secure, explicit_region):
-        if not explicit_region and region_name == 'aws-global':
-            # If the region_name passed was not explicitly set, default to
-            # us-east-1 instead of the modeled default aws-global. Dualstack
-            # does not support aws-global
-            region_name = 'us-east-1'
-        hostname = '{service}.dualstack.{region}.{dns_suffix}'.format(
-            service=service_name, region=region_name,
-            dns_suffix=dns_suffix)
-        # Dualstack supports http and https so were hardcoding this value for
-        # now.  This can potentially move into the endpoints.json file.
-        return self._make_url(hostname, is_secure, ['http', 'https'])
+        if self.scoped_config is not None:
+            enabled = self.scoped_config.get('s3', {}).get(
+                'use_dualstack_endpoint')
+            if enabled in [True, 'True', 'true']:
+                return True
 
     def _assume_endpoint(self, service_name, region_name, endpoint_url,
                          is_secure):

botocore/config.py

@@ -169,6 +169,18 @@ class Config(object):
         Setting this to False disables the injection of operation parameters
         into the prefix of the hostname. This is useful for clients providing
         custom endpoints that should not have their host prefix modified.
+
+    :type use_dualstack_endpoint: bool
+    :param use_dualstack_endpoint: Setting to True enables fips
+        endpoint resolution.
+
+        Defaults to None.
+
+    :type use_fips_endpoint: bool
+    :param use_dualstack_endpoint: Setting to True enables dualstack
+        endpoint resolution.
+
+        Defaults to None.
     """
     OPTION_DEFAULTS = OrderedDict([
         ('region_name', None),
@@ -186,6 +198,8 @@ class Config(object):
         ('client_cert', None),
         ('inject_host_prefix', True),
         ('endpoint_discovery_enabled', None),
+        ('use_dualstack_endpoint', None),
+        ('use_fips_endpoint', None),
     ])
 
     def __init__(self, *args, **kwargs):

botocore/configprovider.py

@@ -83,6 +83,14 @@
         'imds_use_ipv6',
         'AWS_IMDS_USE_IPV6',
         False, utils.ensure_boolean),
+    'use_dualstack_endpoint': (
+        'use_dualstack_endpoint',
+        'AWS_USE_DUALSTACK_ENDPOINT',
+        None, utils.ensure_boolean),
+    'use_fips_endpoint': (
+        'use_fips_endpoint',
+        'AWS_USE_FIPS_ENDPOINT',
+        None, utils.ensure_boolean),
     'parameter_validation': ('parameter_validation', None, True, None),
     # Client side monitoring configurations.
     # Note: These configurations are considered internal to botocore.

botocore/exceptions.py

@@ -205,6 +205,19 @@ class NoRegionError(BaseEndpointResolverError):
     fmt = 'You must specify a region.'
 
 
+class EndpointVariantError(BaseEndpointResolverError):
+    """
+    Could not construct modeled endpoint variant.
+
+    :ivar error_msg: The msg explaining why dualstack endpoint is
+        unable to be constructed.
+
+    """
+
+    fmt = ('Unable to construct a modeled endpoint with the following '
+           'variant(s) {tags}: ')
+
+
 class UnknownEndpointError(BaseEndpointResolverError, ValueError):
     """
     Could not construct an endpoint.