Merge pull request #83 from epam/develop

Add proxies support (#82)

CHANGELOG.md

@@ -4,6 +4,9 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [6.1.1] - 2024-10-04
+- added `MODULAR_SDK_HTTP_PROXY` and `MODULAR_SDK_HTTPS_PROXY` envs
+
 ## [6.1.0] - 2024-09-13
 - Update `pika` library from version `1.0.0b1` to `1.3.2`
 - Update `RabbitMqConnection` class to support `pika` version `1.3.2`

modular_sdk/commons/constants.py

@@ -1,4 +1,5 @@
 from enum import Enum
+import os
 
 HTTP_ATTR, HTTPS_ATTR = 'HTTP', 'HTTPS'
 
@@ -9,22 +10,92 @@
 MODULAR_AWS_SESSION_TOKEN_ENV = 'modular_aws_session_token'
 MODULAR_AWS_CREDENTIALS_EXPIRATION_ENV = 'modular_aws_credentials_expiration'
 
-REGION_ENV = 'AWS_REGION'
-DEFAULT_REGION_ENV = 'AWS_DEFAULT_REGION'
-MODULAR_REGION_ENV = 'MODULAR_AWS_REGION'
+_SENTINEL = object()
 
-MODULAR_SERVICE_MODE_ENV = 'modular_service_mode'
+
+class Env(str, Enum):
+    """
+    Abstract enumeration class for holding environment variables
+    """
+    default: str | None
+
+    def __new__(cls, value: str, default: str | None = None):
+        """
+        All environment variables and optionally their default values.
+        Since envs always have string type the default value also should be
+        of string type and then converted to the necessary type in code.
+        There is no default value if not specified (default equal to unset)
+        """
+        obj = str.__new__(cls, value)
+        obj._value_ = value
+
+        obj.default = default
+        return obj
+
+    def get(self, default=_SENTINEL) -> str | None:
+        if default is _SENTINEL:
+            default = self.default
+        if default is not None:
+            default = str(default)
+        return os.environ.get(self.value, default)
+
+    def set(self, val: str | None):
+        if val is None:
+            os.environ.pop(self.value, None)
+        else:
+            os.environ[self.value] = str(val)
+
+    # OLD ones
+    OLD_SERVICE_MODE = 'modular_service_mode'
+    OLD_MONGO_USER = 'modular_mongo_user'
+    OLD_MONGO_PASSWORD = 'modular_mongo_password'
+    OLD_MONGO_URL = 'modular_mongo_url'
+    OLD_MONGO_DB_NAME = 'modular_mongo_db_name'
+    OLD_ASSUME_ROLE_ARN = 'modular_assume_role_arn'  # may be multiple split by ,
+
+    OLD_MODULAR_AWS_REGION = 'MODULAR_AWS_REGION'  # used for cross account models access
+    OLD_INNER_CACHE_TTL_SECONDS = 'INNER_CACHE_TTL_SECONDS', '600'
+
+    # TODO they are NOT used currently. We should add their support to
+    #  code but make a gradual transition from the old ones so that both
+    #  old and new are supported for some time.
+    # NEW ones
+    SERVICE_MODE = 'MODULAR_SDK_SERVICE_MODE'
+    MONGO_USER = 'MODULAR_SDK_MONGO_USER'
+    MONGO_PASSWORD = 'MODULAR_SDK_MONGO_PASSWORD'
+    MONGO_URL = 'MODULAR_SDK_MONGO_URL'
+    MONGO_DB_NAME = 'MODULAR_SDK_MONGO_DB_NAME'
+    ASSUME_ROLE_ARN = 'MODULAR_SDK_ASSUME_ROLE_ARN'
+    ASSUME_ROLE_REGION = 'MODULAR_SDK_ASSUME_ROLE_REGION'
+    INNER_CACHE_TTL_SECONDS = 'MODULAR_SDK_INNER_CACHE_TTL_SECONDS', '300'
+
+    # these below are used
+
+    # proxies, impacts only our boto3 clients. Boto inside Pynamodb is NOT
+    # routed via these proxies
+    HTTP_PROXY = 'MODULAR_SDK_HTTP_PROXY'
+    HTTPS_PROXY = 'MODULAR_SDK_HTTPS_PROXY'
+
+    AWS_REGION = 'AWS_REGION'
+    AWS_DEFAULT_REGION = 'AWS_DEFAULT_REGION'
+
+
+REGION_ENV = Env.AWS_REGION.value
+DEFAULT_REGION_ENV = Env.AWS_DEFAULT_REGION.value
+MODULAR_REGION_ENV = Env.OLD_MODULAR_AWS_REGION.value
+
+MODULAR_SERVICE_MODE_ENV = Env.OLD_SERVICE_MODE.value
 SERVICE_MODE_DOCKER = 'docker'
 SERVICE_MODE_SAAS = 'saas'
 
-PARAM_MONGO_USER = 'modular_mongo_user'
-PARAM_MONGO_PASSWORD = 'modular_mongo_password'
-PARAM_MONGO_URL = 'modular_mongo_url'
-PARAM_MONGO_DB_NAME = 'modular_mongo_db_name'
-PARAM_ASSUME_ROLE_ARN = 'modular_assume_role_arn'
+PARAM_MONGO_USER = Env.OLD_MONGO_USER.value
+PARAM_MONGO_PASSWORD = Env.OLD_MONGO_PASSWORD.value
+PARAM_MONGO_URL = Env.OLD_MONGO_URL.value
+PARAM_MONGO_DB_NAME = Env.OLD_MONGO_DB_NAME.value
+PARAM_ASSUME_ROLE_ARN = Env.OLD_ASSUME_ROLE_ARN.value
 
-ENV_INNER_CACHE_TTL_SECONDS = 'INNER_CACHE_TTL_SECONDS'
-DEFAULT_INNER_CACHE_TTL_SECONDS: int = 600
+ENV_INNER_CACHE_TTL_SECONDS = Env.OLD_INNER_CACHE_TTL_SECONDS.value
+DEFAULT_INNER_CACHE_TTL_SECONDS: int = int(Env.OLD_INNER_CACHE_TTL_SECONDS.default)
 
 
 class ParentType(str, Enum):

modular_sdk/commons/error_helper.py

@@ -1,3 +1,4 @@
+# TODO remove the file in next major release
 RESPONSE_BAD_REQUEST_CODE = 400
 RESPONSE_UNAUTHORIZED = 401
 RESPONSE_FORBIDDEN_CODE = 403

modular_sdk/helpers/response_helper.py

@@ -4,6 +4,7 @@
 from modular_sdk.commons.error_helper import RESPONSE_OK_CODE, ERROR_MESSAGE_MAP
 
 
+# TODO remove in next major release
 class AbstractResponseHelper(ABC):
     exception = None
 

modular_sdk/services/aws_creds_provider.py

@@ -3,8 +3,10 @@
 import boto3
 from botocore.client import BaseClient
 from datetime import datetime, timedelta
+from botocore.config import Config
 
 from modular_sdk.modular import Modular
+from modular_sdk.commons.constants import Env
 from modular_sdk.commons.time_helper import utc_datetime
 from functools import cached_property
 
@@ -29,6 +31,21 @@ def __init__(self, service_name: str, aws_region: str,
         self._aws_secret_access_key = aws_secret_access_key
         self._aws_session_token = aws_session_token
 
+    @staticmethod
+    def build_default_config() -> Optional[Config]:
+        """
+        Default config for boto3 clients that respects modular proxy env
+        variables
+        """
+        proxy = {}
+        if url := Env.HTTP_PROXY.get():
+            proxy['http'] = url
+        if url := Env.HTTPS_PROXY.get():
+            proxy['https'] = url
+        if proxy:
+            return Config(proxies=proxy)
+        return
+
     @cached_property
     def client(self) -> BaseClient:
         _LOG.info(f'Initializing {self._service_name} boto3 client')
@@ -37,7 +54,8 @@ def client(self) -> BaseClient:
             region_name=self._region_name,
             aws_access_key_id=self._aws_access_key_id,
             aws_secret_access_key=self._aws_secret_access_key,
-            aws_session_token=self._aws_session_token
+            aws_session_token=self._aws_session_token,
+            config=self.build_default_config()
         )
 
 
@@ -59,7 +77,7 @@ def get_parameter(self, name):
 
     def __init__(self, service_name: str,
                  region_name: Optional[str] = None):
-        # TODO add role_name to input params
+        # TODO add role_arn to input params
         self._service_name = service_name
         self._region_name = region_name
         self._client: Optional[BaseClient] = None
@@ -113,6 +131,7 @@ def __get__(self, instance, owner) -> BaseClient:
                       f'ModularAssumeRoleClient descriptor for region {r}')
             self._client = self.get_session().client(
                 service_name=self._service_name,
-                region_name=r
+                region_name=r,
+                config=AWSCredentialsProvider.build_default_config()
             )
         return self._client

modular_sdk/services/sts_service.py

@@ -3,8 +3,7 @@
 from datetime import datetime, timedelta
 from functools import cached_property
 from time import time
-from typing import Optional, TypedDict, List, Tuple, Iterable, Generator
-import boto3
+from typing import Optional, TypedDict, List, Tuple, Generator
 
 from botocore.exceptions import ClientError
 
@@ -117,12 +116,13 @@ def assume_roles_chain(self, payloads: List[AssumeRolePayload]
                 error_message = f'Error while assuming {arn} from chain'
                 _LOG.error(f'{error_message}: {e}')
                 raise ConnectionAbortedError(error_message) from e
-            _sts = boto3.client(
-                'sts',
+            _sts = AWSCredentialsProvider(
+                service_name='sts',
+                aws_region=self._region_name,
                 aws_access_key_id=creds['AccessKeyId'],
                 aws_secret_access_key=creds['SecretAccessKey'],
                 aws_session_token=creds['SessionToken'],
-            )
+            ).client
         # creds variable will exist, ignore warning
         return {
             'aws_access_key_id': creds.get('AccessKeyId'),

pyproject.toml

@@ -5,7 +5,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "modular_sdk"
-version = "6.1.0"
+version = "6.1.1"
 authors = [
     {name = "EPAM Systems", email = "support@syndicate.team"}
 ]