diff --git a/luigi/contrib/s3.py b/luigi/contrib/s3.py index d01995fb79..4d77766756 100644 --- a/luigi/contrib/s3.py +++ b/luigi/contrib/s3.py @@ -73,15 +73,35 @@ class S3Client(FileSystem): boto-powered S3 client. """ + _s3 = None + def __init__(self, aws_access_key_id=None, aws_secret_access_key=None, **kwargs): + from boto.s3.key import Key + options = self._get_s3_config() + options.update(kwargs) + if aws_access_key_id: + options['aws_access_key_id'] = aws_access_key_id + if aws_secret_access_key: + options['aws_secret_access_key'] = aws_secret_access_key + + self.Key = Key + self._options = options + + @property + def s3(self): # only import boto when needed to allow top-lvl s3 module import import boto import boto.s3.connection - from boto.s3.key import Key - options = self._get_s3_config() - options.update(kwargs) + options = dict(self._options) + + if self._s3: + return self._s3 + + aws_access_key_id = options.get('aws_access_key_id') + aws_secret_access_key = options.get('aws_secret_access_key') + # Removing key args would break backwards compability role_arn = options.get('aws_role_arn') role_session_name = options.get('aws_role_session_name') @@ -97,22 +117,18 @@ def __init__(self, aws_access_key_id=None, aws_secret_access_key=None, aws_access_key_id = assumed_role.credentials.access_key aws_session_token = assumed_role.credentials.session_token - else: - if not aws_access_key_id: - aws_access_key_id = options.get('aws_access_key_id') - - if not aws_secret_access_key: - aws_secret_access_key = options.get('aws_secret_access_key') - for key in ['aws_access_key_id', 'aws_secret_access_key', 'aws_role_session_name', 'aws_role_arn']: if key in options: options.pop(key) - - self.s3 = boto.s3.connection.S3Connection(aws_access_key_id, - aws_secret_access_key, - security_token=aws_session_token, - **options) - self.Key = Key + self._s3 = boto.s3.connection.S3Connection(aws_access_key_id, + aws_secret_access_key, + security_token=aws_session_token, + **options) + return self._s3 + + @s3.setter + def s3(self, value): + self._s3 = value def exists(self, path): """