[R] Upgrade moto to 2.0.2 (#1718, PR #2886)

attic/cart/test_cart_export_service.py

@@ -18,9 +18,6 @@
 from azul.service.collection_data_access import (
  CollectionDataAccess,
 )
-from retorts import (
- ResponsesHelper,
-)
 
 
 @skipIf(config.dss_endpoint is None,
@@ -89,7 +86,6 @@ def mock_get_paginable_cart_items(**kwargs):
  self.assertEqual(2, len(content_items))
  self.assertIn(expected_content_item_1, content_items)
 
- @responses.activate
  @patch('azul.deployment.aws.dynamo')
  def test_export_create_new_collection(self, _dynamodb_client):
  expected_collection = dict(uuid='abc', version='123')
@@ -98,7 +94,7 @@ def test_export_create_new_collection(self, _dynamodb_client):
  service = CartExportService()
  with patch.object(service.cart_item_manager, 'get_cart', side_effect=[dict(CartName='abc123')]):
  with patch.object(service, 'get_content', side_effect=[expected_get_content_result]):
- with ResponsesHelper() as helper:
+ with responses.RequestsMock() as helper:
  helper.add(responses.Response(responses.PUT,
  CollectionDataAccess.endpoint_url('collections'),
  status=201,
@@ -114,15 +110,14 @@ def test_export_create_new_collection(self, _dynamodb_client):
  self.assertEqual(expected_get_content_result['resume_token'], result['resume_token'])
  self.assertEqual(len(expected_get_content_result['items']), result['exported_item_count'])
 
- @responses.activate
  @patch('azul.deployment.aws.dynamo')
  def test_export_append_items_to_collection_ok(self, _dynamodb_client):
  expected_collection = dict(uuid='abc', version='123')
  expected_get_content_result = dict(resume_token='rt1',
  items=[1, 2, 3, 4]) # NOTE: This is just for the test.
  service = CartExportService()
  with patch.object(service, 'get_content', side_effect=[expected_get_content_result]):
- with ResponsesHelper() as helper:
+ with responses.RequestsMock() as helper:
  helper.add(responses.Response(
  responses.PATCH,
  CollectionDataAccess.endpoint_url('collections', expected_collection['uuid']),
@@ -139,7 +134,6 @@ def test_export_append_items_to_collection_ok(self, _dynamodb_client):
  self.assertEqual(expected_get_content_result['resume_token'], result['resume_token'])
  self.assertEqual(len(expected_get_content_result['items']), result['exported_item_count'])
 
- @responses.activate
  @patch('azul.deployment.aws.dynamo')
  def test_export_append_items_to_collection_raises_expired_access_token_error(self, _dynamodb_client):
  expected_collection = dict(uuid='abc', version='123')
@@ -148,7 +142,7 @@ def test_export_append_items_to_collection_raises_expired_access_token_error(sel
  service = CartExportService()
  with self.assertRaises(ExpiredAccessTokenError):
  with patch.object(service, 'get_content', side_effect=[expected_get_content_result]):
- with ResponsesHelper() as helper:
+ with responses.RequestsMock() as helper:
  url = CollectionDataAccess.endpoint_url('collections', expected_collection['uuid'])
  helper.add(responses.Response(responses.PATCH, url, status=401, json=dict(code='abc')))
  service.export(export_id='export1',

requirements.dev.trans.txt

@@ -1,28 +1,26 @@
 argh==0.26.2
 arrow==0.12.1
 aws-sam-translator==1.35.0
-aws-xray-sdk==2.7.0
-cfn-lint==0.48.2
+aws-xray-sdk==2.8.0
+cfn-lint==0.49.2
 click==7.1.2
-colorama==0.4.1
-configargparse==1.4
-datetime==4.3
+colorama==0.4.3
+configargparse==1.4.1
 decorator==4.4.2
 docker-pycreds==0.4.0
 ecdsa==0.14.1
 enum-compat==0.0.3
 flask-basicauth==0.2.0
-flask==1.1.2
+flask==2.0.0
 geventhttpclient==1.4.4
 google-auth-httplib2==0.1.0
-greenlet==1.0.0
+greenlet==1.1.0
 httplib2==0.19.1
-itsdangerous==1.1.0
-jsondiff==1.1.2
+itsdangerous==2.0.1
+jsondiff==1.3.0
 jsonpatch==1.32
 junit-xml==1.9
 mccabe==0.6.1
-mock==4.0.3
 msgpack==1.0.2
 mypy-extensions==0.4.3
 networkx==2.5.1
@@ -39,7 +37,8 @@ smmap2==3.0.1
 smmap==4.0.0
 sshpubkeys==3.3.1
 uritemplate==3.0.1
-websocket-client==0.58.0
+websocket-client==1.0.0
 xmltodict==0.12.0
+zipp==3.4.1
 zope.event==4.5.0
-zope.interface==5.3.0
+zope.interface==5.4.0

requirements.dev.txt

@@ -1,4 +1,4 @@
-awscli==1.16.283
+awscli==1.19.33
 git+git://github.com/DataBiosphere/azul-chalice@1.18.0+5#egg=chalice
 coverage==5.1
 docker==3.4.1
@@ -9,7 +9,9 @@ gitpython==2.1.11
 google-api-python-client==1.7.8
 jsonschema==3.2.0
 locust==1.2.3
-moto==1.3.13 # 1.3.14 does not work, possibly related to https://github.com/spulec/moto/issues/2567
+# FIXME: Update Moto when decorator ordering issue is resovled
+# https://github.com/DataBiosphere/azul/issues/3035
+moto[all]==2.0.6
 openapi-spec-validator==0.2.8
 pygithub==1.43.5
 pytest==3.2.3

requirements.trans.txt

@@ -1,37 +1,37 @@
-argcomplete==1.12.2
+argcomplete==1.12.3
 atomicwrites==1.4.0
 bagit==1.7.0
-cachetools==4.2.1
+cachetools==4.2.2
 certifi==2020.12.5
 chardet==3.0.4
 commonmark==0.9.1
 crc32c==2.2
 dcplib==2.1.2
 docutils==0.15.2
-google-api-core==1.26.3
+google-api-core==1.28.0
 google-auth-oauthlib==0.4.4
 google-cloud-core==1.6.0
 google-resumable-media==0.5.1
 googleapis-common-protos==1.53.0
-idna==2.8
-jinja2==2.11.3
+idna==2.10
+jinja2==3.0.1
 jsonpointer==1.14
 jsonschema==3.2.0
-markupsafe==1.1.1
+markupsafe==2.0.1
 oauthlib==3.1.0
 orderedmultidict==1.0.1
 packaging==20.9
 proto-plus==1.18.1
-protobuf==3.15.8
+protobuf==3.17.0
 puremagic==1.4
 pycparser==2.20
 pyopenssl==20.0.1
 pyparsing==2.4.7
 python-dateutil==2.8.1
 pytz==2021.1
 requests-oauthlib==1.3.0
-s3transfer==0.2.1
-six==1.15.0
+s3transfer==0.3.7
+six==1.16.0
 tenacity==5.0.4
 text-unidecode==1.3
 tqdm==4.60.0

requirements.txt

@@ -3,9 +3,9 @@ attrs==19.3.0
 aws-requests-auth==0.4.2
 bdbag==1.5.1
 boltons==20.2.0
-boto3==1.10.19
+boto3==1.17.29
 boto==2.49.0
-botocore==1.13.19
+botocore==1.20.33
 cffi==1.14.0
 cryptography==2.3.1
 deprecated==1.2.9

src/azul/portal_service.py

@@ -14,6 +14,10 @@
  cast,
 )
 
+from botocore.exceptions import (
+ ClientError,
+)
+
 from azul import (
  cached_property,
  config,
@@ -197,7 +201,17 @@ def _read_db(self, version: str) -> JSONs:
  Key=self.object_key,
  VersionId=version)
  except self.client.exceptions.NoSuchKey:
+ # We hypothesize that when S3 was only eventually consistent,
+ # NoSuchKey would have been raised when an object had been
+ # created but hadn't materialized yet …
  raise NoSuchObjectVersion(version)
+ except ClientError as e:
+ # … and that NoSuchVersion would have been raised when the object had
+ # been overwritten but the overwrite hadn't materialized yet.
+ if e.response['Error']['Code'] == 'NoSuchVersion':
+ raise NoSuchObjectVersion(version)
+ else:
+ raise
  else:
  json_bytes = response['Body'].read()
  return json.loads(json_bytes.decode())

src/azul/service/storage_service.py

@@ -116,8 +116,11 @@ def get_presigned_url(self, key: str, file_name: Optional[str] = None) -> str:
  **({} if file_name is None else {'ResponseContentDisposition': f'attachment;filename="{file_name}"'})
  })
 
- def create_bucket(self, bucket_name: str = None):
- self.client.create_bucket(Bucket=(bucket_name or self.bucket_name))
+ def create_bucket(self, bucket_name: Optional[str] = None):
+ self.client.create_bucket(Bucket=(bucket_name or self.bucket_name),
+ CreateBucketConfiguration={
+ 'LocationConstraint': config.region
+ })
 
  def put_object_tagging(self, object_key: str, tagging: Tagging = None):
  deadline = time.time() + 60

test/azul_test_case.py

@@ -18,6 +18,10 @@
  Credentials,
 )
 import botocore.session
+from moto.core.models import (
+ ACCOUNT_ID,
+ moto_api_backend,
+)
 
 from azul import (
  CatalogName,
@@ -153,11 +157,6 @@ def tearDownClass(cls) -> None:
  super().tearDownClass()
 
  def setUp(self) -> None:
- # FIXME: remove local import and update moto
- # https://github.com/DataBiosphere/azul/issues/1718
- from moto.core import (
- moto_api_backend,
- )
  super().setUp()
  # Moto backends are reset to ensure no resources are left over if a test
  # fails to clean up after itself.
@@ -204,12 +203,7 @@ def _mock_aws_account_id(cls):
  # Set AZUL_AWS_ACCOUNT_ID to what the Moto is using. This circumvents
  # assertion errors in azul.deployment.aws.account.
  cls._aws_account_id = os.environ['AZUL_AWS_ACCOUNT_ID']
- # The fake Moto account ID is defined as a constant in a Moto module
- # but we cannot import any Moto modules since doing so has a bad side
- # effect: https://github.com/spulec/moto/issues/2058.
- # FIXME: Switch to overriding MOTO_ACCOUNT_ID as part of
- # https://github.com/DataBiosphere/azul/issues/1718
- os.environ['AZUL_AWS_ACCOUNT_ID'] = '123456789012'
+ os.environ['AZUL_AWS_ACCOUNT_ID'] = ACCOUNT_ID
 
  @classmethod
  def _restore_aws_account(cls):