Merge pull request 5755 from hotfix/v4.3.3 into master

ci/shared-build-and-test-stage.yml

@@ -62,6 +62,19 @@ stages:
  displayName: 'Install Python Packages'
 
  - bash: |
+ # TODO: This is workaround for permission issue found on Windows when pylint
+ # try to create 'Cache' folder in 'AppData' folder. This will needs to be
+ # resolved by permission settings when the issue is determined in the future.
+ if [ "$(echo ${{ parameters.imageName }} | grep -i win)" != "" ]; then
+ echo "Is Windows. Create AppData Cache folder for pylint."
+ echo $LOCALAPPDATA
+ pylintAppPath=$LOCALAPPDATA/pylint/pylint/Cache
+ if [ ! -d "$pylintAppPath" ]; then
+ echo "Local AppData pylint folder does not exist. Create one."
+ mkdir -p $LOCALAPPDATA/pylint/pylint/Cache
+ fi
+ fi
+ 
  cd ./fiftyone_pipeline_cloudrequestengine
  pylint --rcfile=.pylintrc ./fiftyone_pipeline_cloudrequestengine
  if [ $? -ne 0 ]; then

ci/shared-variables.yml

@@ -1,6 +1,6 @@
 variables:
 - name: windowsImage
- value: 'vs2017-win2016'
+ value: 'windows-2019'
 - name: linuxImage
  value: 'ubuntu-18.04'
 - name: macImage

fiftyone_pipeline_cloudrequestengine/fiftyone_pipeline_cloudrequestengine/cloudrequestengine.py

@@ -25,6 +25,7 @@
 from fiftyone_pipeline_engines.engine import Engine
 from fiftyone_pipeline_engines.aspectdata_dictionary import AspectDataDictionary
 from .requestclient import RequestClient
+from .cloudrequestexception import CloudRequestException
 from .constants import Constants
 import warnings
 import json
@@ -165,7 +166,7 @@ def get_engine_properties(self):
 
  return flowElementProperties
 
- def validate_response(self, cloudResponse):
+ def validate_response(self, cloudResponse, checkForErrorMessages = True):
 
  """!
  Validate the JSON response from the cloud service.
@@ -179,11 +180,11 @@ def validate_response(self, cloudResponse):
  hasData = cloudResponse.text and cloudResponse.text.strip()
  messages = []
 
- if hasData:
+ if hasData and checkForErrorMessages:
  try:
  jsonResponse = json.loads(cloudResponse.content)
  except:
- raise Exception("Cloud request engine properties list request returned code '" + 
+ raise CloudRequestException("Cloud request engine properties list request returned code '" + 
  str(cloudResponse.status_code) + "' with content '" + cloudResponse.content+ "'")
 
  hasErrors = "errors" in jsonResponse and len(jsonResponse["errors"])
@@ -203,23 +204,29 @@ def validate_response(self, cloudResponse):
  message = Constants.MESSAGE_NO_DATA_IN_RESPONSE.format(cloudResponse.url)
  messages.append(message)
 
+ # If there were no errors returned but the response code was non
+ # success then throw an exception.
+
+ if (len(messages) == 0 and cloudResponse.status_code != 200):
+ message = Constants.MESSAGE_ERROR_CODE_RETURNED.format(self.baseURL, cloudResponse.status_code, json.loads(cloudResponse.content))
+ messages.append(message)
+
  # If there are any errors returned from the cloud service
  # then throw an exception
 
+ headers = None
+
+ if (len(messages) > 0):
+ # Get the response headers.
+ headers = cloudResponse.headers
+
  if (len(messages) > 1):
  exceptionList = [Constants.EXCEPTION_CLOUD_ERRORS_MULTIPLE]
  exceptionList = exceptionList + messages
- raise Exception(exceptionList)
+ raise CloudRequestException(exceptionList, cloudResponse.status_code, headers)
  elif (len(messages) == 1):
  message = Constants.EXCEPTION_CLOUD_ERROR.format(messages[0])
- raise Exception(message)
-
- # If there were no errors returned but the response code was non
- # success then throw an exception.
-
- if (len(messages) == 0 and cloudResponse.status_code != 200):
- raise Exception("Cloud request engine properties list request returned " + \
- str(cloudResponse.status_code))
+ raise CloudRequestException(message, cloudResponse.status_code, headers)
 
  def make_cloud_request(self, type, url, content = None):
 

fiftyone_pipeline_cloudrequestengine/fiftyone_pipeline_cloudrequestengine/cloudrequestexception.py

@@ -0,0 +1,45 @@
+ # *********************************************************************
+ # This Original Work is copyright of 51 Degrees Mobile Experts Limited\
+ # Copyright 2019 51 Degrees Mobile Experts Limited, 5 Charlotte Close,
+ # Caversham, Reading, Berkshire, United Kingdom RG4 7BY\
+ #
+ # This Original Work is licensed under the European Union Public Licence (EUPL) 
+ # v\1\2 and is subject to its terms as set out below\
+ #
+ # If a copy of the EUPL was not distributed with this file, You can obtain
+ # one at https://opensource\org/licenses/EUPL-1\2\
+ #
+ # The 'Compatible Licences' set out in the Appendix to the EUPL (as may be
+ # amended by the European Commission) shall be deemed incompatible for
+ # the purposes of the Work and the provisions of the compatibility
+ # clause in Article 5 of the EUPL shall not apply\
+ # 
+ # If using the Work as, or as part of, a network application, by 
+ # including the attribution notice(s) required under Article 5 of the EUPL
+ # in the end user terms of the application under an appropriate heading, 
+ # such notice(s) shall fulfill the requirements of that article\
+ # ********************************************************************
+
+"""!
+ Exception that can be thrown when the available data does not match that
+ which is expected.
+"""
+class CloudRequestException(Exception):
+
+ """!
+ Constructor for CloudRequestException
+ @type message: string
+ @param message: Exception message string
+ @type httpStatusCode: int
+ @param httpStatusCode: the status code returned in the HTTP response
+ @type responseHeaders: dict
+ @param responseHeaders: the HTTP headers returned in the response 
+ """
+ def __init__(self, message, httpStatusCode = 0, responseHeaders = {}):
+
+ self.message = message
+ self.httpStatusCode = httpStatusCode
+ self.responseHeaders = responseHeaders
+ super().__init__(self.message)
+
+

fiftyone_pipeline_cloudrequestengine/fiftyone_pipeline_cloudrequestengine/constants.py

@@ -57,4 +57,7 @@ class Constants:
  EVIDENCE_OTHER = "other"
 
  # warning message to be shown for conflicted evidences
- WARNING_MESSAGE = "WARNING: '{}:{}' evidence conflicts with "
+ WARNING_MESSAGE = "WARNING: '{}:{}' evidence conflicts with "
+
+ # error message when non-success status is returned.
+ MESSAGE_ERROR_CODE_RETURNED = "Cloud service at '{}' returned status code '{}' with content {}"

fiftyone_pipeline_cloudrequestengine/tests/classes/constants.py

@@ -40,7 +40,7 @@ class Constants():
  "from cloud service at https://cloud.51degrees.com/api/v4/accessibleProperties?resource=nodatakey'"
  noErrorNoSuccessKey = "noErrorNoSuccessKey"
  noErrorNoSuccessResponse = {'device': {'value': 'abc'}}
- noErrorNoSuccessMessage= "Cloud request engine properties list request returned 400"
+ noErrorNoSuccessMessage= "Error returned from 51Degrees cloud service: 'Cloud service at 'https://cloud.51degrees.com/api/v4/' returned status code '400' with content {'device': {'value': 'abc'}}'"
  accessibleSubPropertiesResponse = \
  { \
  'Products': { \

fiftyone_pipeline_cloudrequestengine/tests/classes/mockrequestclient.py

@@ -39,12 +39,17 @@ def request(self, type, url, content, originHeader):
  response.content = json.dumps(Constants.accessibleSubPropertiesResponse)
  elif Constants.invalidKey in url: 
  response.content = json.dumps(Constants.invalidKeyResponse) 
+ response.status_code = 400
+ response.headers = {'header': 'value'}
  elif Constants.noDataKey in url: 
  response.content = json.dumps(Constants.noDataKeyResponse)
- response.url = url 
+ response.url = url
+ response.status_code = 400
+ response.headers = {'header': 'value'} 
  elif Constants.noErrorNoSuccessKey in url:
  response.content = json.dumps(Constants.noErrorNoSuccessResponse) 
- response.status_code = 400 
+ response.status_code = 400
+ response.headers = {'header': 'value'} 
  else:
  response.text = json.dumps(Constants.accessiblePropertiesResponse)
  response.content = json.dumps(Constants.accessiblePropertiesResponse)

fiftyone_pipeline_cloudrequestengine/tests/test_cloudengine.py

@@ -21,6 +21,7 @@
  # ********************************************************************
 
 from fiftyone_pipeline_cloudrequestengine.cloudrequestengine import CloudRequestEngine
+from fiftyone_pipeline_cloudrequestengine.cloudrequestexception import CloudRequestException
 from fiftyone_pipeline_cloudrequestengine.cloudengine import CloudEngine
 from fiftyone_pipeline_core.pipelinebuilder import PipelineBuilder
 from urllib.parse import urlencode
@@ -159,6 +160,27 @@ def test_cloud_get_request_with_sequence_evidence(self):
  except Exception as ex:
  self.assertTrue("Sequence number not present in evidence. this is mandatory" in str(ex))
 
- # Following statement should be uncommented once error
+ # Following statements should be uncommented once error
  # is fixed in cloud
+ # jsonResponse = cloud.make_cloud_request('GET', url, content=None)
  # self.assertTrue(len(jsonResponse["errors"]) == 0)
+
+ def test_HttpDataSetInException(self):
+
+ """!
+ Check that errors from the cloud service will cause the
+ appropriate data to be set in the CloudRequestException.
+ """
+
+ resource_key = "resource_key"
+
+ pipeline = PipelineBuilder()
+
+ try:
+ cloud = CloudRequestEngine({"resource_key" : resource_key}) 
+ pipeline = pipeline.add(cloud).build()
+ self.assertFalse("Expected exception did not occur")
+ except CloudRequestException as ex:
+ self.assertTrue(ex.httpStatusCode > 0, "Status code should not be 0")
+ self.assertIsNotNone(ex.responseHeaders, "Response headers not populated")
+ self.assertTrue(len(ex.responseHeaders) > 0, "Response headers not populated")

fiftyone_pipeline_cloudrequestengine/tests/test_cloudresponse.py

@@ -22,6 +22,7 @@
 
 from classes.cloudrequestengine_testbase import CloudRequestEngineTestsBase
 from fiftyone_pipeline_cloudrequestengine.cloudrequestengine import CloudRequestEngine
+from fiftyone_pipeline_cloudrequestengine.cloudrequestexception import CloudRequestException
 from fiftyone_pipeline_core.pipelinebuilder import PipelineBuilder
 from classes.constants import *
 import json
@@ -101,7 +102,7 @@ def test_validate_error_handling_invalid_resourceKey(self):
  "resource_key" : Constants.invalidKey,
  "http_client" : httpClient
  })
- except Exception as ex:
+ except CloudRequestException as ex:
  exception = ex
 
  self.assertIsNotNone("Expected exception to occur", exception)
@@ -124,7 +125,7 @@ def test_validate_error_handling_nodata(self):
  "resource_key" : Constants.noDataKey,
  "http_client" : httpClient
  })
- except Exception as ex:
+ except CloudRequestException as ex:
  exception = ex
 
  self.assertIsNotNone("Expected exception to occur", exception)
@@ -147,7 +148,7 @@ def test_validate_error_handling_noerror_nosuccess(self):
  "resource_key" : Constants.noErrorNoSuccessKey,
  "http_client" : httpClient
  })
- except Exception as ex:
+ except CloudRequestException as ex:
  exception = ex
 
  self.assertIsNotNone("Expected exception to occur", exception)