[automl] fix: retry vision_object_detection_predict.predict (#3492)

fixes #3415

Also setting a hard deadline for model deployment.

automl/cloud-client/requirements-test.txt

@@ -1 +1,2 @@
+backoff==1.10.0
 pytest==5.3.2

automl/cloud-client/vision_object_detection_predict_test.py

@@ -14,6 +14,8 @@
 
 import os
 
+import backoff
+from google.api_core.exceptions import DeadlineExceeded
 from google.cloud import automl
 import pytest
 
@@ -32,12 +34,17 @@ def verify_model_state():
  if model.deployment_state == automl.enums.Model.DeploymentState.UNDEPLOYED:
  # Deploy model if it is not deployed
  response = client.deploy_model(model_full_id)
- response.result()
+ response.result(600) # 10 minutes
 
 
 def test_vision_object_detection_predict(capsys, verify_model_state):
- verify_model_state
  file_path = "resources/salad.jpg"
- vision_object_detection_predict.predict(PROJECT_ID, MODEL_ID, file_path)
+
+ # Retry the sample upon DeadlineExceeded, with a hard deadline of 5 mins.
+ @backoff.on_exception(backoff.expo, DeadlineExceeded, max_time=300)
+ def run_sample():
+ vision_object_detection_predict.predict(PROJECT_ID, MODEL_ID, file_path)
+
+ run_sample()
  out, _ = capsys.readouterr()
  assert "Predicted class name:" in out