samples.instanceadmin.test_instanceadmin: test_add_and_delete_cluster failed

[flaky-bot]

Note: #353 was also for this test, but it was closed more than 10 days ago. So, I didn't mark it flaky.

---

commit: 50901c7
buildURL: Build Status, Sponge
status: failed

Test output

args = (name: "operations/projects/python-docs-samples-tests/instances/instanceadmin-684-1633172606/locations/us-central1-f/operations/1930145908071198971"
,)
kwargs = {'metadata': [('x-goog-request-params', 'name=operations/projects/python-docs-samples-tests/instances/instanceadmin-68...-f/operations/1930145908071198971'), ('x-goog-api-client', 'gl-python/3.8.8 grpc/1.41.0 gax/1.31.3')], 'timeout': 20.0}
@six.wraps(callable_)
def error_remapped_callable(*args, **kwargs):
    try:


      return callable_(*args, **kwargs)


.nox/py-3-8/lib/python3.8/site-packages/google/api_core/grpc_helpers.py:67:

self = <grpc._channel._UnaryUnaryMultiCallable object at 0x7f75e5aae130>

request = name: "operations/projects/python-docs-samples-tests/instances/instanceadmin-684-1633172606/locations/us-central1-f/operations/1930145908071198971"
timeout = 20.0

metadata = [('x-goog-request-params', 'name=operations/projects/python-docs-samples-tests/instances/instanceadmin-684-1633172606/locations/us-central1-f/operations/1930145908071198971'), ('x-goog-api-client', 'gl-python/3.8.8 grpc/1.41.0 gax/1.31.3')]

credentials = None, wait_for_ready = None, compression = None
def __call__(self,
             request,
             timeout=None,
             metadata=None,
             credentials=None,
             wait_for_ready=None,
             compression=None):
    state, call, = self._blocking(request, timeout, metadata, credentials,
                                  wait_for_ready, compression)


  return _end_unary_response_blocking(state, call, False, None)


.nox/py-3-8/lib/python3.8/site-packages/grpc/_channel.py:946:

state = <grpc._channel._RPCState object at 0x7f75e7bae370>

call = <grpc._cython.cygrpc.SegregatedCall object at 0x7f75e635c6c0>

with_call = False, deadline = None
def _end_unary_response_blocking(state, call, with_call, deadline):
    if state.code is grpc.StatusCode.OK:
        if with_call:
            rendezvous = _MultiThreadedRendezvous(state, call, None, deadline)
            return state.response, rendezvous
        else:
            return state.response
    else:


      raise _InactiveRpcError(state)


E           grpc._channel._InactiveRpcError: <_InactiveRpcError of RPC that terminated with:

E           	status = StatusCode.DEADLINE_EXCEEDED

E           	details = "Deadline expired before operation could complete."

E           	debug_error_string = "{"created":"@1633172629.973407816","description":"Error received from peer ipv4:74.125.195.95:443","file":"src/core/lib/surface/call.cc","file_line":1069,"grpc_message":"Deadline expired before operation could complete.","grpc_status":4}"

E           >
.nox/py-3-8/lib/python3.8/site-packages/grpc/_channel.py:849: _InactiveRpcError
The above exception was the direct cause of the following exception:
capsys = <_pytest.capture.CaptureFixture object at 0x7f75e7b8a100>

dispose_of = <function dispose_of..disposal at 0x7f75e7b7ed30>
def test_add_and_delete_cluster(capsys, dispose_of):
    dispose_of(INSTANCE)

    # This won't work, because the instance isn't created yet
    instanceadmin.add_cluster(PROJECT, INSTANCE, CLUSTER2)
    out = capsys.readouterr().out
    assert f"Instance {INSTANCE} does not exist" in out

    # Get the instance created


  instanceadmin.run_instance_operations(PROJECT, INSTANCE, CLUSTER1)


test_instanceadmin.py:128:

instanceadmin.py:77: in run_instance_operations

operation.result(timeout=60)

.nox/py-3-8/lib/python3.8/site-packages/google/api_core/future/polling.py:130: in result

self._blocking_poll(timeout=timeout, **kwargs)

.nox/py-3-8/lib/python3.8/site-packages/google/api_core/future/polling.py:108: in blocking_poll

retry(self._done_or_raise)(**kwargs)

.nox/py-3-8/lib/python3.8/site-packages/google/api_core/retry.py:286: in retry_wrapped_func

return retry_target(

.nox/py-3-8/lib/python3.8/site-packages/google/api_core/retry.py:189: in retry_target

return target()

.nox/py-3-8/lib/python3.8/site-packages/google/api_core/future/polling.py:86: in _done_or_raise

if not self.done(**kwargs):

.nox/py-3-8/lib/python3.8/site-packages/google/api_core/operation.py:170: in done

self._refresh_and_update(retry)

.nox/py-3-8/lib/python3.8/site-packages/google/api_core/operation.py:158: in _refresh_and_update

self._operation = self._refresh(retry=retry)

.nox/py-3-8/lib/python3.8/site-packages/google/api_core/operations_v1/operations_client.py:142: in get_operation

return self._get_operation(

.nox/py-3-8/lib/python3.8/site-packages/google/api_core/gapic_v1/method.py:145: in call

return wrapped_func(*args, **kwargs)

.nox/py-3-8/lib/python3.8/site-packages/google/api_core/retry.py:286: in retry_wrapped_func

return retry_target(

.nox/py-3-8/lib/python3.8/site-packages/google/api_core/retry.py:189: in retry_target

return target()

.nox/py-3-8/lib/python3.8/site-packages/google/api_core/timeout.py:214: in func_with_timeout

return func(*args, **kwargs)

.nox/py-3-8/lib/python3.8/site-packages/google/api_core/grpc_helpers.py:69: in error_remapped_callable

six.raise_from(exceptions.from_grpc_error(exc), exc)

value = None

from_value = <_InactiveRpcError of RPC that terminated with:

status = StatusCode.DEADLINE_EXCEEDED

details = "Deadline expired be...urface/call.cc","file_line":1069,"grpc_message":"Deadline expired before operation could complete.","grpc_status":4}"



???

E   google.api_core.exceptions.DeadlineExceeded: 504 Deadline expired before operation could complete.

:3: DeadlineExceeded

[flaky-bot]

Looks like this issue is flaky. 😟

I'm going to leave this open and stop commenting.

A human should fix and close this.

---

When run at the same commit (50901c7), this test passed in one build (Build Status, Sponge) and failed in another build (Build Status, Sponge).

[tseaver]

Failed today.