Properly Shutdown Timer / Transfer / Replication Queue Processor When Shard Lost

[wxing1292]

time="2018-06-20T19:34:01Z" level=error msg="helpers.go:38 Persistent store operation failure. Operation Details: {updateCondition: 258}" Service=cadence-history deployment-name=autobots_prod_dca1a execution-id=5117342d-e419-4a13-913e-bc65c9edc41b-autobots-2579469 run-id=101b245c-ab1c-4601-bf06-84e279831d6f shard-id=9137 store-operation=update-wf-execution wf-component=history-cache wf-error="Failed to update workflow execution.  Request RangeID: -1, Actual RangeID: 49" wf-event-id=2010
time="2018-06-20T19:34:01Z" level=error msg="helpers.go:38 Persistent store operation failure. Operation Details: {updateCondition: 236}" Service=cadence-history deployment-name=autobots_prod_dca1a execution-id=5117342d-e419-4a13-913e-bc65c9edc41b-autobots-2333709 run-id=4b3a0ff8-a0c2-4293-9aec-092b2ef8352d shard-id=11199 store-operation=update-wf-execution wf-component=history-cache wf-error="Failed to update workflow execution.  Request RangeID: -1, Actual RangeID: 42" wf-event-id=2010
time="2018-06-20T19:34:01Z" level=error msg="helpers.go:38 Persistent store operation failure. Operation Details: {updateCondition: 307}" Service=cadence-history deployment-name=autobots_prod_dca1a execution-id=5117342d-e419-4a13-913e-bc65c9edc41b-autobots-430568 run-id=297b4f05-e40d-4dd2-bb36-581211dee8d0 shard-id=443 store-operation=update-wf-execution wf-component=history-cache wf-error="Failed to update workflow execution.  Request RangeID: -1, Actual RangeID: 61" wf-event-id=2010
time="2018-06-20T19:34:01Z" level=error msg="helpers.go:38 Persistent store operation failure. Operation Details: {updateCondition: 477}" Service=cadence-history deployment-name=autobots_prod_dca1a execution-id=5117342d-e419-4a13-913e-bc65c9edc41b-autobots-251930 run-id=44e14666-70bb-4de0-be61-02451ec9ee9e shard-id=13229 store-operation=update-wf-execution wf-component=history-cache wf-error="Failed to update workflow execution.  Request RangeID: -1, Actual RangeID: 50" wf-event-id=2010
time="2018-06-20T19:34:01Z" level=error msg="helpers.go:38 Persistent store operation failure. Operation Details: {updateCondition: 120}" Service=cadence-history deployment-name=autobots_prod_dca1a execution-id=5117342d-e419-4a13-913e-bc65c9edc41b-autobots-2906375 run-id=af0c220e-326e-4574-b826-56a9980e0e88 shard-id=3497 store-operation=update-wf-execution wf-component=history-cache wf-error="Failed to update workflow execution.  Request RangeID: -1, Actual RangeID: 54" wf-event-id=2010
time="2018-06-20T19:34:01Z" level=error msg="helpers.go:38 Persistent store operation failure. Operation Details: {updateCondition: 264}" Service=cadence-history deployment-name=autobots_prod_dca1a execution-id=5117342d-e419-4a13-913e-bc65c9edc41b-autobots-41406 run-id=da072fa2-277a-47b0-8160-c0dd153da718 shard-id=11823 store-operation=update-wf-execution wf-component=history-cache wf-error="Failed to update workflow execution.  Request RangeID: -1, Actual RangeID: 32" wf-event-id=2010
time="2018-06-20T19:34:01Z" level=error msg="helpers.go:38 Persistent store operation failure. Operation Details: {updateCondition: 194}" Service=cadence-history deployment-name=autobots_prod_dca1a execution-id=5117342d-e419-4a13-913e-bc65c9edc41b-autobots-2358149 run-id=2f0aa854-f2dc-4c5a-915e-535858526f55 shard-id=15861 store-operation=update-wf-execution wf-component=history-cache wf-error="Failed to update workflow execution.  Request RangeID: -1, Actual RangeID: 48" wf-event-id=2010
time="2018-06-20T19:34:01Z" level=fatal msg="helpers.go:64 Retry count exceeded for timer taskID: 27262982.  Error: <nil>" Service=cadence-history deployment-name=autobots_prod_dca1a shard-id=6676 wf-cluster=autobots_prod_dca1a wf-component=timer-queue-processor wf-error="<nil>" wf-event-id=9001