From 8fdedfb8636c942d6cb53225b643dbd450c5e418 Mon Sep 17 00:00:00 2001 From: Zijian Date: Wed, 31 Mar 2021 14:39:37 -0700 Subject: [PATCH] Seperate integration test suite definition from actual integration tests (#4098) --- host/{activity_test.go => activityTest.go} | 0 host/{archival_test.go => archivalTest.go} | 0 ...workflow_test.go => cancelworkflowTest.go} | 0 host/clientIntegrationTest.go | 366 ++ host/client_integration_test.go | 341 +- ...inueasnew_test.go => continueasnewTest.go} | 0 host/{decision_test.go => decisionTest.go} | 0 host/elasticsearchTest.go | 1088 +++++ host/elasticsearch_test.go | 1062 ----- host/integrationTest.go | 4078 +++++++++++++++++ host/integration_test.go | 4051 ---------------- host/ndc/nDCIntegrationTest.go | 1802 ++++++++ host/ndc/nDC_integration_test.go | 1775 +------ ..._test.go => replicationIntegrationTest.go} | 0 ...yworkflow_test.go => queryworkflowTest.go} | 0 ...tworkflow_test.go => resetworkflowTest.go} | 0 ...workflow_test.go => signalworkflowTest.go} | 0 host/sizelimitTest.go | 191 + host/sizelimit_test.go | 165 - 19 files changed, 7527 insertions(+), 7392 deletions(-) rename host/{activity_test.go => activityTest.go} (100%) rename host/{archival_test.go => archivalTest.go} (100%) rename host/{cancelworkflow_test.go => cancelworkflowTest.go} (100%) create mode 100644 host/clientIntegrationTest.go rename host/{continueasnew_test.go => continueasnewTest.go} (100%) rename host/{decision_test.go => decisionTest.go} (100%) create mode 100644 host/elasticsearchTest.go create mode 100644 host/integrationTest.go create mode 100644 host/ndc/nDCIntegrationTest.go rename host/ndc/{replication_integration_test.go => replicationIntegrationTest.go} (100%) rename host/{queryworkflow_test.go => queryworkflowTest.go} (100%) rename host/{resetworkflow_test.go => resetworkflowTest.go} (100%) rename host/{signalworkflow_test.go => signalworkflowTest.go} (100%) create mode 100644 host/sizelimitTest.go diff --git a/host/activity_test.go b/host/activityTest.go similarity index 100% rename from host/activity_test.go rename to host/activityTest.go diff --git a/host/archival_test.go b/host/archivalTest.go similarity index 100% rename from host/archival_test.go rename to host/archivalTest.go diff --git a/host/cancelworkflow_test.go b/host/cancelworkflowTest.go similarity index 100% rename from host/cancelworkflow_test.go rename to host/cancelworkflowTest.go diff --git a/host/clientIntegrationTest.go b/host/clientIntegrationTest.go new file mode 100644 index 00000000000..a1b4a5f6c56 --- /dev/null +++ b/host/clientIntegrationTest.go @@ -0,0 +1,366 @@ +// Copyright (c) 2017 Uber Technologies, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package host + +import ( + "bytes" + "context" + "encoding/gob" + "errors" + "fmt" + "reflect" + "strconv" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/require" + "go.uber.org/cadence/.gen/go/cadence/workflowserviceclient" + "go.uber.org/cadence/.gen/go/shared" + "go.uber.org/cadence/activity" + "go.uber.org/cadence/client" + "go.uber.org/cadence/encoded" + cworker "go.uber.org/cadence/worker" + "go.uber.org/cadence/workflow" + "go.uber.org/yarpc" + "go.uber.org/yarpc/transport/tchannel" + "go.uber.org/zap" + + "github.com/uber/cadence/common" + "github.com/uber/cadence/common/log/tag" +) + +func init() { + workflow.Register(testDataConverterWorkflow) + activity.Register(testActivity) + workflow.Register(testParentWorkflow) + workflow.Register(testChildWorkflow) +} + +type ( + ClientIntegrationSuite struct { + // override suite.Suite.Assertions with require.Assertions; this means that s.NotNil(nil) will stop the test, + // not merely log an error + *require.Assertions + IntegrationBase + wfService workflowserviceclient.Interface + wfClient client.Client + worker cworker.Worker + taskList string + } +) + +func (s *ClientIntegrationSuite) SetupSuite() { + s.setupSuite() + + var err error + s.wfService, err = s.buildServiceClient() + if err != nil { + s.Logger.Fatal("Error when build service client", tag.Error(err)) + } + s.wfClient = client.NewClient(s.wfService, s.domainName, nil) + + s.taskList = "client-integration-test-tasklist" + s.worker = cworker.New(s.wfService, s.domainName, s.taskList, cworker.Options{}) + if err := s.worker.Start(); err != nil { + s.Logger.Fatal("Error when start worker", tag.Error(err)) + } +} + +func (s *ClientIntegrationSuite) TearDownSuite() { + s.tearDownSuite() +} + +func (s *ClientIntegrationSuite) buildServiceClient() (workflowserviceclient.Interface, error) { + cadenceClientName := "cadence-client" + cadenceFrontendService := common.FrontendServiceName + hostPort := "127.0.0.1:7104" + if TestFlags.FrontendAddr != "" { + hostPort = TestFlags.FrontendAddr + } + + ch, err := tchannel.NewChannelTransport(tchannel.ServiceName(cadenceClientName)) + if err != nil { + s.Logger.Fatal("Failed to create transport channel", tag.Error(err)) + } + + dispatcher := yarpc.NewDispatcher(yarpc.Config{ + Name: cadenceClientName, + Outbounds: yarpc.Outbounds{ + cadenceFrontendService: {Unary: ch.NewSingleOutbound(hostPort)}, + }, + }) + if dispatcher == nil { + s.Logger.Fatal("No RPC dispatcher provided to create a connection to Cadence Service") + } + if err := dispatcher.Start(); err != nil { + s.Logger.Fatal("Failed to create outbound transport channel", tag.Error(err)) + } + + return workflowserviceclient.New(dispatcher.ClientConfig(cadenceFrontendService)), nil +} + +func (s *ClientIntegrationSuite) SetupTest() { + // Have to define our overridden assertions in the test setup. If we did it earlier, s.T() will return nil + s.Assertions = require.New(s.T()) +} + +// testDataConverter implements encoded.DataConverter using gob +type testDataConverter struct { + NumOfCallToData int // for testing to know testDataConverter is called as expected + NumOfCallFromData int +} + +func (tdc *testDataConverter) ToData(value ...interface{}) ([]byte, error) { + tdc.NumOfCallToData++ + var buf bytes.Buffer + enc := gob.NewEncoder(&buf) + for i, obj := range value { + if err := enc.Encode(obj); err != nil { + return nil, fmt.Errorf( + "unable to encode argument: %d, %v, with gob error: %v", i, reflect.TypeOf(obj), err) + } + } + return buf.Bytes(), nil +} + +func (tdc *testDataConverter) FromData(input []byte, valuePtr ...interface{}) error { + tdc.NumOfCallFromData++ + dec := gob.NewDecoder(bytes.NewBuffer(input)) + for i, obj := range valuePtr { + if err := dec.Decode(obj); err != nil { + return fmt.Errorf( + "unable to decode argument: %d, %v, with gob error: %v", i, reflect.TypeOf(obj), err) + } + } + return nil +} + +func newTestDataConverter() encoded.DataConverter { + return &testDataConverter{} +} + +func testActivity(ctx context.Context, msg string) (string, error) { + return "hello_" + msg, nil +} + +func testDataConverterWorkflow(ctx workflow.Context, tl string) (string, error) { + ao := workflow.ActivityOptions{ + ScheduleToStartTimeout: 20 * time.Second, + StartToCloseTimeout: 40 * time.Second, + } + ctx = workflow.WithActivityOptions(ctx, ao) + + var result string + err := workflow.ExecuteActivity(ctx, testActivity, "world").Get(ctx, &result) + if err != nil { + return "", err + } + + // use another converter to run activity, + // with new taskList so that worker with same data converter can properly process tasks. + var result1 string + ctx1 := workflow.WithDataConverter(ctx, newTestDataConverter()) + ctx1 = workflow.WithTaskList(ctx1, tl) + err1 := workflow.ExecuteActivity(ctx1, testActivity, "world1").Get(ctx1, &result1) + if err1 != nil { + return "", err1 + } + return result + "," + result1, nil +} + +func (s *ClientIntegrationSuite) startWorkerWithDataConverter(tl string, dataConverter encoded.DataConverter) cworker.Worker { + opts := cworker.Options{} + if dataConverter != nil { + opts.DataConverter = dataConverter + } + worker := cworker.New(s.wfService, s.domainName, tl, opts) + if err := worker.Start(); err != nil { + s.Logger.Fatal("Error when start worker with data converter", tag.Error(err)) + } + return worker +} + +func (s *ClientIntegrationSuite) TestClientDataConverter() { + tl := "client-integration-data-converter-activity-tasklist" + dc := newTestDataConverter() + worker := s.startWorkerWithDataConverter(tl, dc) + defer worker.Stop() + + id := "client-integration-data-converter-workflow" + workflowOptions := client.StartWorkflowOptions{ + ID: id, + TaskList: s.taskList, + ExecutionStartToCloseTimeout: 60 * time.Second, + } + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + we, err := s.wfClient.ExecuteWorkflow(ctx, workflowOptions, testDataConverterWorkflow, tl) + if err != nil { + s.Logger.Fatal("Start workflow with err", tag.Error(err)) + } + s.NotNil(we) + s.True(we.GetRunID() != "") + + var res string + err = we.Get(ctx, &res) + s.NoError(err) + s.Equal("hello_world,hello_world1", res) + + // to ensure custom data converter is used, this number might be different if client changed. + d := dc.(*testDataConverter) + s.Equal(1, d.NumOfCallToData) + s.Equal(1, d.NumOfCallFromData) +} + +func (s *ClientIntegrationSuite) TestClientDataConverter_Failed() { + tl := "client-integration-data-converter-activity-failed-tasklist" + worker := s.startWorkerWithDataConverter(tl, nil) // mismatch of data converter + defer worker.Stop() + + id := "client-integration-data-converter-failed-workflow" + workflowOptions := client.StartWorkflowOptions{ + ID: id, + TaskList: s.taskList, + ExecutionStartToCloseTimeout: 60 * time.Second, + } + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + we, err := s.wfClient.ExecuteWorkflow(ctx, workflowOptions, testDataConverterWorkflow, tl) + if err != nil { + s.Logger.Fatal("Start workflow with err", tag.Error(err)) + } + s.NotNil(we) + s.True(we.GetRunID() != "") + + var res string + err = we.Get(ctx, &res) + s.Error(err) + + // Get history to make sure only the 2nd activity is failed because of mismatch of data converter + iter := s.wfClient.GetWorkflowHistory(ctx, id, we.GetRunID(), false, 0) + completedAct := 0 + failedAct := 0 + for iter.HasNext() { + event, err := iter.Next() + s.Nil(err) + if event.GetEventType() == shared.EventTypeActivityTaskCompleted { + completedAct++ + } + if event.GetEventType() == shared.EventTypeActivityTaskFailed { + failedAct++ + attr := event.ActivityTaskFailedEventAttributes + s.True(strings.HasPrefix(string(attr.Details), "unable to decode the activity function input bytes with error")) + } + } + s.Equal(1, completedAct) + s.Equal(1, failedAct) +} + +var childTaskList = "client-integration-data-converter-child-tasklist" + +func testParentWorkflow(ctx workflow.Context) (string, error) { + logger := workflow.GetLogger(ctx) + execution := workflow.GetInfo(ctx).WorkflowExecution + childID := fmt.Sprintf("child_workflow:%v", execution.RunID) + cwo := workflow.ChildWorkflowOptions{ + WorkflowID: childID, + ExecutionStartToCloseTimeout: time.Minute, + } + ctx = workflow.WithChildOptions(ctx, cwo) + var result string + err := workflow.ExecuteChildWorkflow(ctx, testChildWorkflow, 0, 3).Get(ctx, &result) + if err != nil { + logger.Error("Parent execution received child execution failure.", zap.Error(err)) + return "", err + } + + childID1 := fmt.Sprintf("child_workflow1:%v", execution.RunID) + cwo1 := workflow.ChildWorkflowOptions{ + WorkflowID: childID1, + ExecutionStartToCloseTimeout: time.Minute, + TaskList: childTaskList, + } + ctx1 := workflow.WithChildOptions(ctx, cwo1) + ctx1 = workflow.WithDataConverter(ctx1, newTestDataConverter()) + var result1 string + err1 := workflow.ExecuteChildWorkflow(ctx1, testChildWorkflow, 0, 2).Get(ctx1, &result1) + if err1 != nil { + logger.Error("Parent execution received child execution 1 failure.", zap.Error(err1)) + return "", err1 + } + + res := fmt.Sprintf("Complete child1 %s times, complete child2 %s times", result, result1) + logger.Info("Parent execution completed.", zap.String("Result", res)) + return res, nil +} + +func testChildWorkflow(ctx workflow.Context, totalCount, runCount int) (string, error) { + logger := workflow.GetLogger(ctx) + logger.Info("Child workflow execution started.") + if runCount <= 0 { + logger.Error("Invalid valid for run count.", zap.Int("RunCount", runCount)) + return "", errors.New("invalid run count") + } + + totalCount++ + runCount-- + if runCount == 0 { + result := fmt.Sprintf("Child workflow execution completed after %v runs", totalCount) + logger.Info("Child workflow completed.", zap.String("Result", result)) + return strconv.Itoa(totalCount), nil + } + + logger.Info("Child workflow starting new run.", zap.Int("RunCount", runCount), zap.Int("TotalCount", + totalCount)) + return "", workflow.NewContinueAsNewError(ctx, testChildWorkflow, totalCount, runCount) +} + +func (s *ClientIntegrationSuite) TestClientDataConverter_WithChild() { + dc := newTestDataConverter() + worker := s.startWorkerWithDataConverter(childTaskList, dc) + defer worker.Stop() + + id := "client-integration-data-converter-with-child-workflow" + workflowOptions := client.StartWorkflowOptions{ + ID: id, + TaskList: s.taskList, + ExecutionStartToCloseTimeout: 60 * time.Second, + } + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + we, err := s.wfClient.ExecuteWorkflow(ctx, workflowOptions, testParentWorkflow) + if err != nil { + s.Logger.Fatal("Start workflow with err", tag.Error(err)) + } + s.NotNil(we) + s.True(we.GetRunID() != "") + + var res string + err = we.Get(ctx, &res) + s.NoError(err) + s.Equal("Complete child1 3 times, complete child2 2 times", res) + + // to ensure custom data converter is used, this number might be different if client changed. + d := dc.(*testDataConverter) + s.Equal(3, d.NumOfCallToData) + s.Equal(2, d.NumOfCallFromData) +} diff --git a/host/client_integration_test.go b/host/client_integration_test.go index e22cccc3136..faf2ad1785e 100644 --- a/host/client_integration_test.go +++ b/host/client_integration_test.go @@ -1,4 +1,4 @@ -// Copyright (c) 2017 Uber Technologies, Inc. +// Copyright (c) 2016 Uber Technologies, Inc. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -21,53 +21,10 @@ package host import ( - "bytes" - "context" - "encoding/gob" - "errors" "flag" - "fmt" - "reflect" - "strconv" - "strings" "testing" - "time" - "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" - "go.uber.org/cadence/.gen/go/cadence/workflowserviceclient" - "go.uber.org/cadence/.gen/go/shared" - "go.uber.org/cadence/activity" - "go.uber.org/cadence/client" - "go.uber.org/cadence/encoded" - cworker "go.uber.org/cadence/worker" - "go.uber.org/cadence/workflow" - "go.uber.org/yarpc" - "go.uber.org/yarpc/transport/tchannel" - "go.uber.org/zap" - - "github.com/uber/cadence/common" - "github.com/uber/cadence/common/log/tag" -) - -func init() { - workflow.Register(testDataConverterWorkflow) - activity.Register(testActivity) - workflow.Register(testParentWorkflow) - workflow.Register(testChildWorkflow) -} - -type ( - ClientIntegrationSuite struct { - // override suite.Suite.Assertions with require.Assertions; this means that s.NotNil(nil) will stop the test, - // not merely log an error - *require.Assertions - IntegrationBase - wfService workflowserviceclient.Interface - wfClient client.Client - worker cworker.Worker - taskList string - } ) func TestClientIntegrationSuite(t *testing.T) { @@ -89,299 +46,3 @@ func TestClientIntegrationSuite(t *testing.T) { suite.Run(t, s) } -func (s *ClientIntegrationSuite) SetupSuite() { - s.setupSuite() - - var err error - s.wfService, err = s.buildServiceClient() - if err != nil { - s.Logger.Fatal("Error when build service client", tag.Error(err)) - } - s.wfClient = client.NewClient(s.wfService, s.domainName, nil) - - s.taskList = "client-integration-test-tasklist" - s.worker = cworker.New(s.wfService, s.domainName, s.taskList, cworker.Options{}) - if err := s.worker.Start(); err != nil { - s.Logger.Fatal("Error when start worker", tag.Error(err)) - } -} - -func (s *ClientIntegrationSuite) TearDownSuite() { - s.tearDownSuite() -} - -func (s *ClientIntegrationSuite) buildServiceClient() (workflowserviceclient.Interface, error) { - cadenceClientName := "cadence-client" - cadenceFrontendService := common.FrontendServiceName - hostPort := "127.0.0.1:7104" - if TestFlags.FrontendAddr != "" { - hostPort = TestFlags.FrontendAddr - } - - ch, err := tchannel.NewChannelTransport(tchannel.ServiceName(cadenceClientName)) - if err != nil { - s.Logger.Fatal("Failed to create transport channel", tag.Error(err)) - } - - dispatcher := yarpc.NewDispatcher(yarpc.Config{ - Name: cadenceClientName, - Outbounds: yarpc.Outbounds{ - cadenceFrontendService: {Unary: ch.NewSingleOutbound(hostPort)}, - }, - }) - if dispatcher == nil { - s.Logger.Fatal("No RPC dispatcher provided to create a connection to Cadence Service") - } - if err := dispatcher.Start(); err != nil { - s.Logger.Fatal("Failed to create outbound transport channel", tag.Error(err)) - } - - return workflowserviceclient.New(dispatcher.ClientConfig(cadenceFrontendService)), nil -} - -func (s *ClientIntegrationSuite) SetupTest() { - // Have to define our overridden assertions in the test setup. If we did it earlier, s.T() will return nil - s.Assertions = require.New(s.T()) -} - -// testDataConverter implements encoded.DataConverter using gob -type testDataConverter struct { - NumOfCallToData int // for testing to know testDataConverter is called as expected - NumOfCallFromData int -} - -func (tdc *testDataConverter) ToData(value ...interface{}) ([]byte, error) { - tdc.NumOfCallToData++ - var buf bytes.Buffer - enc := gob.NewEncoder(&buf) - for i, obj := range value { - if err := enc.Encode(obj); err != nil { - return nil, fmt.Errorf( - "unable to encode argument: %d, %v, with gob error: %v", i, reflect.TypeOf(obj), err) - } - } - return buf.Bytes(), nil -} - -func (tdc *testDataConverter) FromData(input []byte, valuePtr ...interface{}) error { - tdc.NumOfCallFromData++ - dec := gob.NewDecoder(bytes.NewBuffer(input)) - for i, obj := range valuePtr { - if err := dec.Decode(obj); err != nil { - return fmt.Errorf( - "unable to decode argument: %d, %v, with gob error: %v", i, reflect.TypeOf(obj), err) - } - } - return nil -} - -func newTestDataConverter() encoded.DataConverter { - return &testDataConverter{} -} - -func testActivity(ctx context.Context, msg string) (string, error) { - return "hello_" + msg, nil -} - -func testDataConverterWorkflow(ctx workflow.Context, tl string) (string, error) { - ao := workflow.ActivityOptions{ - ScheduleToStartTimeout: 20 * time.Second, - StartToCloseTimeout: 40 * time.Second, - } - ctx = workflow.WithActivityOptions(ctx, ao) - - var result string - err := workflow.ExecuteActivity(ctx, testActivity, "world").Get(ctx, &result) - if err != nil { - return "", err - } - - // use another converter to run activity, - // with new taskList so that worker with same data converter can properly process tasks. - var result1 string - ctx1 := workflow.WithDataConverter(ctx, newTestDataConverter()) - ctx1 = workflow.WithTaskList(ctx1, tl) - err1 := workflow.ExecuteActivity(ctx1, testActivity, "world1").Get(ctx1, &result1) - if err1 != nil { - return "", err1 - } - return result + "," + result1, nil -} - -func (s *ClientIntegrationSuite) startWorkerWithDataConverter(tl string, dataConverter encoded.DataConverter) cworker.Worker { - opts := cworker.Options{} - if dataConverter != nil { - opts.DataConverter = dataConverter - } - worker := cworker.New(s.wfService, s.domainName, tl, opts) - if err := worker.Start(); err != nil { - s.Logger.Fatal("Error when start worker with data converter", tag.Error(err)) - } - return worker -} - -func (s *ClientIntegrationSuite) TestClientDataConverter() { - tl := "client-integration-data-converter-activity-tasklist" - dc := newTestDataConverter() - worker := s.startWorkerWithDataConverter(tl, dc) - defer worker.Stop() - - id := "client-integration-data-converter-workflow" - workflowOptions := client.StartWorkflowOptions{ - ID: id, - TaskList: s.taskList, - ExecutionStartToCloseTimeout: 60 * time.Second, - } - ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) - defer cancel() - we, err := s.wfClient.ExecuteWorkflow(ctx, workflowOptions, testDataConverterWorkflow, tl) - if err != nil { - s.Logger.Fatal("Start workflow with err", tag.Error(err)) - } - s.NotNil(we) - s.True(we.GetRunID() != "") - - var res string - err = we.Get(ctx, &res) - s.NoError(err) - s.Equal("hello_world,hello_world1", res) - - // to ensure custom data converter is used, this number might be different if client changed. - d := dc.(*testDataConverter) - s.Equal(1, d.NumOfCallToData) - s.Equal(1, d.NumOfCallFromData) -} - -func (s *ClientIntegrationSuite) TestClientDataConverter_Failed() { - tl := "client-integration-data-converter-activity-failed-tasklist" - worker := s.startWorkerWithDataConverter(tl, nil) // mismatch of data converter - defer worker.Stop() - - id := "client-integration-data-converter-failed-workflow" - workflowOptions := client.StartWorkflowOptions{ - ID: id, - TaskList: s.taskList, - ExecutionStartToCloseTimeout: 60 * time.Second, - } - ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) - defer cancel() - we, err := s.wfClient.ExecuteWorkflow(ctx, workflowOptions, testDataConverterWorkflow, tl) - if err != nil { - s.Logger.Fatal("Start workflow with err", tag.Error(err)) - } - s.NotNil(we) - s.True(we.GetRunID() != "") - - var res string - err = we.Get(ctx, &res) - s.Error(err) - - // Get history to make sure only the 2nd activity is failed because of mismatch of data converter - iter := s.wfClient.GetWorkflowHistory(ctx, id, we.GetRunID(), false, 0) - completedAct := 0 - failedAct := 0 - for iter.HasNext() { - event, err := iter.Next() - s.Nil(err) - if event.GetEventType() == shared.EventTypeActivityTaskCompleted { - completedAct++ - } - if event.GetEventType() == shared.EventTypeActivityTaskFailed { - failedAct++ - attr := event.ActivityTaskFailedEventAttributes - s.True(strings.HasPrefix(string(attr.Details), "unable to decode the activity function input bytes with error")) - } - } - s.Equal(1, completedAct) - s.Equal(1, failedAct) -} - -var childTaskList = "client-integration-data-converter-child-tasklist" - -func testParentWorkflow(ctx workflow.Context) (string, error) { - logger := workflow.GetLogger(ctx) - execution := workflow.GetInfo(ctx).WorkflowExecution - childID := fmt.Sprintf("child_workflow:%v", execution.RunID) - cwo := workflow.ChildWorkflowOptions{ - WorkflowID: childID, - ExecutionStartToCloseTimeout: time.Minute, - } - ctx = workflow.WithChildOptions(ctx, cwo) - var result string - err := workflow.ExecuteChildWorkflow(ctx, testChildWorkflow, 0, 3).Get(ctx, &result) - if err != nil { - logger.Error("Parent execution received child execution failure.", zap.Error(err)) - return "", err - } - - childID1 := fmt.Sprintf("child_workflow1:%v", execution.RunID) - cwo1 := workflow.ChildWorkflowOptions{ - WorkflowID: childID1, - ExecutionStartToCloseTimeout: time.Minute, - TaskList: childTaskList, - } - ctx1 := workflow.WithChildOptions(ctx, cwo1) - ctx1 = workflow.WithDataConverter(ctx1, newTestDataConverter()) - var result1 string - err1 := workflow.ExecuteChildWorkflow(ctx1, testChildWorkflow, 0, 2).Get(ctx1, &result1) - if err1 != nil { - logger.Error("Parent execution received child execution 1 failure.", zap.Error(err1)) - return "", err1 - } - - res := fmt.Sprintf("Complete child1 %s times, complete child2 %s times", result, result1) - logger.Info("Parent execution completed.", zap.String("Result", res)) - return res, nil -} - -func testChildWorkflow(ctx workflow.Context, totalCount, runCount int) (string, error) { - logger := workflow.GetLogger(ctx) - logger.Info("Child workflow execution started.") - if runCount <= 0 { - logger.Error("Invalid valid for run count.", zap.Int("RunCount", runCount)) - return "", errors.New("invalid run count") - } - - totalCount++ - runCount-- - if runCount == 0 { - result := fmt.Sprintf("Child workflow execution completed after %v runs", totalCount) - logger.Info("Child workflow completed.", zap.String("Result", result)) - return strconv.Itoa(totalCount), nil - } - - logger.Info("Child workflow starting new run.", zap.Int("RunCount", runCount), zap.Int("TotalCount", - totalCount)) - return "", workflow.NewContinueAsNewError(ctx, testChildWorkflow, totalCount, runCount) -} - -func (s *ClientIntegrationSuite) TestClientDataConverter_WithChild() { - dc := newTestDataConverter() - worker := s.startWorkerWithDataConverter(childTaskList, dc) - defer worker.Stop() - - id := "client-integration-data-converter-with-child-workflow" - workflowOptions := client.StartWorkflowOptions{ - ID: id, - TaskList: s.taskList, - ExecutionStartToCloseTimeout: 60 * time.Second, - } - ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) - defer cancel() - we, err := s.wfClient.ExecuteWorkflow(ctx, workflowOptions, testParentWorkflow) - if err != nil { - s.Logger.Fatal("Start workflow with err", tag.Error(err)) - } - s.NotNil(we) - s.True(we.GetRunID() != "") - - var res string - err = we.Get(ctx, &res) - s.NoError(err) - s.Equal("Complete child1 3 times, complete child2 2 times", res) - - // to ensure custom data converter is used, this number might be different if client changed. - d := dc.(*testDataConverter) - s.Equal(3, d.NumOfCallToData) - s.Equal(2, d.NumOfCallFromData) -} diff --git a/host/continueasnew_test.go b/host/continueasnewTest.go similarity index 100% rename from host/continueasnew_test.go rename to host/continueasnewTest.go diff --git a/host/decision_test.go b/host/decisionTest.go similarity index 100% rename from host/decision_test.go rename to host/decisionTest.go diff --git a/host/elasticsearchTest.go b/host/elasticsearchTest.go new file mode 100644 index 00000000000..9b5992dcd5f --- /dev/null +++ b/host/elasticsearchTest.go @@ -0,0 +1,1088 @@ +// Copyright (c) 2016 Uber Technologies, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +// +build esintegration + +// to run locally, make sure kafka and es is running, +// then run cmd `go test -v ./host -run TestElasticsearchIntegrationSuite -tags esintegration` +package host + +import ( + "bytes" + "encoding/json" + "fmt" + "strconv" + "testing" + "time" + + "github.com/pborman/uuid" + "github.com/stretchr/testify/require" + + "github.com/uber/cadence/common" + "github.com/uber/cadence/common/definition" + "github.com/uber/cadence/common/log/tag" + "github.com/uber/cadence/common/types" + "github.com/uber/cadence/environment" + "github.com/uber/cadence/host/esutils" +) + +const ( + numOfRetry = 50 + waitTimeInMs = 400 + waitForESToSettle = 4 * time.Second // wait es shards for some time ensure data consistent +) + +type ElasticSearchIntegrationSuite struct { + // override suite.Suite.Assertions with require.Assertions; this means that s.NotNil(nil) will stop the test, + // not merely log an error + *require.Assertions + IntegrationBase + esClient esutils.ESClient + + testSearchAttributeKey string + testSearchAttributeVal string +} + +// This cluster use customized threshold for history config +func (s *ElasticSearchIntegrationSuite) SetupSuite() { + s.setupSuite() + s.esClient = esutils.CreateESClient(s.Suite, s.testClusterConfig.ESConfig.URL.String(), environment.GetESVersion()) + s.esClient.PutIndexTemplate(s.Suite, "testdata/es_"+environment.GetESVersion()+"_index_template.json", "test-visibility-template") + indexName := s.testClusterConfig.ESConfig.Indices[common.VisibilityAppName] + s.esClient.CreateIndex(s.Suite, indexName) + s.putIndexSettings(indexName, defaultTestValueOfESIndexMaxResultWindow) +} + +func (s *ElasticSearchIntegrationSuite) TearDownSuite() { + s.tearDownSuite() + s.esClient.DeleteIndex(s.Suite, s.testClusterConfig.ESConfig.Indices[common.VisibilityAppName]) +} + +func (s *ElasticSearchIntegrationSuite) SetupTest() { + // Have to define our overridden assertions in the test setup. If we did it earlier, s.T() will return nil + s.Assertions = require.New(s.T()) + s.testSearchAttributeKey = definition.CustomStringField + s.testSearchAttributeVal = "test value" +} + +func (s *ElasticSearchIntegrationSuite) TestListOpenWorkflow() { + id := "es-integration-start-workflow-test" + wt := "es-integration-start-workflow-test-type" + tl := "es-integration-start-workflow-test-tasklist" + request := s.createStartWorkflowExecutionRequest(id, wt, tl) + + attrValBytes, _ := json.Marshal(s.testSearchAttributeVal) + searchAttr := &types.SearchAttributes{ + IndexedFields: map[string][]byte{ + s.testSearchAttributeKey: attrValBytes, + }, + } + request.SearchAttributes = searchAttr + + startTime := time.Now().UnixNano() + we, err := s.engine.StartWorkflowExecution(createContext(), request) + s.Nil(err) + + startFilter := &types.StartTimeFilter{} + startFilter.EarliestTime = common.Int64Ptr(startTime) + var openExecution *types.WorkflowExecutionInfo + for i := 0; i < numOfRetry; i++ { + startFilter.LatestTime = common.Int64Ptr(time.Now().UnixNano()) + resp, err := s.engine.ListOpenWorkflowExecutions(createContext(), &types.ListOpenWorkflowExecutionsRequest{ + Domain: s.domainName, + MaximumPageSize: defaultTestValueOfESIndexMaxResultWindow, + StartTimeFilter: startFilter, + ExecutionFilter: &types.WorkflowExecutionFilter{ + WorkflowID: id, + }, + }) + s.Nil(err) + if len(resp.GetExecutions()) == 1 { + openExecution = resp.GetExecutions()[0] + break + } + time.Sleep(waitTimeInMs * time.Millisecond) + } + s.NotNil(openExecution) + s.Equal(we.GetRunID(), openExecution.GetExecution().GetRunID()) + s.Equal(attrValBytes, openExecution.SearchAttributes.GetIndexedFields()[s.testSearchAttributeKey]) +} + +func (s *ElasticSearchIntegrationSuite) TestListWorkflow() { + id := "es-integration-list-workflow-test" + wt := "es-integration-list-workflow-test-type" + tl := "es-integration-list-workflow-test-tasklist" + request := s.createStartWorkflowExecutionRequest(id, wt, tl) + + we, err := s.engine.StartWorkflowExecution(createContext(), request) + s.Nil(err) + + query := fmt.Sprintf(`WorkflowID = "%s"`, id) + s.testHelperForReadOnce(we.GetRunID(), query, false) +} + +func (s *ElasticSearchIntegrationSuite) TestListWorkflow_ExecutionTime() { + id := "es-integration-list-workflow-execution-time-test" + wt := "es-integration-list-workflow-execution-time-test-type" + tl := "es-integration-list-workflow-execution-time-test-tasklist" + request := s.createStartWorkflowExecutionRequest(id, wt, tl) + + we, err := s.engine.StartWorkflowExecution(createContext(), request) + s.Nil(err) + + cronID := id + "-cron" + request.CronSchedule = "@every 1m" + request.WorkflowID = cronID + + weCron, err := s.engine.StartWorkflowExecution(createContext(), request) + s.Nil(err) + + query := fmt.Sprintf(`(WorkflowID = "%s" or WorkflowID = "%s") and ExecutionTime < %v`, id, cronID, time.Now().UnixNano()+int64(time.Minute)) + s.testHelperForReadOnce(weCron.GetRunID(), query, false) + + query = fmt.Sprintf(`WorkflowID = "%s"`, id) + s.testHelperForReadOnce(we.GetRunID(), query, false) +} + +func (s *ElasticSearchIntegrationSuite) TestListWorkflow_SearchAttribute() { + id := "es-integration-list-workflow-by-search-attr-test" + wt := "es-integration-list-workflow-by-search-attr-test-type" + tl := "es-integration-list-workflow-by-search-attr-test-tasklist" + request := s.createStartWorkflowExecutionRequest(id, wt, tl) + + attrValBytes, _ := json.Marshal(s.testSearchAttributeVal) + searchAttr := &types.SearchAttributes{ + IndexedFields: map[string][]byte{ + s.testSearchAttributeKey: attrValBytes, + }, + } + request.SearchAttributes = searchAttr + + we, err := s.engine.StartWorkflowExecution(createContext(), request) + s.Nil(err) + query := fmt.Sprintf(`WorkflowID = "%s" and %s = "%s"`, id, s.testSearchAttributeKey, s.testSearchAttributeVal) + s.testHelperForReadOnce(we.GetRunID(), query, false) + + // test upsert + dtHandler := func(execution *types.WorkflowExecution, wt *types.WorkflowType, + previousStartedEventID, startedEventID int64, history *types.History) ([]byte, []*types.Decision, error) { + + upsertDecision := &types.Decision{ + DecisionType: types.DecisionTypeUpsertWorkflowSearchAttributes.Ptr(), + UpsertWorkflowSearchAttributesDecisionAttributes: &types.UpsertWorkflowSearchAttributesDecisionAttributes{ + SearchAttributes: getUpsertSearchAttributes(), + }} + + return nil, []*types.Decision{upsertDecision}, nil + } + taskList := &types.TaskList{Name: tl} + poller := &TaskPoller{ + Engine: s.engine, + Domain: s.domainName, + TaskList: taskList, + StickyTaskList: taskList, + Identity: "worker1", + DecisionHandler: dtHandler, + Logger: s.Logger, + T: s.T(), + } + _, newTask, err := poller.PollAndProcessDecisionTaskWithAttemptAndRetryAndForceNewDecision( + false, + false, + true, + true, + int64(0), + 1, + true, + nil) + s.Nil(err) + s.NotNil(newTask) + s.NotNil(newTask.DecisionTask) + + time.Sleep(waitForESToSettle) + + listRequest := &types.ListWorkflowExecutionsRequest{ + Domain: s.domainName, + PageSize: int32(2), + Query: fmt.Sprintf(`WorkflowType = '%s' and CloseTime = missing and BinaryChecksums = 'binary-v1'`, wt), + } + // verify upsert data is on ES + s.testListResultForUpsertSearchAttributes(listRequest) + + // verify DescribeWorkflowExecution + descRequest := &types.DescribeWorkflowExecutionRequest{ + Domain: s.domainName, + Execution: &types.WorkflowExecution{ + WorkflowID: id, + }, + } + descResp, err := s.engine.DescribeWorkflowExecution(createContext(), descRequest) + s.Nil(err) + expectedSearchAttributes := getUpsertSearchAttributes() + s.Equal(expectedSearchAttributes, descResp.WorkflowExecutionInfo.GetSearchAttributes()) +} + +func (s *ElasticSearchIntegrationSuite) TestListWorkflow_PageToken() { + id := "es-integration-list-workflow-token-test" + wt := "es-integration-list-workflow-token-test-type" + tl := "es-integration-list-workflow-token-test-tasklist" + request := s.createStartWorkflowExecutionRequest(id, wt, tl) + + numOfWorkflows := defaultTestValueOfESIndexMaxResultWindow - 1 // == 4 + pageSize := 3 + + s.testListWorkflowHelper(numOfWorkflows, pageSize, request, id, wt, false) +} + +func (s *ElasticSearchIntegrationSuite) TestListWorkflow_SearchAfter() { + id := "es-integration-list-workflow-searchAfter-test" + wt := "es-integration-list-workflow-searchAfter-test-type" + tl := "es-integration-list-workflow-searchAfter-test-tasklist" + request := s.createStartWorkflowExecutionRequest(id, wt, tl) + + numOfWorkflows := defaultTestValueOfESIndexMaxResultWindow + 1 // == 6 + pageSize := 4 + + s.testListWorkflowHelper(numOfWorkflows, pageSize, request, id, wt, false) +} + +func (s *ElasticSearchIntegrationSuite) TestListWorkflow_OrQuery() { + id := "es-integration-list-workflow-or-query-test" + wt := "es-integration-list-workflow-or-query-test-type" + tl := "es-integration-list-workflow-or-query-test-tasklist" + request := s.createStartWorkflowExecutionRequest(id, wt, tl) + + // start 3 workflows + key := definition.CustomIntField + attrValBytes, _ := json.Marshal(1) + searchAttr := &types.SearchAttributes{ + IndexedFields: map[string][]byte{ + key: attrValBytes, + }, + } + request.SearchAttributes = searchAttr + we1, err := s.engine.StartWorkflowExecution(createContext(), request) + s.Nil(err) + + request.RequestID = uuid.New() + request.WorkflowID = id + "-2" + attrValBytes, _ = json.Marshal(2) + searchAttr.IndexedFields[key] = attrValBytes + we2, err := s.engine.StartWorkflowExecution(createContext(), request) + s.Nil(err) + + request.RequestID = uuid.New() + request.WorkflowID = id + "-3" + attrValBytes, _ = json.Marshal(3) + searchAttr.IndexedFields[key] = attrValBytes + we3, err := s.engine.StartWorkflowExecution(createContext(), request) + s.Nil(err) + + time.Sleep(waitForESToSettle) + + // query 1 workflow with search attr + query1 := fmt.Sprintf(`CustomIntField = %d`, 1) + var openExecution *types.WorkflowExecutionInfo + listRequest := &types.ListWorkflowExecutionsRequest{ + Domain: s.domainName, + PageSize: defaultTestValueOfESIndexMaxResultWindow, + Query: query1, + } + for i := 0; i < numOfRetry; i++ { + resp, err := s.engine.ListWorkflowExecutions(createContext(), listRequest) + s.Nil(err) + if len(resp.GetExecutions()) == 1 { + openExecution = resp.GetExecutions()[0] + break + } + time.Sleep(waitTimeInMs * time.Millisecond) + } + s.NotNil(openExecution) + s.Equal(we1.GetRunID(), openExecution.GetExecution().GetRunID()) + s.True(openExecution.GetExecutionTime() >= openExecution.GetStartTime()) + searchValBytes := openExecution.SearchAttributes.GetIndexedFields()[key] + var searchVal int + json.Unmarshal(searchValBytes, &searchVal) + s.Equal(1, searchVal) + + // query with or clause + query2 := fmt.Sprintf(`CustomIntField = %d or CustomIntField = %d`, 1, 2) + listRequest.Query = query2 + var openExecutions []*types.WorkflowExecutionInfo + for i := 0; i < numOfRetry; i++ { + resp, err := s.engine.ListWorkflowExecutions(createContext(), listRequest) + s.Nil(err) + if len(resp.GetExecutions()) == 2 { + openExecutions = resp.GetExecutions() + break + } + time.Sleep(waitTimeInMs * time.Millisecond) + } + s.Equal(2, len(openExecutions)) + e1 := openExecutions[0] + e2 := openExecutions[1] + if e1.GetExecution().GetRunID() != we1.GetRunID() { + // results are sorted by [CloseTime,RunID] desc, so find the correct mapping first + e1, e2 = e2, e1 + } + s.Equal(we1.GetRunID(), e1.GetExecution().GetRunID()) + s.Equal(we2.GetRunID(), e2.GetExecution().GetRunID()) + searchValBytes = e2.SearchAttributes.GetIndexedFields()[key] + json.Unmarshal(searchValBytes, &searchVal) + s.Equal(2, searchVal) + + // query for open + query3 := fmt.Sprintf(`(CustomIntField = %d or CustomIntField = %d) and CloseTime = missing`, 2, 3) + listRequest.Query = query3 + for i := 0; i < numOfRetry; i++ { + resp, err := s.engine.ListWorkflowExecutions(createContext(), listRequest) + s.Nil(err) + if len(resp.GetExecutions()) == 2 { + openExecutions = resp.GetExecutions() + break + } + time.Sleep(waitTimeInMs * time.Millisecond) + } + s.Equal(2, len(openExecutions)) + e1 = openExecutions[0] + e2 = openExecutions[1] + s.Equal(we3.GetRunID(), e1.GetExecution().GetRunID()) + s.Equal(we2.GetRunID(), e2.GetExecution().GetRunID()) + searchValBytes = e1.SearchAttributes.GetIndexedFields()[key] + json.Unmarshal(searchValBytes, &searchVal) + s.Equal(3, searchVal) +} + +// To test last page search trigger max window size error +func (s *ElasticSearchIntegrationSuite) TestListWorkflow_MaxWindowSize() { + id := "es-integration-list-workflow-max-window-size-test" + wt := "es-integration-list-workflow-max-window-size-test-type" + tl := "es-integration-list-workflow-max-window-size-test-tasklist" + startRequest := s.createStartWorkflowExecutionRequest(id, wt, tl) + + for i := 0; i < defaultTestValueOfESIndexMaxResultWindow; i++ { + startRequest.RequestID = uuid.New() + startRequest.WorkflowID = id + strconv.Itoa(i) + _, err := s.engine.StartWorkflowExecution(createContext(), startRequest) + s.Nil(err) + } + + time.Sleep(waitForESToSettle) + + var listResp *types.ListWorkflowExecutionsResponse + var nextPageToken []byte + + listRequest := &types.ListWorkflowExecutionsRequest{ + Domain: s.domainName, + PageSize: int32(defaultTestValueOfESIndexMaxResultWindow), + NextPageToken: nextPageToken, + Query: fmt.Sprintf(`WorkflowType = '%s' and CloseTime = missing`, wt), + } + // get first page + for i := 0; i < numOfRetry; i++ { + resp, err := s.engine.ListWorkflowExecutions(createContext(), listRequest) + s.Nil(err) + if len(resp.GetExecutions()) == defaultTestValueOfESIndexMaxResultWindow { + listResp = resp + break + } + time.Sleep(waitTimeInMs * time.Millisecond) + } + s.NotNil(listResp) + s.True(len(listResp.GetNextPageToken()) != 0) + + // the last request + listRequest.NextPageToken = listResp.GetNextPageToken() + resp, err := s.engine.ListWorkflowExecutions(createContext(), listRequest) + s.Nil(err) + s.True(len(resp.GetExecutions()) == 0) + s.True(len(resp.GetNextPageToken()) == 0) +} + +func (s *ElasticSearchIntegrationSuite) TestListWorkflow_OrderBy() { + id := "es-integration-list-workflow-order-by-test" + wt := "es-integration-list-workflow-order-by-test-type" + tl := "es-integration-list-workflow-order-by-test-tasklist" + startRequest := s.createStartWorkflowExecutionRequest(id, wt, tl) + + for i := 0; i < defaultTestValueOfESIndexMaxResultWindow+1; i++ { // start 6 + startRequest.RequestID = uuid.New() + startRequest.WorkflowID = id + strconv.Itoa(i) + + if i < defaultTestValueOfESIndexMaxResultWindow-1 { // 4 workflow has search attr + intVal, _ := json.Marshal(i) + doubleVal, _ := json.Marshal(float64(i)) + strVal, _ := json.Marshal(strconv.Itoa(i)) + timeVal, _ := json.Marshal(time.Now()) + searchAttr := &types.SearchAttributes{ + IndexedFields: map[string][]byte{ + definition.CustomIntField: intVal, + definition.CustomDoubleField: doubleVal, + definition.CustomKeywordField: strVal, + definition.CustomDatetimeField: timeVal, + }, + } + startRequest.SearchAttributes = searchAttr + } else { + startRequest.SearchAttributes = &types.SearchAttributes{} + } + + _, err := s.engine.StartWorkflowExecution(createContext(), startRequest) + s.Nil(err) + } + + time.Sleep(waitForESToSettle) + + desc := "desc" + asc := "asc" + queryTemplate := `WorkflowType = "%s" order by %s %s` + pageSize := int32(defaultTestValueOfESIndexMaxResultWindow) + + // order by CloseTime asc + query1 := fmt.Sprintf(queryTemplate, wt, definition.CloseTime, asc) + var openExecutions []*types.WorkflowExecutionInfo + listRequest := &types.ListWorkflowExecutionsRequest{ + Domain: s.domainName, + PageSize: pageSize, + Query: query1, + } + for i := 0; i < numOfRetry; i++ { + resp, err := s.engine.ListWorkflowExecutions(createContext(), listRequest) + s.Nil(err) + if int32(len(resp.GetExecutions())) == listRequest.GetPageSize() { + openExecutions = resp.GetExecutions() + break + } + time.Sleep(waitTimeInMs * time.Millisecond) + } + s.NotNil(openExecutions) + for i := int32(1); i < pageSize; i++ { + s.True(openExecutions[i-1].GetCloseTime() <= openExecutions[i].GetCloseTime()) + } + + // greatest effort to reduce duplicate code + testHelper := func(query, searchAttrKey string, prevVal, currVal interface{}) { + listRequest.Query = query + listRequest.NextPageToken = []byte{} + resp, err := s.engine.ListWorkflowExecutions(createContext(), listRequest) + s.Nil(err) + openExecutions = resp.GetExecutions() + dec := json.NewDecoder(bytes.NewReader(openExecutions[0].GetSearchAttributes().GetIndexedFields()[searchAttrKey])) + dec.UseNumber() + err = dec.Decode(&prevVal) + s.Nil(err) + for i := int32(1); i < pageSize; i++ { + indexedFields := openExecutions[i].GetSearchAttributes().GetIndexedFields() + searchAttrBytes, ok := indexedFields[searchAttrKey] + if !ok { // last one doesn't have search attr + s.Equal(pageSize-1, i) + break + } + dec := json.NewDecoder(bytes.NewReader(searchAttrBytes)) + dec.UseNumber() + err = dec.Decode(&currVal) + s.Nil(err) + var v1, v2 interface{} + switch searchAttrKey { + case definition.CustomIntField: + v1, _ = prevVal.(json.Number).Int64() + v2, _ = currVal.(json.Number).Int64() + s.True(v1.(int64) >= v2.(int64)) + case definition.CustomDoubleField: + v1, _ = prevVal.(json.Number).Float64() + v2, _ = currVal.(json.Number).Float64() + s.True(v1.(float64) >= v2.(float64)) + case definition.CustomKeywordField: + s.True(prevVal.(string) >= currVal.(string)) + case definition.CustomDatetimeField: + v1, _ = time.Parse(time.RFC3339, prevVal.(string)) + v2, _ = time.Parse(time.RFC3339, currVal.(string)) + s.True(v1.(time.Time).After(v2.(time.Time))) + } + prevVal = currVal + } + listRequest.NextPageToken = resp.GetNextPageToken() + resp, err = s.engine.ListWorkflowExecutions(createContext(), listRequest) // last page + s.Nil(err) + s.Equal(1, len(resp.GetExecutions())) + } + + // order by CustomIntField desc + field := definition.CustomIntField + query := fmt.Sprintf(queryTemplate, wt, field, desc) + var int1, int2 int + testHelper(query, field, int1, int2) + + // order by CustomDoubleField desc + field = definition.CustomDoubleField + query = fmt.Sprintf(queryTemplate, wt, field, desc) + var double1, double2 float64 + testHelper(query, field, double1, double2) + + // order by CustomKeywordField desc + field = definition.CustomKeywordField + query = fmt.Sprintf(queryTemplate, wt, field, desc) + var s1, s2 string + testHelper(query, field, s1, s2) + + // order by CustomDatetimeField desc + field = definition.CustomDatetimeField + query = fmt.Sprintf(queryTemplate, wt, field, desc) + var t1, t2 time.Time + testHelper(query, field, t1, t2) +} + +func (s *ElasticSearchIntegrationSuite) testListWorkflowHelper(numOfWorkflows, pageSize int, + startRequest *types.StartWorkflowExecutionRequest, wid, wType string, isScan bool) { + + // start enough number of workflows + for i := 0; i < numOfWorkflows; i++ { + startRequest.RequestID = uuid.New() + startRequest.WorkflowID = wid + strconv.Itoa(i) + _, err := s.engine.StartWorkflowExecution(createContext(), startRequest) + s.Nil(err) + } + + time.Sleep(waitForESToSettle) + + var openExecutions []*types.WorkflowExecutionInfo + var nextPageToken []byte + + listRequest := &types.ListWorkflowExecutionsRequest{ + Domain: s.domainName, + PageSize: int32(pageSize), + NextPageToken: nextPageToken, + Query: fmt.Sprintf(`WorkflowType = '%s' and CloseTime = missing`, wType), + } + // test first page + for i := 0; i < numOfRetry; i++ { + var resp *types.ListWorkflowExecutionsResponse + var err error + + if isScan { + resp, err = s.engine.ScanWorkflowExecutions(createContext(), listRequest) + } else { + resp, err = s.engine.ListWorkflowExecutions(createContext(), listRequest) + } + s.Nil(err) + if len(resp.GetExecutions()) == pageSize { + openExecutions = resp.GetExecutions() + nextPageToken = resp.GetNextPageToken() + break + } + time.Sleep(waitTimeInMs * time.Millisecond) + } + s.NotNil(openExecutions) + s.NotNil(nextPageToken) + s.True(len(nextPageToken) > 0) + + // test last page + listRequest.NextPageToken = nextPageToken + inIf := false + for i := 0; i < numOfRetry; i++ { + var resp *types.ListWorkflowExecutionsResponse + var err error + + if isScan { + resp, err = s.engine.ScanWorkflowExecutions(createContext(), listRequest) + } else { + resp, err = s.engine.ListWorkflowExecutions(createContext(), listRequest) + } + s.Nil(err) + if len(resp.GetExecutions()) == numOfWorkflows-pageSize { + inIf = true + openExecutions = resp.GetExecutions() + nextPageToken = resp.GetNextPageToken() + break + } + time.Sleep(waitTimeInMs * time.Millisecond) + } + s.True(inIf) + s.NotNil(openExecutions) + s.Nil(nextPageToken) +} + +func (s *ElasticSearchIntegrationSuite) testHelperForReadOnce(runID, query string, isScan bool) { + var openExecution *types.WorkflowExecutionInfo + listRequest := &types.ListWorkflowExecutionsRequest{ + Domain: s.domainName, + PageSize: defaultTestValueOfESIndexMaxResultWindow, + Query: query, + } + for i := 0; i < numOfRetry; i++ { + var resp *types.ListWorkflowExecutionsResponse + var err error + + if isScan { + resp, err = s.engine.ScanWorkflowExecutions(createContext(), listRequest) + } else { + resp, err = s.engine.ListWorkflowExecutions(createContext(), listRequest) + } + + s.Nil(err) + if len(resp.GetExecutions()) == 1 { + openExecution = resp.GetExecutions()[0] + break + } + time.Sleep(waitTimeInMs * time.Millisecond) + } + s.NotNil(openExecution) + s.Equal(runID, openExecution.GetExecution().GetRunID()) + s.True(openExecution.GetExecutionTime() >= openExecution.GetStartTime()) + if openExecution.SearchAttributes != nil && len(openExecution.SearchAttributes.GetIndexedFields()) > 0 { + searchValBytes := openExecution.SearchAttributes.GetIndexedFields()[s.testSearchAttributeKey] + var searchVal string + json.Unmarshal(searchValBytes, &searchVal) + s.Equal(s.testSearchAttributeVal, searchVal) + } +} + +func (s *ElasticSearchIntegrationSuite) TestScanWorkflow() { + id := "es-integration-scan-workflow-test" + wt := "es-integration-scan-workflow-test-type" + tl := "es-integration-scan-workflow-test-tasklist" + identity := "worker1" + + workflowType := &types.WorkflowType{} + workflowType.Name = wt + + taskList := &types.TaskList{} + taskList.Name = tl + + request := &types.StartWorkflowExecutionRequest{ + RequestID: uuid.New(), + Domain: s.domainName, + WorkflowID: id, + WorkflowType: workflowType, + TaskList: taskList, + Input: nil, + ExecutionStartToCloseTimeoutSeconds: common.Int32Ptr(100), + TaskStartToCloseTimeoutSeconds: common.Int32Ptr(1), + Identity: identity, + } + + we, err := s.engine.StartWorkflowExecution(createContext(), request) + s.Nil(err) + query := fmt.Sprintf(`WorkflowID = "%s"`, id) + s.testHelperForReadOnce(we.GetRunID(), query, true) +} + +func (s *ElasticSearchIntegrationSuite) TestScanWorkflow_SearchAttribute() { + id := "es-integration-scan-workflow-search-attr-test" + wt := "es-integration-scan-workflow-search-attr-test-type" + tl := "es-integration-scan-workflow-search-attr-test-tasklist" + request := s.createStartWorkflowExecutionRequest(id, wt, tl) + + attrValBytes, _ := json.Marshal(s.testSearchAttributeVal) + searchAttr := &types.SearchAttributes{ + IndexedFields: map[string][]byte{ + s.testSearchAttributeKey: attrValBytes, + }, + } + request.SearchAttributes = searchAttr + + we, err := s.engine.StartWorkflowExecution(createContext(), request) + s.Nil(err) + query := fmt.Sprintf(`WorkflowID = "%s" and %s = "%s"`, id, s.testSearchAttributeKey, s.testSearchAttributeVal) + s.testHelperForReadOnce(we.GetRunID(), query, true) +} + +func (s *ElasticSearchIntegrationSuite) TestScanWorkflow_PageToken() { + id := "es-integration-scan-workflow-token-test" + wt := "es-integration-scan-workflow-token-test-type" + tl := "es-integration-scan-workflow-token-test-tasklist" + identity := "worker1" + + workflowType := &types.WorkflowType{} + workflowType.Name = wt + + taskList := &types.TaskList{} + taskList.Name = tl + + request := &types.StartWorkflowExecutionRequest{ + Domain: s.domainName, + WorkflowType: workflowType, + TaskList: taskList, + Input: nil, + ExecutionStartToCloseTimeoutSeconds: common.Int32Ptr(100), + TaskStartToCloseTimeoutSeconds: common.Int32Ptr(1), + Identity: identity, + } + + numOfWorkflows := 4 + pageSize := 3 + + s.testListWorkflowHelper(numOfWorkflows, pageSize, request, id, wt, true) +} + +func (s *ElasticSearchIntegrationSuite) TestCountWorkflow() { + id := "es-integration-count-workflow-test" + wt := "es-integration-count-workflow-test-type" + tl := "es-integration-count-workflow-test-tasklist" + request := s.createStartWorkflowExecutionRequest(id, wt, tl) + + attrValBytes, _ := json.Marshal(s.testSearchAttributeVal) + searchAttr := &types.SearchAttributes{ + IndexedFields: map[string][]byte{ + s.testSearchAttributeKey: attrValBytes, + }, + } + request.SearchAttributes = searchAttr + + _, err := s.engine.StartWorkflowExecution(createContext(), request) + s.Nil(err) + + query := fmt.Sprintf(`WorkflowID = "%s" and %s = "%s"`, id, s.testSearchAttributeKey, s.testSearchAttributeVal) + countRequest := &types.CountWorkflowExecutionsRequest{ + Domain: s.domainName, + Query: query, + } + var resp *types.CountWorkflowExecutionsResponse + for i := 0; i < numOfRetry; i++ { + resp, err = s.engine.CountWorkflowExecutions(createContext(), countRequest) + s.Nil(err) + if resp.GetCount() == int64(1) { + break + } + time.Sleep(waitTimeInMs * time.Millisecond) + } + s.Equal(int64(1), resp.GetCount()) + + query = fmt.Sprintf(`WorkflowID = "%s" and %s = "%s"`, id, s.testSearchAttributeKey, "noMatch") + countRequest.Query = query + resp, err = s.engine.CountWorkflowExecutions(createContext(), countRequest) + s.Nil(err) + s.Equal(int64(0), resp.GetCount()) +} + +func (s *ElasticSearchIntegrationSuite) createStartWorkflowExecutionRequest(id, wt, tl string) *types.StartWorkflowExecutionRequest { + identity := "worker1" + workflowType := &types.WorkflowType{} + workflowType.Name = wt + + taskList := &types.TaskList{} + taskList.Name = tl + + request := &types.StartWorkflowExecutionRequest{ + RequestID: uuid.New(), + Domain: s.domainName, + WorkflowID: id, + WorkflowType: workflowType, + TaskList: taskList, + Input: nil, + ExecutionStartToCloseTimeoutSeconds: common.Int32Ptr(100), + TaskStartToCloseTimeoutSeconds: common.Int32Ptr(1), + Identity: identity, + } + return request +} + +func (s *ElasticSearchIntegrationSuite) TestUpsertWorkflowExecution() { + id := "es-integration-upsert-workflow-test" + wt := "es-integration-upsert-workflow-test-type" + tl := "es-integration-upsert-workflow-test-tasklist" + identity := "worker1" + + workflowType := &types.WorkflowType{} + workflowType.Name = wt + + taskList := &types.TaskList{} + taskList.Name = tl + + request := &types.StartWorkflowExecutionRequest{ + RequestID: uuid.New(), + Domain: s.domainName, + WorkflowID: id, + WorkflowType: workflowType, + TaskList: taskList, + Input: nil, + ExecutionStartToCloseTimeoutSeconds: common.Int32Ptr(100), + TaskStartToCloseTimeoutSeconds: common.Int32Ptr(1), + Identity: identity, + } + + we, err0 := s.engine.StartWorkflowExecution(createContext(), request) + s.Nil(err0) + + s.Logger.Info("StartWorkflowExecution", tag.WorkflowRunID(we.RunID)) + + decisionCount := 0 + dtHandler := func(execution *types.WorkflowExecution, wt *types.WorkflowType, + previousStartedEventID, startedEventID int64, history *types.History) ([]byte, []*types.Decision, error) { + + upsertDecision := &types.Decision{ + DecisionType: types.DecisionTypeUpsertWorkflowSearchAttributes.Ptr(), + UpsertWorkflowSearchAttributesDecisionAttributes: &types.UpsertWorkflowSearchAttributesDecisionAttributes{}} + + // handle first upsert + if decisionCount == 0 { + decisionCount++ + + attrValBytes, _ := json.Marshal(s.testSearchAttributeVal) + upsertSearchAttr := &types.SearchAttributes{ + IndexedFields: map[string][]byte{ + s.testSearchAttributeKey: attrValBytes, + }, + } + upsertDecision.UpsertWorkflowSearchAttributesDecisionAttributes.SearchAttributes = upsertSearchAttr + return nil, []*types.Decision{upsertDecision}, nil + } + // handle second upsert, which update existing field and add new field + if decisionCount == 1 { + decisionCount++ + upsertDecision.UpsertWorkflowSearchAttributesDecisionAttributes.SearchAttributes = getUpsertSearchAttributes() + return nil, []*types.Decision{upsertDecision}, nil + } + + return nil, []*types.Decision{{ + DecisionType: types.DecisionTypeCompleteWorkflowExecution.Ptr(), + CompleteWorkflowExecutionDecisionAttributes: &types.CompleteWorkflowExecutionDecisionAttributes{ + Result: []byte("Done."), + }, + }}, nil + } + + poller := &TaskPoller{ + Engine: s.engine, + Domain: s.domainName, + TaskList: taskList, + StickyTaskList: taskList, + Identity: identity, + DecisionHandler: dtHandler, + Logger: s.Logger, + T: s.T(), + } + + // process 1st decision and assert decision is handled correctly. + _, newTask, err := poller.PollAndProcessDecisionTaskWithAttemptAndRetryAndForceNewDecision( + false, + false, + true, + true, + int64(0), + 1, + true, + nil) + s.Nil(err) + s.NotNil(newTask) + s.NotNil(newTask.DecisionTask) + s.Equal(int64(3), newTask.DecisionTask.GetPreviousStartedEventID()) + s.Equal(int64(7), newTask.DecisionTask.GetStartedEventID()) + s.Equal(4, len(newTask.DecisionTask.History.Events)) + s.Equal(types.EventTypeDecisionTaskCompleted, newTask.DecisionTask.History.Events[0].GetEventType()) + s.Equal(types.EventTypeUpsertWorkflowSearchAttributes, newTask.DecisionTask.History.Events[1].GetEventType()) + s.Equal(types.EventTypeDecisionTaskScheduled, newTask.DecisionTask.History.Events[2].GetEventType()) + s.Equal(types.EventTypeDecisionTaskStarted, newTask.DecisionTask.History.Events[3].GetEventType()) + + time.Sleep(waitForESToSettle) + + // verify upsert data is on ES + listRequest := &types.ListWorkflowExecutionsRequest{ + Domain: s.domainName, + PageSize: int32(2), + Query: fmt.Sprintf(`WorkflowType = '%s' and CloseTime = missing`, wt), + } + verified := false + for i := 0; i < numOfRetry; i++ { + resp, err := s.engine.ListWorkflowExecutions(createContext(), listRequest) + s.Nil(err) + if len(resp.GetExecutions()) == 1 { + execution := resp.GetExecutions()[0] + retrievedSearchAttr := execution.SearchAttributes + if retrievedSearchAttr != nil && len(retrievedSearchAttr.GetIndexedFields()) > 0 { + searchValBytes := retrievedSearchAttr.GetIndexedFields()[s.testSearchAttributeKey] + var searchVal string + json.Unmarshal(searchValBytes, &searchVal) + s.Equal(s.testSearchAttributeVal, searchVal) + verified = true + break + } + } + time.Sleep(waitTimeInMs * time.Millisecond) + } + s.True(verified) + + // process 2nd decision and assert decision is handled correctly. + _, newTask, err = poller.PollAndProcessDecisionTaskWithAttemptAndRetryAndForceNewDecision( + false, + false, + true, + true, + int64(0), + 1, + true, + nil) + s.Nil(err) + s.NotNil(newTask) + s.NotNil(newTask.DecisionTask) + s.Equal(4, len(newTask.DecisionTask.History.Events)) + s.Equal(types.EventTypeDecisionTaskCompleted, newTask.DecisionTask.History.Events[0].GetEventType()) + s.Equal(types.EventTypeUpsertWorkflowSearchAttributes, newTask.DecisionTask.History.Events[1].GetEventType()) + s.Equal(types.EventTypeDecisionTaskScheduled, newTask.DecisionTask.History.Events[2].GetEventType()) + s.Equal(types.EventTypeDecisionTaskStarted, newTask.DecisionTask.History.Events[3].GetEventType()) + + time.Sleep(waitForESToSettle) + + // verify upsert data is on ES + s.testListResultForUpsertSearchAttributes(listRequest) +} + +func (s *ElasticSearchIntegrationSuite) testListResultForUpsertSearchAttributes(listRequest *types.ListWorkflowExecutionsRequest) { + verified := false + for i := 0; i < numOfRetry; i++ { + resp, err := s.engine.ListWorkflowExecutions(createContext(), listRequest) + s.Nil(err) + if len(resp.GetExecutions()) == 1 { + execution := resp.GetExecutions()[0] + retrievedSearchAttr := execution.SearchAttributes + if retrievedSearchAttr != nil && len(retrievedSearchAttr.GetIndexedFields()) == 3 { + fields := retrievedSearchAttr.GetIndexedFields() + searchValBytes := fields[s.testSearchAttributeKey] + var searchVal string + err := json.Unmarshal(searchValBytes, &searchVal) + s.Nil(err) + s.Equal("another string", searchVal) + + searchValBytes2 := fields[definition.CustomIntField] + var searchVal2 int + err = json.Unmarshal(searchValBytes2, &searchVal2) + s.Nil(err) + s.Equal(123, searchVal2) + + binaryChecksumsBytes := fields[definition.BinaryChecksums] + var binaryChecksums []string + err = json.Unmarshal(binaryChecksumsBytes, &binaryChecksums) + s.Nil(err) + s.Equal([]string{"binary-v1", "binary-v2"}, binaryChecksums) + + verified = true + break + } + } + time.Sleep(waitTimeInMs * time.Millisecond) + } + s.True(verified) +} + +func getUpsertSearchAttributes() *types.SearchAttributes { + attrValBytes1, _ := json.Marshal("another string") + attrValBytes2, _ := json.Marshal(123) + binaryChecksums, _ := json.Marshal([]string{"binary-v1", "binary-v2"}) + upsertSearchAttr := &types.SearchAttributes{ + IndexedFields: map[string][]byte{ + definition.CustomStringField: attrValBytes1, + definition.CustomIntField: attrValBytes2, + definition.BinaryChecksums: binaryChecksums, + }, + } + return upsertSearchAttr +} + +func (s *ElasticSearchIntegrationSuite) TestUpsertWorkflowExecution_InvalidKey() { + id := "es-integration-upsert-workflow-failed-test" + wt := "es-integration-upsert-workflow-failed-test-type" + tl := "es-integration-upsert-workflow-failed-test-tasklist" + identity := "worker1" + + workflowType := &types.WorkflowType{} + workflowType.Name = wt + + taskList := &types.TaskList{} + taskList.Name = tl + + request := &types.StartWorkflowExecutionRequest{ + RequestID: uuid.New(), + Domain: s.domainName, + WorkflowID: id, + WorkflowType: workflowType, + TaskList: taskList, + Input: nil, + ExecutionStartToCloseTimeoutSeconds: common.Int32Ptr(100), + TaskStartToCloseTimeoutSeconds: common.Int32Ptr(1), + Identity: identity, + } + + we, err0 := s.engine.StartWorkflowExecution(createContext(), request) + s.Nil(err0) + + s.Logger.Info("StartWorkflowExecution", tag.WorkflowRunID(we.RunID)) + + dtHandler := func(execution *types.WorkflowExecution, wt *types.WorkflowType, + previousStartedEventID, startedEventID int64, history *types.History) ([]byte, []*types.Decision, error) { + + upsertDecision := &types.Decision{ + DecisionType: types.DecisionTypeUpsertWorkflowSearchAttributes.Ptr(), + UpsertWorkflowSearchAttributesDecisionAttributes: &types.UpsertWorkflowSearchAttributesDecisionAttributes{ + SearchAttributes: &types.SearchAttributes{ + IndexedFields: map[string][]byte{ + "INVALIDKEY": []byte(`1`), + }, + }, + }} + return nil, []*types.Decision{upsertDecision}, nil + } + + poller := &TaskPoller{ + Engine: s.engine, + Domain: s.domainName, + TaskList: taskList, + StickyTaskList: taskList, + Identity: identity, + DecisionHandler: dtHandler, + Logger: s.Logger, + T: s.T(), + } + + _, err := poller.PollAndProcessDecisionTask(false, false) + s.Nil(err) + + historyResponse, err := s.engine.GetWorkflowExecutionHistory(createContext(), &types.GetWorkflowExecutionHistoryRequest{ + Domain: s.domainName, + Execution: &types.WorkflowExecution{ + WorkflowID: id, + RunID: we.RunID, + }, + }) + s.Nil(err) + history := historyResponse.History + decisionFailedEvent := history.GetEvents()[3] + s.Equal(types.EventTypeDecisionTaskFailed, decisionFailedEvent.GetEventType()) + failedDecisionAttr := decisionFailedEvent.DecisionTaskFailedEventAttributes + s.Equal(types.DecisionTaskFailedCauseBadSearchAttributes, failedDecisionAttr.GetCause()) + s.True(len(failedDecisionAttr.GetDetails()) > 0) +} + +func (s *ElasticSearchIntegrationSuite) putIndexSettings(indexName string, maxResultWindowSize int) { + err := s.esClient.PutMaxResultWindow(indexName, maxResultWindowSize) + s.Require().NoError(err) + s.verifyMaxResultWindowSize(indexName, maxResultWindowSize) +} + +func (s *ElasticSearchIntegrationSuite) verifyMaxResultWindowSize(indexName string, targetSize int) { + for i := 0; i < numOfRetry; i++ { + currentWindow, err := s.esClient.GetMaxResultWindow(indexName) + s.Require().NoError(err) + if currentWindow == strconv.Itoa(targetSize) { + return + } + time.Sleep(waitTimeInMs * time.Millisecond) + } + s.FailNow(fmt.Sprintf("ES max result window size hasn't reach target size within %v.", (numOfRetry*waitTimeInMs)*time.Millisecond)) +} diff --git a/host/elasticsearch_test.go b/host/elasticsearch_test.go index 5bcbe597d18..fafd111c8f0 100644 --- a/host/elasticsearch_test.go +++ b/host/elasticsearch_test.go @@ -18,72 +18,15 @@ // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN // THE SOFTWARE. -// +build esintegration - -// to run locally, make sure kafka and es is running, -// then run cmd `go test -v ./host -run TestElasticsearchIntegrationSuite -tags esintegration` package host import ( - "bytes" - "encoding/json" "flag" - "fmt" - "strconv" "testing" - "time" - "github.com/pborman/uuid" - "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" - - "github.com/uber/cadence/common" - "github.com/uber/cadence/common/definition" - "github.com/uber/cadence/common/log/tag" - "github.com/uber/cadence/common/types" - "github.com/uber/cadence/environment" - "github.com/uber/cadence/host/esutils" -) - -const ( - numOfRetry = 50 - waitTimeInMs = 400 - waitForESToSettle = 4 * time.Second // wait es shards for some time ensure data consistent ) -type ElasticSearchIntegrationSuite struct { - // override suite.Suite.Assertions with require.Assertions; this means that s.NotNil(nil) will stop the test, - // not merely log an error - *require.Assertions - IntegrationBase - esClient esutils.ESClient - - testSearchAttributeKey string - testSearchAttributeVal string -} - -// This cluster use customized threshold for history config -func (s *ElasticSearchIntegrationSuite) SetupSuite() { - s.setupSuite() - s.esClient = esutils.CreateESClient(s.Suite, s.testClusterConfig.ESConfig.URL.String(), environment.GetESVersion()) - s.esClient.PutIndexTemplate(s.Suite, "testdata/es_"+environment.GetESVersion()+"_index_template.json", "test-visibility-template") - indexName := s.testClusterConfig.ESConfig.Indices[common.VisibilityAppName] - s.esClient.CreateIndex(s.Suite, indexName) - s.putIndexSettings(indexName, defaultTestValueOfESIndexMaxResultWindow) -} - -func (s *ElasticSearchIntegrationSuite) TearDownSuite() { - s.tearDownSuite() - s.esClient.DeleteIndex(s.Suite, s.testClusterConfig.ESConfig.Indices[common.VisibilityAppName]) -} - -func (s *ElasticSearchIntegrationSuite) SetupTest() { - // Have to define our overridden assertions in the test setup. If we did it earlier, s.T() will return nil - s.Assertions = require.New(s.T()) - s.testSearchAttributeKey = definition.CustomStringField - s.testSearchAttributeVal = "test value" -} - func TestElasticsearchIntegrationSuite(t *testing.T) { flag.Parse() @@ -102,1008 +45,3 @@ func TestElasticsearchIntegrationSuite(t *testing.T) { s.IntegrationBase = NewIntegrationBase(params) suite.Run(t, s) } - -func (s *ElasticSearchIntegrationSuite) TestListOpenWorkflow() { - id := "es-integration-start-workflow-test" - wt := "es-integration-start-workflow-test-type" - tl := "es-integration-start-workflow-test-tasklist" - request := s.createStartWorkflowExecutionRequest(id, wt, tl) - - attrValBytes, _ := json.Marshal(s.testSearchAttributeVal) - searchAttr := &types.SearchAttributes{ - IndexedFields: map[string][]byte{ - s.testSearchAttributeKey: attrValBytes, - }, - } - request.SearchAttributes = searchAttr - - startTime := time.Now().UnixNano() - we, err := s.engine.StartWorkflowExecution(createContext(), request) - s.Nil(err) - - startFilter := &types.StartTimeFilter{} - startFilter.EarliestTime = common.Int64Ptr(startTime) - var openExecution *types.WorkflowExecutionInfo - for i := 0; i < numOfRetry; i++ { - startFilter.LatestTime = common.Int64Ptr(time.Now().UnixNano()) - resp, err := s.engine.ListOpenWorkflowExecutions(createContext(), &types.ListOpenWorkflowExecutionsRequest{ - Domain: s.domainName, - MaximumPageSize: defaultTestValueOfESIndexMaxResultWindow, - StartTimeFilter: startFilter, - ExecutionFilter: &types.WorkflowExecutionFilter{ - WorkflowID: id, - }, - }) - s.Nil(err) - if len(resp.GetExecutions()) == 1 { - openExecution = resp.GetExecutions()[0] - break - } - time.Sleep(waitTimeInMs * time.Millisecond) - } - s.NotNil(openExecution) - s.Equal(we.GetRunID(), openExecution.GetExecution().GetRunID()) - s.Equal(attrValBytes, openExecution.SearchAttributes.GetIndexedFields()[s.testSearchAttributeKey]) -} - -func (s *ElasticSearchIntegrationSuite) TestListWorkflow() { - id := "es-integration-list-workflow-test" - wt := "es-integration-list-workflow-test-type" - tl := "es-integration-list-workflow-test-tasklist" - request := s.createStartWorkflowExecutionRequest(id, wt, tl) - - we, err := s.engine.StartWorkflowExecution(createContext(), request) - s.Nil(err) - - query := fmt.Sprintf(`WorkflowID = "%s"`, id) - s.testHelperForReadOnce(we.GetRunID(), query, false) -} - -func (s *ElasticSearchIntegrationSuite) TestListWorkflow_ExecutionTime() { - id := "es-integration-list-workflow-execution-time-test" - wt := "es-integration-list-workflow-execution-time-test-type" - tl := "es-integration-list-workflow-execution-time-test-tasklist" - request := s.createStartWorkflowExecutionRequest(id, wt, tl) - - we, err := s.engine.StartWorkflowExecution(createContext(), request) - s.Nil(err) - - cronID := id + "-cron" - request.CronSchedule = "@every 1m" - request.WorkflowID = cronID - - weCron, err := s.engine.StartWorkflowExecution(createContext(), request) - s.Nil(err) - - query := fmt.Sprintf(`(WorkflowID = "%s" or WorkflowID = "%s") and ExecutionTime < %v`, id, cronID, time.Now().UnixNano()+int64(time.Minute)) - s.testHelperForReadOnce(weCron.GetRunID(), query, false) - - query = fmt.Sprintf(`WorkflowID = "%s"`, id) - s.testHelperForReadOnce(we.GetRunID(), query, false) -} - -func (s *ElasticSearchIntegrationSuite) TestListWorkflow_SearchAttribute() { - id := "es-integration-list-workflow-by-search-attr-test" - wt := "es-integration-list-workflow-by-search-attr-test-type" - tl := "es-integration-list-workflow-by-search-attr-test-tasklist" - request := s.createStartWorkflowExecutionRequest(id, wt, tl) - - attrValBytes, _ := json.Marshal(s.testSearchAttributeVal) - searchAttr := &types.SearchAttributes{ - IndexedFields: map[string][]byte{ - s.testSearchAttributeKey: attrValBytes, - }, - } - request.SearchAttributes = searchAttr - - we, err := s.engine.StartWorkflowExecution(createContext(), request) - s.Nil(err) - query := fmt.Sprintf(`WorkflowID = "%s" and %s = "%s"`, id, s.testSearchAttributeKey, s.testSearchAttributeVal) - s.testHelperForReadOnce(we.GetRunID(), query, false) - - // test upsert - dtHandler := func(execution *types.WorkflowExecution, wt *types.WorkflowType, - previousStartedEventID, startedEventID int64, history *types.History) ([]byte, []*types.Decision, error) { - - upsertDecision := &types.Decision{ - DecisionType: types.DecisionTypeUpsertWorkflowSearchAttributes.Ptr(), - UpsertWorkflowSearchAttributesDecisionAttributes: &types.UpsertWorkflowSearchAttributesDecisionAttributes{ - SearchAttributes: getUpsertSearchAttributes(), - }} - - return nil, []*types.Decision{upsertDecision}, nil - } - taskList := &types.TaskList{Name: tl} - poller := &TaskPoller{ - Engine: s.engine, - Domain: s.domainName, - TaskList: taskList, - StickyTaskList: taskList, - Identity: "worker1", - DecisionHandler: dtHandler, - Logger: s.Logger, - T: s.T(), - } - _, newTask, err := poller.PollAndProcessDecisionTaskWithAttemptAndRetryAndForceNewDecision( - false, - false, - true, - true, - int64(0), - 1, - true, - nil) - s.Nil(err) - s.NotNil(newTask) - s.NotNil(newTask.DecisionTask) - - time.Sleep(waitForESToSettle) - - listRequest := &types.ListWorkflowExecutionsRequest{ - Domain: s.domainName, - PageSize: int32(2), - Query: fmt.Sprintf(`WorkflowType = '%s' and CloseTime = missing and BinaryChecksums = 'binary-v1'`, wt), - } - // verify upsert data is on ES - s.testListResultForUpsertSearchAttributes(listRequest) - - // verify DescribeWorkflowExecution - descRequest := &types.DescribeWorkflowExecutionRequest{ - Domain: s.domainName, - Execution: &types.WorkflowExecution{ - WorkflowID: id, - }, - } - descResp, err := s.engine.DescribeWorkflowExecution(createContext(), descRequest) - s.Nil(err) - expectedSearchAttributes := getUpsertSearchAttributes() - s.Equal(expectedSearchAttributes, descResp.WorkflowExecutionInfo.GetSearchAttributes()) -} - -func (s *ElasticSearchIntegrationSuite) TestListWorkflow_PageToken() { - id := "es-integration-list-workflow-token-test" - wt := "es-integration-list-workflow-token-test-type" - tl := "es-integration-list-workflow-token-test-tasklist" - request := s.createStartWorkflowExecutionRequest(id, wt, tl) - - numOfWorkflows := defaultTestValueOfESIndexMaxResultWindow - 1 // == 4 - pageSize := 3 - - s.testListWorkflowHelper(numOfWorkflows, pageSize, request, id, wt, false) -} - -func (s *ElasticSearchIntegrationSuite) TestListWorkflow_SearchAfter() { - id := "es-integration-list-workflow-searchAfter-test" - wt := "es-integration-list-workflow-searchAfter-test-type" - tl := "es-integration-list-workflow-searchAfter-test-tasklist" - request := s.createStartWorkflowExecutionRequest(id, wt, tl) - - numOfWorkflows := defaultTestValueOfESIndexMaxResultWindow + 1 // == 6 - pageSize := 4 - - s.testListWorkflowHelper(numOfWorkflows, pageSize, request, id, wt, false) -} - -func (s *ElasticSearchIntegrationSuite) TestListWorkflow_OrQuery() { - id := "es-integration-list-workflow-or-query-test" - wt := "es-integration-list-workflow-or-query-test-type" - tl := "es-integration-list-workflow-or-query-test-tasklist" - request := s.createStartWorkflowExecutionRequest(id, wt, tl) - - // start 3 workflows - key := definition.CustomIntField - attrValBytes, _ := json.Marshal(1) - searchAttr := &types.SearchAttributes{ - IndexedFields: map[string][]byte{ - key: attrValBytes, - }, - } - request.SearchAttributes = searchAttr - we1, err := s.engine.StartWorkflowExecution(createContext(), request) - s.Nil(err) - - request.RequestID = uuid.New() - request.WorkflowID = id + "-2" - attrValBytes, _ = json.Marshal(2) - searchAttr.IndexedFields[key] = attrValBytes - we2, err := s.engine.StartWorkflowExecution(createContext(), request) - s.Nil(err) - - request.RequestID = uuid.New() - request.WorkflowID = id + "-3" - attrValBytes, _ = json.Marshal(3) - searchAttr.IndexedFields[key] = attrValBytes - we3, err := s.engine.StartWorkflowExecution(createContext(), request) - s.Nil(err) - - time.Sleep(waitForESToSettle) - - // query 1 workflow with search attr - query1 := fmt.Sprintf(`CustomIntField = %d`, 1) - var openExecution *types.WorkflowExecutionInfo - listRequest := &types.ListWorkflowExecutionsRequest{ - Domain: s.domainName, - PageSize: defaultTestValueOfESIndexMaxResultWindow, - Query: query1, - } - for i := 0; i < numOfRetry; i++ { - resp, err := s.engine.ListWorkflowExecutions(createContext(), listRequest) - s.Nil(err) - if len(resp.GetExecutions()) == 1 { - openExecution = resp.GetExecutions()[0] - break - } - time.Sleep(waitTimeInMs * time.Millisecond) - } - s.NotNil(openExecution) - s.Equal(we1.GetRunID(), openExecution.GetExecution().GetRunID()) - s.True(openExecution.GetExecutionTime() >= openExecution.GetStartTime()) - searchValBytes := openExecution.SearchAttributes.GetIndexedFields()[key] - var searchVal int - json.Unmarshal(searchValBytes, &searchVal) - s.Equal(1, searchVal) - - // query with or clause - query2 := fmt.Sprintf(`CustomIntField = %d or CustomIntField = %d`, 1, 2) - listRequest.Query = query2 - var openExecutions []*types.WorkflowExecutionInfo - for i := 0; i < numOfRetry; i++ { - resp, err := s.engine.ListWorkflowExecutions(createContext(), listRequest) - s.Nil(err) - if len(resp.GetExecutions()) == 2 { - openExecutions = resp.GetExecutions() - break - } - time.Sleep(waitTimeInMs * time.Millisecond) - } - s.Equal(2, len(openExecutions)) - e1 := openExecutions[0] - e2 := openExecutions[1] - if e1.GetExecution().GetRunID() != we1.GetRunID() { - // results are sorted by [CloseTime,RunID] desc, so find the correct mapping first - e1, e2 = e2, e1 - } - s.Equal(we1.GetRunID(), e1.GetExecution().GetRunID()) - s.Equal(we2.GetRunID(), e2.GetExecution().GetRunID()) - searchValBytes = e2.SearchAttributes.GetIndexedFields()[key] - json.Unmarshal(searchValBytes, &searchVal) - s.Equal(2, searchVal) - - // query for open - query3 := fmt.Sprintf(`(CustomIntField = %d or CustomIntField = %d) and CloseTime = missing`, 2, 3) - listRequest.Query = query3 - for i := 0; i < numOfRetry; i++ { - resp, err := s.engine.ListWorkflowExecutions(createContext(), listRequest) - s.Nil(err) - if len(resp.GetExecutions()) == 2 { - openExecutions = resp.GetExecutions() - break - } - time.Sleep(waitTimeInMs * time.Millisecond) - } - s.Equal(2, len(openExecutions)) - e1 = openExecutions[0] - e2 = openExecutions[1] - s.Equal(we3.GetRunID(), e1.GetExecution().GetRunID()) - s.Equal(we2.GetRunID(), e2.GetExecution().GetRunID()) - searchValBytes = e1.SearchAttributes.GetIndexedFields()[key] - json.Unmarshal(searchValBytes, &searchVal) - s.Equal(3, searchVal) -} - -// To test last page search trigger max window size error -func (s *ElasticSearchIntegrationSuite) TestListWorkflow_MaxWindowSize() { - id := "es-integration-list-workflow-max-window-size-test" - wt := "es-integration-list-workflow-max-window-size-test-type" - tl := "es-integration-list-workflow-max-window-size-test-tasklist" - startRequest := s.createStartWorkflowExecutionRequest(id, wt, tl) - - for i := 0; i < defaultTestValueOfESIndexMaxResultWindow; i++ { - startRequest.RequestID = uuid.New() - startRequest.WorkflowID = id + strconv.Itoa(i) - _, err := s.engine.StartWorkflowExecution(createContext(), startRequest) - s.Nil(err) - } - - time.Sleep(waitForESToSettle) - - var listResp *types.ListWorkflowExecutionsResponse - var nextPageToken []byte - - listRequest := &types.ListWorkflowExecutionsRequest{ - Domain: s.domainName, - PageSize: int32(defaultTestValueOfESIndexMaxResultWindow), - NextPageToken: nextPageToken, - Query: fmt.Sprintf(`WorkflowType = '%s' and CloseTime = missing`, wt), - } - // get first page - for i := 0; i < numOfRetry; i++ { - resp, err := s.engine.ListWorkflowExecutions(createContext(), listRequest) - s.Nil(err) - if len(resp.GetExecutions()) == defaultTestValueOfESIndexMaxResultWindow { - listResp = resp - break - } - time.Sleep(waitTimeInMs * time.Millisecond) - } - s.NotNil(listResp) - s.True(len(listResp.GetNextPageToken()) != 0) - - // the last request - listRequest.NextPageToken = listResp.GetNextPageToken() - resp, err := s.engine.ListWorkflowExecutions(createContext(), listRequest) - s.Nil(err) - s.True(len(resp.GetExecutions()) == 0) - s.True(len(resp.GetNextPageToken()) == 0) -} - -func (s *ElasticSearchIntegrationSuite) TestListWorkflow_OrderBy() { - id := "es-integration-list-workflow-order-by-test" - wt := "es-integration-list-workflow-order-by-test-type" - tl := "es-integration-list-workflow-order-by-test-tasklist" - startRequest := s.createStartWorkflowExecutionRequest(id, wt, tl) - - for i := 0; i < defaultTestValueOfESIndexMaxResultWindow+1; i++ { // start 6 - startRequest.RequestID = uuid.New() - startRequest.WorkflowID = id + strconv.Itoa(i) - - if i < defaultTestValueOfESIndexMaxResultWindow-1 { // 4 workflow has search attr - intVal, _ := json.Marshal(i) - doubleVal, _ := json.Marshal(float64(i)) - strVal, _ := json.Marshal(strconv.Itoa(i)) - timeVal, _ := json.Marshal(time.Now()) - searchAttr := &types.SearchAttributes{ - IndexedFields: map[string][]byte{ - definition.CustomIntField: intVal, - definition.CustomDoubleField: doubleVal, - definition.CustomKeywordField: strVal, - definition.CustomDatetimeField: timeVal, - }, - } - startRequest.SearchAttributes = searchAttr - } else { - startRequest.SearchAttributes = &types.SearchAttributes{} - } - - _, err := s.engine.StartWorkflowExecution(createContext(), startRequest) - s.Nil(err) - } - - time.Sleep(waitForESToSettle) - - desc := "desc" - asc := "asc" - queryTemplate := `WorkflowType = "%s" order by %s %s` - pageSize := int32(defaultTestValueOfESIndexMaxResultWindow) - - // order by CloseTime asc - query1 := fmt.Sprintf(queryTemplate, wt, definition.CloseTime, asc) - var openExecutions []*types.WorkflowExecutionInfo - listRequest := &types.ListWorkflowExecutionsRequest{ - Domain: s.domainName, - PageSize: pageSize, - Query: query1, - } - for i := 0; i < numOfRetry; i++ { - resp, err := s.engine.ListWorkflowExecutions(createContext(), listRequest) - s.Nil(err) - if int32(len(resp.GetExecutions())) == listRequest.GetPageSize() { - openExecutions = resp.GetExecutions() - break - } - time.Sleep(waitTimeInMs * time.Millisecond) - } - s.NotNil(openExecutions) - for i := int32(1); i < pageSize; i++ { - s.True(openExecutions[i-1].GetCloseTime() <= openExecutions[i].GetCloseTime()) - } - - // greatest effort to reduce duplicate code - testHelper := func(query, searchAttrKey string, prevVal, currVal interface{}) { - listRequest.Query = query - listRequest.NextPageToken = []byte{} - resp, err := s.engine.ListWorkflowExecutions(createContext(), listRequest) - s.Nil(err) - openExecutions = resp.GetExecutions() - dec := json.NewDecoder(bytes.NewReader(openExecutions[0].GetSearchAttributes().GetIndexedFields()[searchAttrKey])) - dec.UseNumber() - err = dec.Decode(&prevVal) - s.Nil(err) - for i := int32(1); i < pageSize; i++ { - indexedFields := openExecutions[i].GetSearchAttributes().GetIndexedFields() - searchAttrBytes, ok := indexedFields[searchAttrKey] - if !ok { // last one doesn't have search attr - s.Equal(pageSize-1, i) - break - } - dec := json.NewDecoder(bytes.NewReader(searchAttrBytes)) - dec.UseNumber() - err = dec.Decode(&currVal) - s.Nil(err) - var v1, v2 interface{} - switch searchAttrKey { - case definition.CustomIntField: - v1, _ = prevVal.(json.Number).Int64() - v2, _ = currVal.(json.Number).Int64() - s.True(v1.(int64) >= v2.(int64)) - case definition.CustomDoubleField: - v1, _ = prevVal.(json.Number).Float64() - v2, _ = currVal.(json.Number).Float64() - s.True(v1.(float64) >= v2.(float64)) - case definition.CustomKeywordField: - s.True(prevVal.(string) >= currVal.(string)) - case definition.CustomDatetimeField: - v1, _ = time.Parse(time.RFC3339, prevVal.(string)) - v2, _ = time.Parse(time.RFC3339, currVal.(string)) - s.True(v1.(time.Time).After(v2.(time.Time))) - } - prevVal = currVal - } - listRequest.NextPageToken = resp.GetNextPageToken() - resp, err = s.engine.ListWorkflowExecutions(createContext(), listRequest) // last page - s.Nil(err) - s.Equal(1, len(resp.GetExecutions())) - } - - // order by CustomIntField desc - field := definition.CustomIntField - query := fmt.Sprintf(queryTemplate, wt, field, desc) - var int1, int2 int - testHelper(query, field, int1, int2) - - // order by CustomDoubleField desc - field = definition.CustomDoubleField - query = fmt.Sprintf(queryTemplate, wt, field, desc) - var double1, double2 float64 - testHelper(query, field, double1, double2) - - // order by CustomKeywordField desc - field = definition.CustomKeywordField - query = fmt.Sprintf(queryTemplate, wt, field, desc) - var s1, s2 string - testHelper(query, field, s1, s2) - - // order by CustomDatetimeField desc - field = definition.CustomDatetimeField - query = fmt.Sprintf(queryTemplate, wt, field, desc) - var t1, t2 time.Time - testHelper(query, field, t1, t2) -} - -func (s *ElasticSearchIntegrationSuite) testListWorkflowHelper(numOfWorkflows, pageSize int, - startRequest *types.StartWorkflowExecutionRequest, wid, wType string, isScan bool) { - - // start enough number of workflows - for i := 0; i < numOfWorkflows; i++ { - startRequest.RequestID = uuid.New() - startRequest.WorkflowID = wid + strconv.Itoa(i) - _, err := s.engine.StartWorkflowExecution(createContext(), startRequest) - s.Nil(err) - } - - time.Sleep(waitForESToSettle) - - var openExecutions []*types.WorkflowExecutionInfo - var nextPageToken []byte - - listRequest := &types.ListWorkflowExecutionsRequest{ - Domain: s.domainName, - PageSize: int32(pageSize), - NextPageToken: nextPageToken, - Query: fmt.Sprintf(`WorkflowType = '%s' and CloseTime = missing`, wType), - } - // test first page - for i := 0; i < numOfRetry; i++ { - var resp *types.ListWorkflowExecutionsResponse - var err error - - if isScan { - resp, err = s.engine.ScanWorkflowExecutions(createContext(), listRequest) - } else { - resp, err = s.engine.ListWorkflowExecutions(createContext(), listRequest) - } - s.Nil(err) - if len(resp.GetExecutions()) == pageSize { - openExecutions = resp.GetExecutions() - nextPageToken = resp.GetNextPageToken() - break - } - time.Sleep(waitTimeInMs * time.Millisecond) - } - s.NotNil(openExecutions) - s.NotNil(nextPageToken) - s.True(len(nextPageToken) > 0) - - // test last page - listRequest.NextPageToken = nextPageToken - inIf := false - for i := 0; i < numOfRetry; i++ { - var resp *types.ListWorkflowExecutionsResponse - var err error - - if isScan { - resp, err = s.engine.ScanWorkflowExecutions(createContext(), listRequest) - } else { - resp, err = s.engine.ListWorkflowExecutions(createContext(), listRequest) - } - s.Nil(err) - if len(resp.GetExecutions()) == numOfWorkflows-pageSize { - inIf = true - openExecutions = resp.GetExecutions() - nextPageToken = resp.GetNextPageToken() - break - } - time.Sleep(waitTimeInMs * time.Millisecond) - } - s.True(inIf) - s.NotNil(openExecutions) - s.Nil(nextPageToken) -} - -func (s *ElasticSearchIntegrationSuite) testHelperForReadOnce(runID, query string, isScan bool) { - var openExecution *types.WorkflowExecutionInfo - listRequest := &types.ListWorkflowExecutionsRequest{ - Domain: s.domainName, - PageSize: defaultTestValueOfESIndexMaxResultWindow, - Query: query, - } - for i := 0; i < numOfRetry; i++ { - var resp *types.ListWorkflowExecutionsResponse - var err error - - if isScan { - resp, err = s.engine.ScanWorkflowExecutions(createContext(), listRequest) - } else { - resp, err = s.engine.ListWorkflowExecutions(createContext(), listRequest) - } - - s.Nil(err) - if len(resp.GetExecutions()) == 1 { - openExecution = resp.GetExecutions()[0] - break - } - time.Sleep(waitTimeInMs * time.Millisecond) - } - s.NotNil(openExecution) - s.Equal(runID, openExecution.GetExecution().GetRunID()) - s.True(openExecution.GetExecutionTime() >= openExecution.GetStartTime()) - if openExecution.SearchAttributes != nil && len(openExecution.SearchAttributes.GetIndexedFields()) > 0 { - searchValBytes := openExecution.SearchAttributes.GetIndexedFields()[s.testSearchAttributeKey] - var searchVal string - json.Unmarshal(searchValBytes, &searchVal) - s.Equal(s.testSearchAttributeVal, searchVal) - } -} - -func (s *ElasticSearchIntegrationSuite) TestScanWorkflow() { - id := "es-integration-scan-workflow-test" - wt := "es-integration-scan-workflow-test-type" - tl := "es-integration-scan-workflow-test-tasklist" - identity := "worker1" - - workflowType := &types.WorkflowType{} - workflowType.Name = wt - - taskList := &types.TaskList{} - taskList.Name = tl - - request := &types.StartWorkflowExecutionRequest{ - RequestID: uuid.New(), - Domain: s.domainName, - WorkflowID: id, - WorkflowType: workflowType, - TaskList: taskList, - Input: nil, - ExecutionStartToCloseTimeoutSeconds: common.Int32Ptr(100), - TaskStartToCloseTimeoutSeconds: common.Int32Ptr(1), - Identity: identity, - } - - we, err := s.engine.StartWorkflowExecution(createContext(), request) - s.Nil(err) - query := fmt.Sprintf(`WorkflowID = "%s"`, id) - s.testHelperForReadOnce(we.GetRunID(), query, true) -} - -func (s *ElasticSearchIntegrationSuite) TestScanWorkflow_SearchAttribute() { - id := "es-integration-scan-workflow-search-attr-test" - wt := "es-integration-scan-workflow-search-attr-test-type" - tl := "es-integration-scan-workflow-search-attr-test-tasklist" - request := s.createStartWorkflowExecutionRequest(id, wt, tl) - - attrValBytes, _ := json.Marshal(s.testSearchAttributeVal) - searchAttr := &types.SearchAttributes{ - IndexedFields: map[string][]byte{ - s.testSearchAttributeKey: attrValBytes, - }, - } - request.SearchAttributes = searchAttr - - we, err := s.engine.StartWorkflowExecution(createContext(), request) - s.Nil(err) - query := fmt.Sprintf(`WorkflowID = "%s" and %s = "%s"`, id, s.testSearchAttributeKey, s.testSearchAttributeVal) - s.testHelperForReadOnce(we.GetRunID(), query, true) -} - -func (s *ElasticSearchIntegrationSuite) TestScanWorkflow_PageToken() { - id := "es-integration-scan-workflow-token-test" - wt := "es-integration-scan-workflow-token-test-type" - tl := "es-integration-scan-workflow-token-test-tasklist" - identity := "worker1" - - workflowType := &types.WorkflowType{} - workflowType.Name = wt - - taskList := &types.TaskList{} - taskList.Name = tl - - request := &types.StartWorkflowExecutionRequest{ - Domain: s.domainName, - WorkflowType: workflowType, - TaskList: taskList, - Input: nil, - ExecutionStartToCloseTimeoutSeconds: common.Int32Ptr(100), - TaskStartToCloseTimeoutSeconds: common.Int32Ptr(1), - Identity: identity, - } - - numOfWorkflows := 4 - pageSize := 3 - - s.testListWorkflowHelper(numOfWorkflows, pageSize, request, id, wt, true) -} - -func (s *ElasticSearchIntegrationSuite) TestCountWorkflow() { - id := "es-integration-count-workflow-test" - wt := "es-integration-count-workflow-test-type" - tl := "es-integration-count-workflow-test-tasklist" - request := s.createStartWorkflowExecutionRequest(id, wt, tl) - - attrValBytes, _ := json.Marshal(s.testSearchAttributeVal) - searchAttr := &types.SearchAttributes{ - IndexedFields: map[string][]byte{ - s.testSearchAttributeKey: attrValBytes, - }, - } - request.SearchAttributes = searchAttr - - _, err := s.engine.StartWorkflowExecution(createContext(), request) - s.Nil(err) - - query := fmt.Sprintf(`WorkflowID = "%s" and %s = "%s"`, id, s.testSearchAttributeKey, s.testSearchAttributeVal) - countRequest := &types.CountWorkflowExecutionsRequest{ - Domain: s.domainName, - Query: query, - } - var resp *types.CountWorkflowExecutionsResponse - for i := 0; i < numOfRetry; i++ { - resp, err = s.engine.CountWorkflowExecutions(createContext(), countRequest) - s.Nil(err) - if resp.GetCount() == int64(1) { - break - } - time.Sleep(waitTimeInMs * time.Millisecond) - } - s.Equal(int64(1), resp.GetCount()) - - query = fmt.Sprintf(`WorkflowID = "%s" and %s = "%s"`, id, s.testSearchAttributeKey, "noMatch") - countRequest.Query = query - resp, err = s.engine.CountWorkflowExecutions(createContext(), countRequest) - s.Nil(err) - s.Equal(int64(0), resp.GetCount()) -} - -func (s *ElasticSearchIntegrationSuite) createStartWorkflowExecutionRequest(id, wt, tl string) *types.StartWorkflowExecutionRequest { - identity := "worker1" - workflowType := &types.WorkflowType{} - workflowType.Name = wt - - taskList := &types.TaskList{} - taskList.Name = tl - - request := &types.StartWorkflowExecutionRequest{ - RequestID: uuid.New(), - Domain: s.domainName, - WorkflowID: id, - WorkflowType: workflowType, - TaskList: taskList, - Input: nil, - ExecutionStartToCloseTimeoutSeconds: common.Int32Ptr(100), - TaskStartToCloseTimeoutSeconds: common.Int32Ptr(1), - Identity: identity, - } - return request -} - -func (s *ElasticSearchIntegrationSuite) TestUpsertWorkflowExecution() { - id := "es-integration-upsert-workflow-test" - wt := "es-integration-upsert-workflow-test-type" - tl := "es-integration-upsert-workflow-test-tasklist" - identity := "worker1" - - workflowType := &types.WorkflowType{} - workflowType.Name = wt - - taskList := &types.TaskList{} - taskList.Name = tl - - request := &types.StartWorkflowExecutionRequest{ - RequestID: uuid.New(), - Domain: s.domainName, - WorkflowID: id, - WorkflowType: workflowType, - TaskList: taskList, - Input: nil, - ExecutionStartToCloseTimeoutSeconds: common.Int32Ptr(100), - TaskStartToCloseTimeoutSeconds: common.Int32Ptr(1), - Identity: identity, - } - - we, err0 := s.engine.StartWorkflowExecution(createContext(), request) - s.Nil(err0) - - s.Logger.Info("StartWorkflowExecution", tag.WorkflowRunID(we.RunID)) - - decisionCount := 0 - dtHandler := func(execution *types.WorkflowExecution, wt *types.WorkflowType, - previousStartedEventID, startedEventID int64, history *types.History) ([]byte, []*types.Decision, error) { - - upsertDecision := &types.Decision{ - DecisionType: types.DecisionTypeUpsertWorkflowSearchAttributes.Ptr(), - UpsertWorkflowSearchAttributesDecisionAttributes: &types.UpsertWorkflowSearchAttributesDecisionAttributes{}} - - // handle first upsert - if decisionCount == 0 { - decisionCount++ - - attrValBytes, _ := json.Marshal(s.testSearchAttributeVal) - upsertSearchAttr := &types.SearchAttributes{ - IndexedFields: map[string][]byte{ - s.testSearchAttributeKey: attrValBytes, - }, - } - upsertDecision.UpsertWorkflowSearchAttributesDecisionAttributes.SearchAttributes = upsertSearchAttr - return nil, []*types.Decision{upsertDecision}, nil - } - // handle second upsert, which update existing field and add new field - if decisionCount == 1 { - decisionCount++ - upsertDecision.UpsertWorkflowSearchAttributesDecisionAttributes.SearchAttributes = getUpsertSearchAttributes() - return nil, []*types.Decision{upsertDecision}, nil - } - - return nil, []*types.Decision{{ - DecisionType: types.DecisionTypeCompleteWorkflowExecution.Ptr(), - CompleteWorkflowExecutionDecisionAttributes: &types.CompleteWorkflowExecutionDecisionAttributes{ - Result: []byte("Done."), - }, - }}, nil - } - - poller := &TaskPoller{ - Engine: s.engine, - Domain: s.domainName, - TaskList: taskList, - StickyTaskList: taskList, - Identity: identity, - DecisionHandler: dtHandler, - Logger: s.Logger, - T: s.T(), - } - - // process 1st decision and assert decision is handled correctly. - _, newTask, err := poller.PollAndProcessDecisionTaskWithAttemptAndRetryAndForceNewDecision( - false, - false, - true, - true, - int64(0), - 1, - true, - nil) - s.Nil(err) - s.NotNil(newTask) - s.NotNil(newTask.DecisionTask) - s.Equal(int64(3), newTask.DecisionTask.GetPreviousStartedEventID()) - s.Equal(int64(7), newTask.DecisionTask.GetStartedEventID()) - s.Equal(4, len(newTask.DecisionTask.History.Events)) - s.Equal(types.EventTypeDecisionTaskCompleted, newTask.DecisionTask.History.Events[0].GetEventType()) - s.Equal(types.EventTypeUpsertWorkflowSearchAttributes, newTask.DecisionTask.History.Events[1].GetEventType()) - s.Equal(types.EventTypeDecisionTaskScheduled, newTask.DecisionTask.History.Events[2].GetEventType()) - s.Equal(types.EventTypeDecisionTaskStarted, newTask.DecisionTask.History.Events[3].GetEventType()) - - time.Sleep(waitForESToSettle) - - // verify upsert data is on ES - listRequest := &types.ListWorkflowExecutionsRequest{ - Domain: s.domainName, - PageSize: int32(2), - Query: fmt.Sprintf(`WorkflowType = '%s' and CloseTime = missing`, wt), - } - verified := false - for i := 0; i < numOfRetry; i++ { - resp, err := s.engine.ListWorkflowExecutions(createContext(), listRequest) - s.Nil(err) - if len(resp.GetExecutions()) == 1 { - execution := resp.GetExecutions()[0] - retrievedSearchAttr := execution.SearchAttributes - if retrievedSearchAttr != nil && len(retrievedSearchAttr.GetIndexedFields()) > 0 { - searchValBytes := retrievedSearchAttr.GetIndexedFields()[s.testSearchAttributeKey] - var searchVal string - json.Unmarshal(searchValBytes, &searchVal) - s.Equal(s.testSearchAttributeVal, searchVal) - verified = true - break - } - } - time.Sleep(waitTimeInMs * time.Millisecond) - } - s.True(verified) - - // process 2nd decision and assert decision is handled correctly. - _, newTask, err = poller.PollAndProcessDecisionTaskWithAttemptAndRetryAndForceNewDecision( - false, - false, - true, - true, - int64(0), - 1, - true, - nil) - s.Nil(err) - s.NotNil(newTask) - s.NotNil(newTask.DecisionTask) - s.Equal(4, len(newTask.DecisionTask.History.Events)) - s.Equal(types.EventTypeDecisionTaskCompleted, newTask.DecisionTask.History.Events[0].GetEventType()) - s.Equal(types.EventTypeUpsertWorkflowSearchAttributes, newTask.DecisionTask.History.Events[1].GetEventType()) - s.Equal(types.EventTypeDecisionTaskScheduled, newTask.DecisionTask.History.Events[2].GetEventType()) - s.Equal(types.EventTypeDecisionTaskStarted, newTask.DecisionTask.History.Events[3].GetEventType()) - - time.Sleep(waitForESToSettle) - - // verify upsert data is on ES - s.testListResultForUpsertSearchAttributes(listRequest) -} - -func (s *ElasticSearchIntegrationSuite) testListResultForUpsertSearchAttributes(listRequest *types.ListWorkflowExecutionsRequest) { - verified := false - for i := 0; i < numOfRetry; i++ { - resp, err := s.engine.ListWorkflowExecutions(createContext(), listRequest) - s.Nil(err) - if len(resp.GetExecutions()) == 1 { - execution := resp.GetExecutions()[0] - retrievedSearchAttr := execution.SearchAttributes - if retrievedSearchAttr != nil && len(retrievedSearchAttr.GetIndexedFields()) == 3 { - fields := retrievedSearchAttr.GetIndexedFields() - searchValBytes := fields[s.testSearchAttributeKey] - var searchVal string - err := json.Unmarshal(searchValBytes, &searchVal) - s.Nil(err) - s.Equal("another string", searchVal) - - searchValBytes2 := fields[definition.CustomIntField] - var searchVal2 int - err = json.Unmarshal(searchValBytes2, &searchVal2) - s.Nil(err) - s.Equal(123, searchVal2) - - binaryChecksumsBytes := fields[definition.BinaryChecksums] - var binaryChecksums []string - err = json.Unmarshal(binaryChecksumsBytes, &binaryChecksums) - s.Nil(err) - s.Equal([]string{"binary-v1", "binary-v2"}, binaryChecksums) - - verified = true - break - } - } - time.Sleep(waitTimeInMs * time.Millisecond) - } - s.True(verified) -} - -func getUpsertSearchAttributes() *types.SearchAttributes { - attrValBytes1, _ := json.Marshal("another string") - attrValBytes2, _ := json.Marshal(123) - binaryChecksums, _ := json.Marshal([]string{"binary-v1", "binary-v2"}) - upsertSearchAttr := &types.SearchAttributes{ - IndexedFields: map[string][]byte{ - definition.CustomStringField: attrValBytes1, - definition.CustomIntField: attrValBytes2, - definition.BinaryChecksums: binaryChecksums, - }, - } - return upsertSearchAttr -} - -func (s *ElasticSearchIntegrationSuite) TestUpsertWorkflowExecution_InvalidKey() { - id := "es-integration-upsert-workflow-failed-test" - wt := "es-integration-upsert-workflow-failed-test-type" - tl := "es-integration-upsert-workflow-failed-test-tasklist" - identity := "worker1" - - workflowType := &types.WorkflowType{} - workflowType.Name = wt - - taskList := &types.TaskList{} - taskList.Name = tl - - request := &types.StartWorkflowExecutionRequest{ - RequestID: uuid.New(), - Domain: s.domainName, - WorkflowID: id, - WorkflowType: workflowType, - TaskList: taskList, - Input: nil, - ExecutionStartToCloseTimeoutSeconds: common.Int32Ptr(100), - TaskStartToCloseTimeoutSeconds: common.Int32Ptr(1), - Identity: identity, - } - - we, err0 := s.engine.StartWorkflowExecution(createContext(), request) - s.Nil(err0) - - s.Logger.Info("StartWorkflowExecution", tag.WorkflowRunID(we.RunID)) - - dtHandler := func(execution *types.WorkflowExecution, wt *types.WorkflowType, - previousStartedEventID, startedEventID int64, history *types.History) ([]byte, []*types.Decision, error) { - - upsertDecision := &types.Decision{ - DecisionType: types.DecisionTypeUpsertWorkflowSearchAttributes.Ptr(), - UpsertWorkflowSearchAttributesDecisionAttributes: &types.UpsertWorkflowSearchAttributesDecisionAttributes{ - SearchAttributes: &types.SearchAttributes{ - IndexedFields: map[string][]byte{ - "INVALIDKEY": []byte(`1`), - }, - }, - }} - return nil, []*types.Decision{upsertDecision}, nil - } - - poller := &TaskPoller{ - Engine: s.engine, - Domain: s.domainName, - TaskList: taskList, - StickyTaskList: taskList, - Identity: identity, - DecisionHandler: dtHandler, - Logger: s.Logger, - T: s.T(), - } - - _, err := poller.PollAndProcessDecisionTask(false, false) - s.Nil(err) - - historyResponse, err := s.engine.GetWorkflowExecutionHistory(createContext(), &types.GetWorkflowExecutionHistoryRequest{ - Domain: s.domainName, - Execution: &types.WorkflowExecution{ - WorkflowID: id, - RunID: we.RunID, - }, - }) - s.Nil(err) - history := historyResponse.History - decisionFailedEvent := history.GetEvents()[3] - s.Equal(types.EventTypeDecisionTaskFailed, decisionFailedEvent.GetEventType()) - failedDecisionAttr := decisionFailedEvent.DecisionTaskFailedEventAttributes - s.Equal(types.DecisionTaskFailedCauseBadSearchAttributes, failedDecisionAttr.GetCause()) - s.True(len(failedDecisionAttr.GetDetails()) > 0) -} - -func (s *ElasticSearchIntegrationSuite) putIndexSettings(indexName string, maxResultWindowSize int) { - err := s.esClient.PutMaxResultWindow(indexName, maxResultWindowSize) - s.Require().NoError(err) - s.verifyMaxResultWindowSize(indexName, maxResultWindowSize) -} - -func (s *ElasticSearchIntegrationSuite) verifyMaxResultWindowSize(indexName string, targetSize int) { - for i := 0; i < numOfRetry; i++ { - currentWindow, err := s.esClient.GetMaxResultWindow(indexName) - s.Require().NoError(err) - if currentWindow == strconv.Itoa(targetSize) { - return - } - time.Sleep(waitTimeInMs * time.Millisecond) - } - s.FailNow(fmt.Sprintf("ES max result window size hasn't reach target size within %v.", (numOfRetry*waitTimeInMs)*time.Millisecond)) -} diff --git a/host/integrationTest.go b/host/integrationTest.go new file mode 100644 index 00000000000..03672448e2c --- /dev/null +++ b/host/integrationTest.go @@ -0,0 +1,4078 @@ +// Copyright (c) 2016 Uber Technologies, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package host + +import ( + "bytes" + "encoding/binary" + "encoding/json" + "errors" + "fmt" + "math" + "sort" + "strconv" + "testing" + "time" + + "github.com/pborman/uuid" + log "github.com/sirupsen/logrus" + "github.com/stretchr/testify/require" + + "github.com/uber/cadence/common" + "github.com/uber/cadence/common/log/tag" + "github.com/uber/cadence/common/types" + cadencehistory "github.com/uber/cadence/service/history" + "github.com/uber/cadence/service/history/execution" + "github.com/uber/cadence/service/matching" +) + +type ( + IntegrationSuite struct { + // override suite.Suite.Assertions with require.Assertions; this means that s.NotNil(nil) will stop the test, + // not merely log an error + *require.Assertions + IntegrationBase + } +) + +func (s *IntegrationSuite) SetupSuite() { + s.setupSuite() +} + +func (s *IntegrationSuite) TearDownSuite() { + s.tearDownSuite() +} + +func (s *IntegrationSuite) SetupTest() { + // Have to define our overridden assertions in the test setup. If we did it earlier, s.T() will return nil + s.Assertions = require.New(s.T()) +} + +func (s *IntegrationSuite) TestStartWorkflowExecution() { + id := "integration-start-workflow-test" + wt := "integration-start-workflow-test-type" + tl := "integration-start-workflow-test-tasklist" + identity := "worker1" + + workflowType := &types.WorkflowType{} + workflowType.Name = wt + + taskList := &types.TaskList{} + taskList.Name = tl + + request := &types.StartWorkflowExecutionRequest{ + RequestID: uuid.New(), + Domain: s.domainName, + WorkflowID: id, + WorkflowType: workflowType, + TaskList: taskList, + Input: nil, + ExecutionStartToCloseTimeoutSeconds: common.Int32Ptr(100), + TaskStartToCloseTimeoutSeconds: common.Int32Ptr(1), + Identity: identity, + } + + we0, err0 := s.engine.StartWorkflowExecution(createContext(), request) + s.Nil(err0) + + we1, err1 := s.engine.StartWorkflowExecution(createContext(), request) + s.Nil(err1) + s.Equal(we0.RunID, we1.RunID) + + newRequest := &types.StartWorkflowExecutionRequest{ + RequestID: uuid.New(), + Domain: s.domainName, + WorkflowID: id, + WorkflowType: workflowType, + TaskList: taskList, + Input: nil, + ExecutionStartToCloseTimeoutSeconds: common.Int32Ptr(100), + TaskStartToCloseTimeoutSeconds: common.Int32Ptr(1), + Identity: identity, + } + we2, err2 := s.engine.StartWorkflowExecution(createContext(), newRequest) + s.NotNil(err2) + s.IsType(&types.WorkflowExecutionAlreadyStartedError{}, err2) + log.Infof("Unable to start workflow execution: %v", err2.Error()) + s.Nil(we2) +} + +func (s *IntegrationSuite) TestStartWorkflowExecution_IDReusePolicy() { + id := "integration-start-workflow-id-reuse-test" + wt := "integration-start-workflow-id-reuse-type" + tl := "integration-start-workflow-id-reuse-tasklist" + identity := "worker1" + + workflowType := &types.WorkflowType{} + workflowType.Name = wt + + taskList := &types.TaskList{} + taskList.Name = tl + + createStartRequest := func(policy types.WorkflowIDReusePolicy) *types.StartWorkflowExecutionRequest { + return &types.StartWorkflowExecutionRequest{ + RequestID: uuid.New(), + Domain: s.domainName, + WorkflowID: id, + WorkflowType: workflowType, + TaskList: taskList, + Input: nil, + ExecutionStartToCloseTimeoutSeconds: common.Int32Ptr(100), + TaskStartToCloseTimeoutSeconds: common.Int32Ptr(1), + Identity: identity, + WorkflowIDReusePolicy: &policy, + } + } + + request := createStartRequest(types.WorkflowIDReusePolicyAllowDuplicateFailedOnly) + we, err := s.engine.StartWorkflowExecution(createContext(), request) + s.Nil(err) + + // Test policies when workflow is running + policies := []types.WorkflowIDReusePolicy{ + types.WorkflowIDReusePolicyAllowDuplicateFailedOnly, + types.WorkflowIDReusePolicyAllowDuplicate, + types.WorkflowIDReusePolicyRejectDuplicate, + } + for _, policy := range policies { + newRequest := createStartRequest(policy) + we1, err1 := s.engine.StartWorkflowExecution(createContext(), newRequest) + s.Error(err1) + s.IsType(&types.WorkflowExecutionAlreadyStartedError{}, err1) + s.Nil(we1) + } + + // Test TerminateIfRunning policy when workflow is running + policy := types.WorkflowIDReusePolicyTerminateIfRunning + newRequest := createStartRequest(policy) + we1, err1 := s.engine.StartWorkflowExecution(createContext(), newRequest) + s.NoError(err1) + s.NotEqual(we.GetRunID(), we1.GetRunID()) + // verify terminate status + executionTerminated := false +GetHistoryLoop: + for i := 0; i < 10; i++ { + historyResponse, err := s.engine.GetWorkflowExecutionHistory(createContext(), &types.GetWorkflowExecutionHistoryRequest{ + Domain: s.domainName, + Execution: &types.WorkflowExecution{ + WorkflowID: id, + RunID: we.RunID, + }, + }) + s.Nil(err) + history := historyResponse.History + + lastEvent := history.Events[len(history.Events)-1] + if lastEvent.GetEventType() != types.EventTypeWorkflowExecutionTerminated { + s.Logger.Warn("Execution not terminated yet.") + time.Sleep(100 * time.Millisecond) + continue GetHistoryLoop + } + + terminateEventAttributes := lastEvent.WorkflowExecutionTerminatedEventAttributes + s.Equal(cadencehistory.TerminateIfRunningReason, terminateEventAttributes.GetReason()) + s.Equal(fmt.Sprintf(cadencehistory.TerminateIfRunningDetailsTemplate, we1.GetRunID()), string(terminateEventAttributes.Details)) + s.Equal(execution.IdentityHistoryService, terminateEventAttributes.GetIdentity()) + executionTerminated = true + break GetHistoryLoop + } + s.True(executionTerminated) + + // Terminate current workflow execution + err = s.engine.TerminateWorkflowExecution(createContext(), &types.TerminateWorkflowExecutionRequest{ + Domain: s.domainName, + WorkflowExecution: &types.WorkflowExecution{ + WorkflowID: id, + RunID: we1.RunID, + }, + Reason: "kill workflow", + Identity: identity, + }) + s.Nil(err) + + // test policy AllowDuplicateFailedOnly + policy = types.WorkflowIDReusePolicyAllowDuplicateFailedOnly + newRequest = createStartRequest(policy) + we2, err2 := s.engine.StartWorkflowExecution(createContext(), newRequest) + s.NoError(err2) + s.NotEqual(we1.GetRunID(), we2.GetRunID()) + // complete workflow instead of terminate + dtHandler := func(execution *types.WorkflowExecution, wt *types.WorkflowType, + previousStartedEventID, startedEventID int64, history *types.History) ([]byte, []*types.Decision, error) { + return []byte(strconv.Itoa(0)), []*types.Decision{{ + DecisionType: types.DecisionTypeCompleteWorkflowExecution.Ptr(), + CompleteWorkflowExecutionDecisionAttributes: &types.CompleteWorkflowExecutionDecisionAttributes{ + Result: []byte("Done."), + }, + }}, nil + } + poller := &TaskPoller{ + Engine: s.engine, + Domain: s.domainName, + TaskList: taskList, + Identity: identity, + DecisionHandler: dtHandler, + Logger: s.Logger, + T: s.T(), + } + _, err = poller.PollAndProcessDecisionTask(false, false) + s.Logger.Info("PollAndProcessDecisionTask", tag.Error(err)) + s.Nil(err) + // duplicate requests + we3, err3 := s.engine.StartWorkflowExecution(createContext(), newRequest) + s.NoError(err3) + s.Equal(we2.GetRunID(), we3.GetRunID()) + // new request, same policy + newRequest = createStartRequest(policy) + we3, err3 = s.engine.StartWorkflowExecution(createContext(), newRequest) + s.Error(err3) + s.IsType(&types.WorkflowExecutionAlreadyStartedError{}, err3) + s.Nil(we3) + + // test policy RejectDuplicate + policy = types.WorkflowIDReusePolicyRejectDuplicate + newRequest = createStartRequest(policy) + we3, err3 = s.engine.StartWorkflowExecution(createContext(), newRequest) + s.Error(err3) + s.IsType(&types.WorkflowExecutionAlreadyStartedError{}, err3) + s.Nil(we3) + + // test policy AllowDuplicate + policy = types.WorkflowIDReusePolicyAllowDuplicate + newRequest = createStartRequest(policy) + we4, err4 := s.engine.StartWorkflowExecution(createContext(), newRequest) + s.NoError(err4) + s.NotEqual(we3.GetRunID(), we4.GetRunID()) + + // complete workflow + _, err = poller.PollAndProcessDecisionTask(false, false) + s.Logger.Info("PollAndProcessDecisionTask", tag.Error(err)) + s.Nil(err) + + // test policy TerminateIfRunning + policy = types.WorkflowIDReusePolicyTerminateIfRunning + newRequest = createStartRequest(policy) + we5, err5 := s.engine.StartWorkflowExecution(createContext(), newRequest) + s.NoError(err5) + s.NotEqual(we4.GetRunID(), we5.GetRunID()) +} + +func (s *IntegrationSuite) TestTerminateWorkflow() { + id := "integration-terminate-workflow-test" + wt := "integration-terminate-workflow-test-type" + tl := "integration-terminate-workflow-test-tasklist" + identity := "worker1" + activityName := "activity_type1" + + workflowType := &types.WorkflowType{} + workflowType.Name = wt + + taskList := &types.TaskList{} + taskList.Name = tl + + request := &types.StartWorkflowExecutionRequest{ + RequestID: uuid.New(), + Domain: s.domainName, + WorkflowID: id, + WorkflowType: workflowType, + TaskList: taskList, + Input: nil, + ExecutionStartToCloseTimeoutSeconds: common.Int32Ptr(100), + TaskStartToCloseTimeoutSeconds: common.Int32Ptr(1), + Identity: identity, + } + + we, err0 := s.engine.StartWorkflowExecution(createContext(), request) + s.Nil(err0) + + s.Logger.Info("StartWorkflowExecution", tag.WorkflowRunID(we.RunID)) + + activityCount := int32(1) + activityCounter := int32(0) + dtHandler := func(execution *types.WorkflowExecution, wt *types.WorkflowType, + previousStartedEventID, startedEventID int64, history *types.History) ([]byte, []*types.Decision, error) { + if activityCounter < activityCount { + activityCounter++ + buf := new(bytes.Buffer) + s.Nil(binary.Write(buf, binary.LittleEndian, activityCounter)) + + return []byte(strconv.Itoa(int(activityCounter))), []*types.Decision{{ + DecisionType: types.DecisionTypeScheduleActivityTask.Ptr(), + ScheduleActivityTaskDecisionAttributes: &types.ScheduleActivityTaskDecisionAttributes{ + ActivityID: strconv.Itoa(int(activityCounter)), + ActivityType: &types.ActivityType{Name: activityName}, + TaskList: &types.TaskList{Name: tl}, + Input: buf.Bytes(), + ScheduleToCloseTimeoutSeconds: common.Int32Ptr(100), + ScheduleToStartTimeoutSeconds: common.Int32Ptr(10), + StartToCloseTimeoutSeconds: common.Int32Ptr(50), + HeartbeatTimeoutSeconds: common.Int32Ptr(5), + }, + }}, nil + } + + return []byte(strconv.Itoa(int(activityCounter))), []*types.Decision{{ + DecisionType: types.DecisionTypeCompleteWorkflowExecution.Ptr(), + CompleteWorkflowExecutionDecisionAttributes: &types.CompleteWorkflowExecutionDecisionAttributes{ + Result: []byte("Done."), + }, + }}, nil + } + + atHandler := func(execution *types.WorkflowExecution, activityType *types.ActivityType, + ActivityID string, input []byte, taskToken []byte) ([]byte, bool, error) { + + return []byte("Activity Result."), false, nil + } + + poller := &TaskPoller{ + Engine: s.engine, + Domain: s.domainName, + TaskList: taskList, + Identity: identity, + DecisionHandler: dtHandler, + ActivityHandler: atHandler, + Logger: s.Logger, + T: s.T(), + } + + _, err := poller.PollAndProcessDecisionTask(false, false) + s.Logger.Info("PollAndProcessDecisionTask", tag.Error(err)) + s.Nil(err) + + terminateReason := "terminate reason." + terminateDetails := []byte("terminate details.") + err = s.engine.TerminateWorkflowExecution(createContext(), &types.TerminateWorkflowExecutionRequest{ + Domain: s.domainName, + WorkflowExecution: &types.WorkflowExecution{ + WorkflowID: id, + RunID: we.RunID, + }, + Reason: terminateReason, + Details: terminateDetails, + Identity: identity, + }) + s.Nil(err) + + executionTerminated := false +GetHistoryLoop: + for i := 0; i < 10; i++ { + historyResponse, err := s.engine.GetWorkflowExecutionHistory(createContext(), &types.GetWorkflowExecutionHistoryRequest{ + Domain: s.domainName, + Execution: &types.WorkflowExecution{ + WorkflowID: id, + RunID: we.RunID, + }, + }) + s.Nil(err) + history := historyResponse.History + + lastEvent := history.Events[len(history.Events)-1] + if *lastEvent.EventType != types.EventTypeWorkflowExecutionTerminated { + s.Logger.Warn("Execution not terminated yet.") + time.Sleep(100 * time.Millisecond) + continue GetHistoryLoop + } + + terminateEventAttributes := lastEvent.WorkflowExecutionTerminatedEventAttributes + s.Equal(terminateReason, terminateEventAttributes.Reason) + s.Equal(terminateDetails, terminateEventAttributes.Details) + s.Equal(identity, terminateEventAttributes.Identity) + executionTerminated = true + break GetHistoryLoop + } + + s.True(executionTerminated) + + newExecutionStarted := false +StartNewExecutionLoop: + for i := 0; i < 10; i++ { + request := &types.StartWorkflowExecutionRequest{ + RequestID: uuid.New(), + Domain: s.domainName, + WorkflowID: id, + WorkflowType: workflowType, + TaskList: taskList, + Input: nil, + ExecutionStartToCloseTimeoutSeconds: common.Int32Ptr(100), + TaskStartToCloseTimeoutSeconds: common.Int32Ptr(1), + Identity: identity, + } + + newExecution, err := s.engine.StartWorkflowExecution(createContext(), request) + if err != nil { + s.Logger.Warn("Start New Execution failed. Error", tag.Error(err)) + time.Sleep(100 * time.Millisecond) + continue StartNewExecutionLoop + } + + s.Logger.Info("New Execution Started with the same ID", tag.WorkflowID(id), + tag.WorkflowRunID(newExecution.RunID)) + newExecutionStarted = true + break StartNewExecutionLoop + } + + s.True(newExecutionStarted) +} + +func (s *IntegrationSuite) TestSequentialWorkflow() { + id := "integration-sequential-workflow-test" + wt := "integration-sequential-workflow-test-type" + tl := "integration-sequential-workflow-test-tasklist" + identity := "worker1" + activityName := "activity_type1" + + workflowType := &types.WorkflowType{} + workflowType.Name = wt + + taskList := &types.TaskList{} + taskList.Name = tl + + request := &types.StartWorkflowExecutionRequest{ + RequestID: uuid.New(), + Domain: s.domainName, + WorkflowID: id, + WorkflowType: workflowType, + TaskList: taskList, + Input: nil, + ExecutionStartToCloseTimeoutSeconds: common.Int32Ptr(100), + TaskStartToCloseTimeoutSeconds: common.Int32Ptr(1), + Identity: identity, + } + + we, err0 := s.engine.StartWorkflowExecution(createContext(), request) + s.Nil(err0) + + s.Logger.Info("StartWorkflowExecution", tag.WorkflowRunID(we.RunID)) + + workflowComplete := false + activityCount := int32(10) + activityCounter := int32(0) + dtHandler := func(execution *types.WorkflowExecution, wt *types.WorkflowType, + previousStartedEventID, startedEventID int64, history *types.History) ([]byte, []*types.Decision, error) { + if activityCounter < activityCount { + activityCounter++ + buf := new(bytes.Buffer) + s.Nil(binary.Write(buf, binary.LittleEndian, activityCounter)) + + return []byte(strconv.Itoa(int(activityCounter))), []*types.Decision{{ + DecisionType: types.DecisionTypeScheduleActivityTask.Ptr(), + ScheduleActivityTaskDecisionAttributes: &types.ScheduleActivityTaskDecisionAttributes{ + ActivityID: strconv.Itoa(int(activityCounter)), + ActivityType: &types.ActivityType{Name: activityName}, + TaskList: &types.TaskList{Name: tl}, + Input: buf.Bytes(), + ScheduleToCloseTimeoutSeconds: common.Int32Ptr(100), + ScheduleToStartTimeoutSeconds: common.Int32Ptr(10), + StartToCloseTimeoutSeconds: common.Int32Ptr(50), + HeartbeatTimeoutSeconds: common.Int32Ptr(5), + }, + }}, nil + } + + workflowComplete = true + return []byte(strconv.Itoa(int(activityCounter))), []*types.Decision{{ + DecisionType: types.DecisionTypeCompleteWorkflowExecution.Ptr(), + CompleteWorkflowExecutionDecisionAttributes: &types.CompleteWorkflowExecutionDecisionAttributes{ + Result: []byte("Done."), + }, + }}, nil + } + + expectedActivity := int32(1) + atHandler := func(execution *types.WorkflowExecution, activityType *types.ActivityType, + ActivityID string, input []byte, taskToken []byte) ([]byte, bool, error) { + s.Equal(id, execution.WorkflowID) + s.Equal(activityName, activityType.Name) + id, _ := strconv.Atoi(ActivityID) + s.Equal(int(expectedActivity), id) + buf := bytes.NewReader(input) + var in int32 + binary.Read(buf, binary.LittleEndian, &in) + s.Equal(expectedActivity, in) + expectedActivity++ + + return []byte("Activity Result."), false, nil + } + + poller := &TaskPoller{ + Engine: s.engine, + Domain: s.domainName, + TaskList: taskList, + Identity: identity, + DecisionHandler: dtHandler, + ActivityHandler: atHandler, + Logger: s.Logger, + T: s.T(), + } + + for i := 0; i < 10; i++ { + _, err := poller.PollAndProcessDecisionTask(false, false) + s.Logger.Info("PollAndProcessDecisionTask", tag.Error(err)) + s.Nil(err) + if i%2 == 0 { + err = poller.PollAndProcessActivityTask(false) + } else { // just for testing respondActivityTaskCompleteByID + err = poller.PollAndProcessActivityTaskWithID(false) + } + s.Logger.Info("PollAndProcessActivityTask", tag.Error(err)) + s.Nil(err) + } + + s.False(workflowComplete) + _, err := poller.PollAndProcessDecisionTask(false, false) + s.Nil(err) + s.True(workflowComplete) +} + +func (s *IntegrationSuite) TestCompleteDecisionTaskAndCreateNewOne() { + id := "integration-complete-decision-create-new-test" + wt := "integration-complete-decision-create-new-test-type" + tl := "integration-complete-decision-create-new-test-tasklist" + identity := "worker1" + + workflowType := &types.WorkflowType{} + workflowType.Name = wt + + taskList := &types.TaskList{} + taskList.Name = tl + + request := &types.StartWorkflowExecutionRequest{ + RequestID: uuid.New(), + Domain: s.domainName, + WorkflowID: id, + WorkflowType: workflowType, + TaskList: taskList, + Input: nil, + ExecutionStartToCloseTimeoutSeconds: common.Int32Ptr(100), + TaskStartToCloseTimeoutSeconds: common.Int32Ptr(1), + Identity: identity, + } + + we, err0 := s.engine.StartWorkflowExecution(createContext(), request) + s.Nil(err0) + + s.Logger.Info("StartWorkflowExecution", tag.WorkflowRunID(we.RunID)) + + decisionCount := 0 + dtHandler := func(execution *types.WorkflowExecution, wt *types.WorkflowType, + previousStartedEventID, startedEventID int64, history *types.History) ([]byte, []*types.Decision, error) { + + if decisionCount < 2 { + decisionCount++ + return nil, []*types.Decision{{ + DecisionType: types.DecisionTypeRecordMarker.Ptr(), + RecordMarkerDecisionAttributes: &types.RecordMarkerDecisionAttributes{ + MarkerName: "test-marker", + }, + }}, nil + } + + return nil, []*types.Decision{{ + DecisionType: types.DecisionTypeCompleteWorkflowExecution.Ptr(), + CompleteWorkflowExecutionDecisionAttributes: &types.CompleteWorkflowExecutionDecisionAttributes{ + Result: []byte("Done."), + }, + }}, nil + } + + poller := &TaskPoller{ + Engine: s.engine, + Domain: s.domainName, + TaskList: taskList, + StickyTaskList: taskList, + Identity: identity, + DecisionHandler: dtHandler, + Logger: s.Logger, + T: s.T(), + } + + _, newTask, err := poller.PollAndProcessDecisionTaskWithAttemptAndRetryAndForceNewDecision( + false, + false, + true, + true, + int64(0), + 1, + true, + nil) + s.Nil(err) + s.NotNil(newTask) + s.NotNil(newTask.DecisionTask) + + s.Equal(int64(3), newTask.DecisionTask.GetPreviousStartedEventID()) + s.Equal(int64(7), newTask.DecisionTask.GetStartedEventID()) + s.Equal(4, len(newTask.DecisionTask.History.Events)) + s.Equal(types.EventTypeDecisionTaskCompleted, newTask.DecisionTask.History.Events[0].GetEventType()) + s.Equal(types.EventTypeMarkerRecorded, newTask.DecisionTask.History.Events[1].GetEventType()) + s.Equal(types.EventTypeDecisionTaskScheduled, newTask.DecisionTask.History.Events[2].GetEventType()) + s.Equal(types.EventTypeDecisionTaskStarted, newTask.DecisionTask.History.Events[3].GetEventType()) +} + +func (s *IntegrationSuite) TestDecisionAndActivityTimeoutsWorkflow() { + id := "integration-timeouts-workflow-test" + wt := "integration-timeouts-workflow-test-type" + tl := "integration-timeouts-workflow-test-tasklist" + identity := "worker1" + activityName := "activity_timer" + + workflowType := &types.WorkflowType{} + workflowType.Name = wt + + taskList := &types.TaskList{} + taskList.Name = tl + + request := &types.StartWorkflowExecutionRequest{ + RequestID: uuid.New(), + Domain: s.domainName, + WorkflowID: id, + WorkflowType: workflowType, + TaskList: taskList, + Input: nil, + ExecutionStartToCloseTimeoutSeconds: common.Int32Ptr(100), + TaskStartToCloseTimeoutSeconds: common.Int32Ptr(1), + Identity: identity, + } + + we, err0 := s.engine.StartWorkflowExecution(createContext(), request) + s.Nil(err0) + + s.Logger.Info("StartWorkflowExecution", tag.WorkflowRunID(we.RunID)) + + workflowComplete := false + activityCount := int32(4) + activityCounter := int32(0) + + dtHandler := func(execution *types.WorkflowExecution, wt *types.WorkflowType, + previousStartedEventID, startedEventID int64, history *types.History) ([]byte, []*types.Decision, error) { + if activityCounter < activityCount { + activityCounter++ + buf := new(bytes.Buffer) + s.Nil(binary.Write(buf, binary.LittleEndian, activityCounter)) + + return []byte(strconv.Itoa(int(activityCounter))), []*types.Decision{{ + DecisionType: types.DecisionTypeScheduleActivityTask.Ptr(), + ScheduleActivityTaskDecisionAttributes: &types.ScheduleActivityTaskDecisionAttributes{ + ActivityID: strconv.Itoa(int(activityCounter)), + ActivityType: &types.ActivityType{Name: activityName}, + TaskList: &types.TaskList{Name: tl}, + Input: buf.Bytes(), + ScheduleToCloseTimeoutSeconds: common.Int32Ptr(1), + ScheduleToStartTimeoutSeconds: common.Int32Ptr(1), + StartToCloseTimeoutSeconds: common.Int32Ptr(1), + HeartbeatTimeoutSeconds: common.Int32Ptr(1), + }, + }}, nil + } + + s.Logger.Info("Completing types.") + + workflowComplete = true + return []byte(strconv.Itoa(int(activityCounter))), []*types.Decision{{ + DecisionType: types.DecisionTypeCompleteWorkflowExecution.Ptr(), + CompleteWorkflowExecutionDecisionAttributes: &types.CompleteWorkflowExecutionDecisionAttributes{ + Result: []byte("Done."), + }, + }}, nil + } + + atHandler := func(execution *types.WorkflowExecution, activityType *types.ActivityType, + ActivityID string, input []byte, taskToken []byte) ([]byte, bool, error) { + s.Equal(id, execution.WorkflowID) + s.Equal(activityName, activityType.Name) + s.Logger.Info("Activity ID", tag.WorkflowActivityID(ActivityID)) + return []byte("Activity Result."), false, nil + } + + poller := &TaskPoller{ + Engine: s.engine, + Domain: s.domainName, + TaskList: taskList, + Identity: identity, + DecisionHandler: dtHandler, + ActivityHandler: atHandler, + Logger: s.Logger, + T: s.T(), + } + + for i := 0; i < 8; i++ { + dropDecisionTask := (i%2 == 0) + s.Logger.Info("Calling Decision Task", tag.Counter(i)) + var err error + if dropDecisionTask { + _, err = poller.PollAndProcessDecisionTask(false, true) + } else { + _, err = poller.PollAndProcessDecisionTaskWithAttempt(true, false, false, false, int64(1)) + } + if err != nil { + historyResponse, err := s.engine.GetWorkflowExecutionHistory(createContext(), &types.GetWorkflowExecutionHistoryRequest{ + Domain: s.domainName, + Execution: &types.WorkflowExecution{ + WorkflowID: id, + RunID: we.RunID, + }, + }) + s.Nil(err) + history := historyResponse.History + common.PrettyPrintHistory(history, s.Logger) + } + s.True(err == nil || err == matching.ErrNoTasks, "%v", err) + if !dropDecisionTask { + s.Logger.Info("Calling Activity Task: %d", tag.Counter(i)) + err = poller.PollAndProcessActivityTask(i%4 == 0) + s.True(err == nil || err == matching.ErrNoTasks) + } + } + + s.Logger.Info("Waiting for workflow to complete", tag.WorkflowRunID(we.RunID)) + + s.False(workflowComplete) + _, err := poller.PollAndProcessDecisionTask(false, false) + s.Nil(err) + s.True(workflowComplete) +} + +func (s *IntegrationSuite) TestWorkflowRetry() { + id := "integration-wf-retry-test" + wt := "integration-wf-retry-type" + tl := "integration-wf-retry-tasklist" + identity := "worker1" + + workflowType := &types.WorkflowType{} + workflowType.Name = wt + + taskList := &types.TaskList{} + taskList.Name = tl + + initialIntervalInSeconds := 1 + backoffCoefficient := 1.5 + maximumAttempts := 5 + request := &types.StartWorkflowExecutionRequest{ + RequestID: uuid.New(), + Domain: s.domainName, + WorkflowID: id, + WorkflowType: workflowType, + TaskList: taskList, + Input: nil, + ExecutionStartToCloseTimeoutSeconds: common.Int32Ptr(100), + TaskStartToCloseTimeoutSeconds: common.Int32Ptr(1), + Identity: identity, + RetryPolicy: &types.RetryPolicy{ + InitialIntervalInSeconds: int32(initialIntervalInSeconds), + MaximumAttempts: int32(maximumAttempts), + MaximumIntervalInSeconds: 1, + NonRetriableErrorReasons: []string{"bad-bug"}, + BackoffCoefficient: backoffCoefficient, + ExpirationIntervalInSeconds: 100, + }, + } + + we, err0 := s.engine.StartWorkflowExecution(createContext(), request) + s.Nil(err0) + + s.Logger.Info("StartWorkflowExecution", tag.WorkflowRunID(we.RunID)) + + var executions []*types.WorkflowExecution + + attemptCount := 0 + + dtHandler := func(execution *types.WorkflowExecution, wt *types.WorkflowType, + previousStartedEventID, startedEventID int64, history *types.History) ([]byte, []*types.Decision, error) { + executions = append(executions, execution) + attemptCount++ + if attemptCount == maximumAttempts { + return nil, []*types.Decision{ + { + DecisionType: types.DecisionTypeCompleteWorkflowExecution.Ptr(), + CompleteWorkflowExecutionDecisionAttributes: &types.CompleteWorkflowExecutionDecisionAttributes{ + Result: []byte("succeed-after-retry"), + }, + }}, nil + } + return nil, []*types.Decision{ + { + DecisionType: types.DecisionTypeFailWorkflowExecution.Ptr(), + FailWorkflowExecutionDecisionAttributes: &types.FailWorkflowExecutionDecisionAttributes{ + Reason: common.StringPtr("retryable-error"), + Details: nil, + }, + }}, nil + } + + poller := &TaskPoller{ + Engine: s.engine, + Domain: s.domainName, + TaskList: taskList, + Identity: identity, + DecisionHandler: dtHandler, + Logger: s.Logger, + T: s.T(), + } + + describeWorkflowExecution := func(execution *types.WorkflowExecution) (*types.DescribeWorkflowExecutionResponse, error) { + return s.engine.DescribeWorkflowExecution(createContext(), &types.DescribeWorkflowExecutionRequest{ + Domain: s.domainName, + Execution: execution, + }) + } + + for i := 0; i != maximumAttempts; i++ { + _, err := poller.PollAndProcessDecisionTask(false, false) + s.True(err == nil, err) + events := s.getHistory(s.domainName, executions[i]) + if i == maximumAttempts-1 { + s.Equal(types.EventTypeWorkflowExecutionCompleted, events[len(events)-1].GetEventType()) + } else { + s.Equal(types.EventTypeWorkflowExecutionContinuedAsNew, events[len(events)-1].GetEventType()) + } + s.Equal(int32(i), events[0].GetWorkflowExecutionStartedEventAttributes().GetAttempt()) + + dweResponse, err := describeWorkflowExecution(executions[i]) + s.Nil(err) + backoff := time.Duration(0) + if i > 0 { + backoff = time.Duration(float64(initialIntervalInSeconds)*math.Pow(backoffCoefficient, float64(i-1))) * time.Second + // retry backoff cannot larger than MaximumIntervalInSeconds + if backoff > time.Second { + backoff = time.Second + } + } + expectedExecutionTime := dweResponse.WorkflowExecutionInfo.GetStartTime() + backoff.Nanoseconds() + s.Equal(expectedExecutionTime, dweResponse.WorkflowExecutionInfo.GetExecutionTime()) + } +} + +func (s *IntegrationSuite) TestWorkflowRetryFailures() { + id := "integration-wf-retry-failures-test" + wt := "integration-wf-retry-failures-type" + tl := "integration-wf-retry-failures-tasklist" + identity := "worker1" + + workflowType := &types.WorkflowType{} + workflowType.Name = wt + + taskList := &types.TaskList{} + taskList.Name = tl + + workflowImpl := func(attempts int, errorReason string, executions *[]*types.WorkflowExecution) decisionTaskHandler { + attemptCount := 0 + + dtHandler := func(execution *types.WorkflowExecution, wt *types.WorkflowType, + previousStartedEventID, startedEventID int64, history *types.History) ([]byte, []*types.Decision, error) { + *executions = append(*executions, execution) + attemptCount++ + if attemptCount == attempts { + return nil, []*types.Decision{ + { + DecisionType: types.DecisionTypeCompleteWorkflowExecution.Ptr(), + CompleteWorkflowExecutionDecisionAttributes: &types.CompleteWorkflowExecutionDecisionAttributes{ + Result: []byte("succeed-after-retry"), + }, + }}, nil + } + return nil, []*types.Decision{ + { + DecisionType: types.DecisionTypeFailWorkflowExecution.Ptr(), + FailWorkflowExecutionDecisionAttributes: &types.FailWorkflowExecutionDecisionAttributes{ + //Reason: common.StringPtr("retryable-error"), + Reason: common.StringPtr(errorReason), + Details: nil, + }, + }}, nil + } + + return dtHandler + } + + // Fail using attempt + request := &types.StartWorkflowExecutionRequest{ + RequestID: uuid.New(), + Domain: s.domainName, + WorkflowID: id, + WorkflowType: workflowType, + TaskList: taskList, + Input: nil, + ExecutionStartToCloseTimeoutSeconds: common.Int32Ptr(100), + TaskStartToCloseTimeoutSeconds: common.Int32Ptr(1), + Identity: identity, + RetryPolicy: &types.RetryPolicy{ + InitialIntervalInSeconds: 1, + MaximumAttempts: 3, + MaximumIntervalInSeconds: 1, + NonRetriableErrorReasons: []string{"bad-bug"}, + BackoffCoefficient: 1, + ExpirationIntervalInSeconds: 100, + }, + } + + we, err0 := s.engine.StartWorkflowExecution(createContext(), request) + s.Nil(err0) + + s.Logger.Info("StartWorkflowExecution", tag.WorkflowRunID(we.RunID)) + + executions := []*types.WorkflowExecution{} + dtHandler := workflowImpl(5, "retryable-error", &executions) + poller := &TaskPoller{ + Engine: s.engine, + Domain: s.domainName, + TaskList: taskList, + Identity: identity, + DecisionHandler: dtHandler, + Logger: s.Logger, + T: s.T(), + } + + _, err := poller.PollAndProcessDecisionTask(false, false) + s.True(err == nil, err) + events := s.getHistory(s.domainName, executions[0]) + s.Equal(types.EventTypeWorkflowExecutionContinuedAsNew, events[len(events)-1].GetEventType()) + s.Equal(int32(0), events[0].GetWorkflowExecutionStartedEventAttributes().GetAttempt()) + + _, err = poller.PollAndProcessDecisionTask(false, false) + s.True(err == nil, err) + events = s.getHistory(s.domainName, executions[1]) + s.Equal(types.EventTypeWorkflowExecutionContinuedAsNew, events[len(events)-1].GetEventType()) + s.Equal(int32(1), events[0].GetWorkflowExecutionStartedEventAttributes().GetAttempt()) + + _, err = poller.PollAndProcessDecisionTask(false, false) + s.True(err == nil, err) + events = s.getHistory(s.domainName, executions[2]) + s.Equal(types.EventTypeWorkflowExecutionFailed, events[len(events)-1].GetEventType()) + s.Equal(int32(2), events[0].GetWorkflowExecutionStartedEventAttributes().GetAttempt()) + + // Fail error reason + request = &types.StartWorkflowExecutionRequest{ + RequestID: uuid.New(), + Domain: s.domainName, + WorkflowID: id, + WorkflowType: workflowType, + TaskList: taskList, + Input: nil, + ExecutionStartToCloseTimeoutSeconds: common.Int32Ptr(100), + TaskStartToCloseTimeoutSeconds: common.Int32Ptr(1), + Identity: identity, + RetryPolicy: &types.RetryPolicy{ + InitialIntervalInSeconds: 1, + MaximumAttempts: 3, + MaximumIntervalInSeconds: 1, + NonRetriableErrorReasons: []string{"bad-bug"}, + BackoffCoefficient: 1, + ExpirationIntervalInSeconds: 100, + }, + } + + we, err0 = s.engine.StartWorkflowExecution(createContext(), request) + s.Nil(err0) + + s.Logger.Info("StartWorkflowExecution", tag.WorkflowRunID(we.RunID)) + + executions = []*types.WorkflowExecution{} + dtHandler = workflowImpl(5, "bad-bug", &executions) + poller = &TaskPoller{ + Engine: s.engine, + Domain: s.domainName, + TaskList: taskList, + Identity: identity, + DecisionHandler: dtHandler, + Logger: s.Logger, + T: s.T(), + } + + _, err = poller.PollAndProcessDecisionTask(false, false) + s.True(err == nil, err) + events = s.getHistory(s.domainName, executions[0]) + s.Equal(types.EventTypeWorkflowExecutionFailed, events[len(events)-1].GetEventType()) + s.Equal(int32(0), events[0].GetWorkflowExecutionStartedEventAttributes().GetAttempt()) + +} + +func (s *IntegrationSuite) TestCronWorkflow() { + id := "integration-wf-cron-test" + wt := "integration-wf-cron-type" + tl := "integration-wf-cron-tasklist" + identity := "worker1" + cronSchedule := "@every 3s" + + targetBackoffDuration := time.Second * 3 + backoffDurationTolerance := time.Millisecond * 500 + + workflowType := &types.WorkflowType{} + workflowType.Name = wt + + taskList := &types.TaskList{} + taskList.Name = tl + + memo := &types.Memo{ + Fields: map[string][]byte{"memoKey": []byte("memoVal")}, + } + searchAttr := &types.SearchAttributes{ + IndexedFields: map[string][]byte{"CustomKeywordField": []byte(`"1"`)}, + } + + request := &types.StartWorkflowExecutionRequest{ + RequestID: uuid.New(), + Domain: s.domainName, + WorkflowID: id, + WorkflowType: workflowType, + TaskList: taskList, + Input: nil, + ExecutionStartToCloseTimeoutSeconds: common.Int32Ptr(100), + TaskStartToCloseTimeoutSeconds: common.Int32Ptr(1), + Identity: identity, + CronSchedule: cronSchedule, //minimum interval by standard spec is 1m (* * * * *), use non-standard descriptor for short interval for test + Memo: memo, + SearchAttributes: searchAttr, + } + + startWorkflowTS := time.Now() + we, err0 := s.engine.StartWorkflowExecution(createContext(), request) + s.Nil(err0) + + s.Logger.Info("StartWorkflowExecution", tag.WorkflowRunID(we.RunID)) + + var executions []*types.WorkflowExecution + + attemptCount := 0 + + dtHandler := func(execution *types.WorkflowExecution, wt *types.WorkflowType, + previousStartedEventID, startedEventID int64, history *types.History) ([]byte, []*types.Decision, error) { + executions = append(executions, execution) + attemptCount++ + if attemptCount == 2 { + return nil, []*types.Decision{ + { + DecisionType: types.DecisionTypeCompleteWorkflowExecution.Ptr(), + CompleteWorkflowExecutionDecisionAttributes: &types.CompleteWorkflowExecutionDecisionAttributes{ + Result: []byte("cron-test-result"), + }, + }}, nil + } + return nil, []*types.Decision{ + { + DecisionType: types.DecisionTypeFailWorkflowExecution.Ptr(), + FailWorkflowExecutionDecisionAttributes: &types.FailWorkflowExecutionDecisionAttributes{ + Reason: common.StringPtr("cron-test-error"), + Details: nil, + }, + }}, nil + } + + poller := &TaskPoller{ + Engine: s.engine, + Domain: s.domainName, + TaskList: taskList, + Identity: identity, + DecisionHandler: dtHandler, + Logger: s.Logger, + T: s.T(), + } + + startFilter := &types.StartTimeFilter{} + startFilter.EarliestTime = common.Int64Ptr(startWorkflowTS.UnixNano()) + startFilter.LatestTime = common.Int64Ptr(time.Now().UnixNano()) + + // Sleep some time before checking the open executions. + // This will not cost extra time as the polling for first decision task will be blocked for 3 seconds. + time.Sleep(2 * time.Second) + resp, err := s.engine.ListOpenWorkflowExecutions(createContext(), &types.ListOpenWorkflowExecutionsRequest{ + Domain: s.domainName, + MaximumPageSize: 100, + StartTimeFilter: startFilter, + ExecutionFilter: &types.WorkflowExecutionFilter{ + WorkflowID: id, + }, + }) + s.Nil(err) + s.Equal(1, len(resp.GetExecutions())) + executionInfo := resp.GetExecutions()[0] + s.Equal(targetBackoffDuration.Nanoseconds(), executionInfo.GetExecutionTime()-executionInfo.GetStartTime()) + + _, err = poller.PollAndProcessDecisionTask(false, false) + s.True(err == nil, err) + + // Make sure the cron workflow start running at a proper time, in this case 3 seconds after the + // startWorkflowExecution request + backoffDuration := time.Now().Sub(startWorkflowTS) + s.True(backoffDuration > targetBackoffDuration) + s.True(backoffDuration < targetBackoffDuration+backoffDurationTolerance) + + _, err = poller.PollAndProcessDecisionTask(false, false) + s.True(err == nil, err) + + _, err = poller.PollAndProcessDecisionTask(false, false) + s.True(err == nil, err) + + s.Equal(3, attemptCount) + + terminateErr := s.engine.TerminateWorkflowExecution(createContext(), &types.TerminateWorkflowExecutionRequest{ + Domain: s.domainName, + WorkflowExecution: &types.WorkflowExecution{ + WorkflowID: id, + }, + }) + s.NoError(terminateErr) + events := s.getHistory(s.domainName, executions[0]) + lastEvent := events[len(events)-1] + s.Equal(types.EventTypeWorkflowExecutionContinuedAsNew, lastEvent.GetEventType()) + attributes := lastEvent.WorkflowExecutionContinuedAsNewEventAttributes + s.Equal(types.ContinueAsNewInitiatorCronSchedule, attributes.GetInitiator()) + s.Equal("cron-test-error", attributes.GetFailureReason()) + s.Equal(0, len(attributes.GetLastCompletionResult())) + s.Equal(memo, attributes.Memo) + s.Equal(searchAttr, attributes.SearchAttributes) + + events = s.getHistory(s.domainName, executions[1]) + lastEvent = events[len(events)-1] + s.Equal(types.EventTypeWorkflowExecutionContinuedAsNew, lastEvent.GetEventType()) + attributes = lastEvent.WorkflowExecutionContinuedAsNewEventAttributes + s.Equal(types.ContinueAsNewInitiatorCronSchedule, attributes.GetInitiator()) + s.Equal("", attributes.GetFailureReason()) + s.Equal("cron-test-result", string(attributes.GetLastCompletionResult())) + s.Equal(memo, attributes.Memo) + s.Equal(searchAttr, attributes.SearchAttributes) + + events = s.getHistory(s.domainName, executions[2]) + lastEvent = events[len(events)-1] + s.Equal(types.EventTypeWorkflowExecutionContinuedAsNew, lastEvent.GetEventType()) + attributes = lastEvent.WorkflowExecutionContinuedAsNewEventAttributes + s.Equal(types.ContinueAsNewInitiatorCronSchedule, attributes.GetInitiator()) + s.Equal("cron-test-error", attributes.GetFailureReason()) + s.Equal("cron-test-result", string(attributes.GetLastCompletionResult())) + s.Equal(memo, attributes.Memo) + s.Equal(searchAttr, attributes.SearchAttributes) + + startFilter.LatestTime = common.Int64Ptr(time.Now().UnixNano()) + var closedExecutions []*types.WorkflowExecutionInfo + for i := 0; i < 10; i++ { + resp, err := s.engine.ListClosedWorkflowExecutions(createContext(), &types.ListClosedWorkflowExecutionsRequest{ + Domain: s.domainName, + MaximumPageSize: 100, + StartTimeFilter: startFilter, + ExecutionFilter: &types.WorkflowExecutionFilter{ + WorkflowID: id, + }, + }) + s.Nil(err) + if len(resp.GetExecutions()) == 4 { + closedExecutions = resp.GetExecutions() + break + } + time.Sleep(200 * time.Millisecond) + } + s.NotNil(closedExecutions) + dweResponse, err := s.engine.DescribeWorkflowExecution(createContext(), &types.DescribeWorkflowExecutionRequest{ + Domain: s.domainName, + Execution: &types.WorkflowExecution{ + WorkflowID: id, + RunID: we.RunID, + }, + }) + s.Nil(err) + expectedExecutionTime := dweResponse.WorkflowExecutionInfo.GetStartTime() + 3*time.Second.Nanoseconds() + s.Equal(expectedExecutionTime, dweResponse.WorkflowExecutionInfo.GetExecutionTime()) + + sort.Slice(closedExecutions, func(i, j int) bool { + return closedExecutions[i].GetStartTime() < closedExecutions[j].GetStartTime() + }) + lastExecution := closedExecutions[0] + + // TODO https://github.com/uber/cadence/issues/3540 + // the rest assertion can cause transient failure + if s.testClusterConfig.Persistence.SQLDBPluginName == "postgres" { + return + } + for i := 1; i != 4; i++ { + executionInfo := closedExecutions[i] + expectedBackoff := executionInfo.GetExecutionTime() - lastExecution.GetExecutionTime() + // The execution time calculate based on last execution close time + // However, the current execution time is based on the current start time + // This code is to remove the diff between current start time and last execution close time + // TODO: Remove this line once we unify the time source + executionTimeDiff := executionInfo.GetStartTime() - lastExecution.GetCloseTime() + // The backoff between any two executions should be multiplier of the target backoff duration which is 3 in this test + backoffSeconds := int(time.Duration(expectedBackoff-executionTimeDiff).Round(time.Second).Seconds()) + targetBackoffSeconds := int(targetBackoffDuration.Seconds()) + s.Equal( + 0, + backoffSeconds % targetBackoffSeconds, + "Still Flaky?: backoffSeconds: %v ((%v-%v) - (%v-%v)), targetBackoffSeconds: %v", + backoffSeconds, + executionInfo.GetExecutionTime(), + lastExecution.GetExecutionTime(), + executionInfo.GetStartTime(), + lastExecution.GetCloseTime(), + targetBackoffSeconds, + ) + lastExecution = executionInfo + } +} + +func (s *IntegrationSuite) TestCronWorkflowTimeout() { + id := "integration-wf-cron-timeout-test" + wt := "integration-wf-cron-timeout-type" + tl := "integration-wf-cron-timeout-tasklist" + identity := "worker1" + cronSchedule := "@every 3s" + + workflowType := &types.WorkflowType{} + workflowType.Name = wt + + taskList := &types.TaskList{} + taskList.Name = tl + + memo := &types.Memo{ + Fields: map[string][]byte{"memoKey": []byte("memoVal")}, + } + searchAttr := &types.SearchAttributes{ + IndexedFields: map[string][]byte{"CustomKeywordField": []byte(`"1"`)}, + } + + request := &types.StartWorkflowExecutionRequest{ + RequestID: uuid.New(), + Domain: s.domainName, + WorkflowID: id, + WorkflowType: workflowType, + TaskList: taskList, + Input: nil, + ExecutionStartToCloseTimeoutSeconds: common.Int32Ptr(1), // set workflow timeout to 1s + TaskStartToCloseTimeoutSeconds: common.Int32Ptr(1), + Identity: identity, + CronSchedule: cronSchedule, //minimum interval by standard spec is 1m (* * * * *), use non-standard descriptor for short interval for test + Memo: memo, + SearchAttributes: searchAttr, + } + + we, err0 := s.engine.StartWorkflowExecution(createContext(), request) + s.Nil(err0) + + s.Logger.Info("StartWorkflowExecution", tag.WorkflowRunID(we.RunID)) + + var executions []*types.WorkflowExecution + dtHandler := func(execution *types.WorkflowExecution, wt *types.WorkflowType, + previousStartedEventID, startedEventID int64, history *types.History) ([]byte, []*types.Decision, error) { + + executions = append(executions, execution) + return nil, []*types.Decision{ + { + DecisionType: types.DecisionTypeStartTimer.Ptr(), + + StartTimerDecisionAttributes: &types.StartTimerDecisionAttributes{ + TimerID: "timer-id", + StartToFireTimeoutSeconds: common.Int64Ptr(5), + }, + }}, nil + } + + poller := &TaskPoller{ + Engine: s.engine, + Domain: s.domainName, + TaskList: taskList, + Identity: identity, + DecisionHandler: dtHandler, + Logger: s.Logger, + T: s.T(), + } + + _, err := poller.PollAndProcessDecisionTask(false, false) + s.True(err == nil, err) + + time.Sleep(1 * time.Second) // wait for workflow timeout + + // check when workflow timeout, continueAsNew event contains expected fields + events := s.getHistory(s.domainName, executions[0]) + lastEvent := events[len(events)-1] + s.Equal(types.EventTypeWorkflowExecutionContinuedAsNew, lastEvent.GetEventType()) + attributes := lastEvent.WorkflowExecutionContinuedAsNewEventAttributes + s.Equal(types.ContinueAsNewInitiatorCronSchedule, attributes.GetInitiator()) + s.Equal("cadenceInternal:Timeout START_TO_CLOSE", attributes.GetFailureReason()) + s.Equal(memo, attributes.Memo) + s.Equal(searchAttr, attributes.SearchAttributes) + + _, err = poller.PollAndProcessDecisionTask(false, false) + s.True(err == nil, err) + + // check new run contains expected fields + events = s.getHistory(s.domainName, executions[1]) + firstEvent := events[0] + s.Equal(types.EventTypeWorkflowExecutionStarted, firstEvent.GetEventType()) + startAttributes := firstEvent.WorkflowExecutionStartedEventAttributes + s.Equal(memo, startAttributes.Memo) + s.Equal(searchAttr, startAttributes.SearchAttributes) + + // terminate cron + terminateErr := s.engine.TerminateWorkflowExecution(createContext(), &types.TerminateWorkflowExecutionRequest{ + Domain: s.domainName, + WorkflowExecution: &types.WorkflowExecution{ + WorkflowID: id, + }, + }) + s.NoError(terminateErr) +} + +func (s *IntegrationSuite) TestSequential_UserTimers() { + id := "integration-sequential-user-timers-test" + wt := "integration-sequential-user-timers-test-type" + tl := "integration-sequential-user-timers-test-tasklist" + identity := "worker1" + + workflowType := &types.WorkflowType{} + workflowType.Name = wt + + taskList := &types.TaskList{} + taskList.Name = tl + + request := &types.StartWorkflowExecutionRequest{ + RequestID: uuid.New(), + Domain: s.domainName, + WorkflowID: id, + WorkflowType: workflowType, + TaskList: taskList, + Input: nil, + ExecutionStartToCloseTimeoutSeconds: common.Int32Ptr(100), + TaskStartToCloseTimeoutSeconds: common.Int32Ptr(1), + Identity: identity, + } + + we, err0 := s.engine.StartWorkflowExecution(createContext(), request) + s.Nil(err0) + + s.Logger.Info("StartWorkflowExecution", tag.WorkflowRunID(we.RunID)) + + workflowComplete := false + timerCount := int32(4) + timerCounter := int32(0) + dtHandler := func(execution *types.WorkflowExecution, wt *types.WorkflowType, + previousStartedEventID, startedEventID int64, history *types.History) ([]byte, []*types.Decision, error) { + if timerCounter < timerCount { + timerCounter++ + buf := new(bytes.Buffer) + s.Nil(binary.Write(buf, binary.LittleEndian, timerCounter)) + return []byte(strconv.Itoa(int(timerCounter))), []*types.Decision{{ + DecisionType: types.DecisionTypeStartTimer.Ptr(), + StartTimerDecisionAttributes: &types.StartTimerDecisionAttributes{ + TimerID: fmt.Sprintf("timer-id-%d", timerCounter), + StartToFireTimeoutSeconds: common.Int64Ptr(1), + }, + }}, nil + } + + workflowComplete = true + return []byte(strconv.Itoa(int(timerCounter))), []*types.Decision{{ + DecisionType: types.DecisionTypeCompleteWorkflowExecution.Ptr(), + CompleteWorkflowExecutionDecisionAttributes: &types.CompleteWorkflowExecutionDecisionAttributes{ + Result: []byte("Done."), + }, + }}, nil + } + + poller := &TaskPoller{ + Engine: s.engine, + Domain: s.domainName, + TaskList: taskList, + Identity: identity, + DecisionHandler: dtHandler, + ActivityHandler: nil, + Logger: s.Logger, + T: s.T(), + } + + for i := 0; i < 4; i++ { + _, err := poller.PollAndProcessDecisionTask(false, false) + s.Logger.Info("PollAndProcessDecisionTask: completed") + s.Nil(err) + } + + s.False(workflowComplete) + _, err := poller.PollAndProcessDecisionTask(false, false) + s.Nil(err) + s.True(workflowComplete) +} + +func (s *IntegrationSuite) TestRateLimitBufferedEvents() { + id := "integration-rate-limit-buffered-events-test" + wt := "integration-rate-limit-buffered-events-test-type" + tl := "integration-rate-limit-buffered-events-test-tasklist" + identity := "worker1" + + workflowType := &types.WorkflowType{} + workflowType.Name = wt + + taskList := &types.TaskList{} + taskList.Name = tl + + // Start workflow execution + request := &types.StartWorkflowExecutionRequest{ + RequestID: uuid.New(), + Domain: s.domainName, + WorkflowID: id, + WorkflowType: workflowType, + TaskList: taskList, + Input: nil, + ExecutionStartToCloseTimeoutSeconds: common.Int32Ptr(100), + TaskStartToCloseTimeoutSeconds: common.Int32Ptr(10), + Identity: identity, + } + + we, err0 := s.engine.StartWorkflowExecution(createContext(), request) + s.Nil(err0) + + s.Logger.Info("StartWorkflowExecution", tag.WorkflowRunID(we.RunID)) + workflowExecution := &types.WorkflowExecution{ + WorkflowID: id, + RunID: we.RunID, + } + + // decider logic + workflowComplete := false + signalsSent := false + signalCount := 0 + dtHandler := func(execution *types.WorkflowExecution, wt *types.WorkflowType, + previousStartedEventID, startedEventID int64, h *types.History) ([]byte, []*types.Decision, error) { + + // Count signals + for _, event := range h.Events[previousStartedEventID:] { + if event.GetEventType() == types.EventTypeWorkflowExecutionSignaled { + signalCount++ + } + } + + if !signalsSent { + signalsSent = true + // Buffered Signals + for i := 0; i < 100; i++ { + buf := new(bytes.Buffer) + binary.Write(buf, binary.LittleEndian, i) + s.Nil(s.sendSignal(s.domainName, workflowExecution, "SignalName", buf.Bytes(), identity)) + } + + buf := new(bytes.Buffer) + binary.Write(buf, binary.LittleEndian, 101) + signalErr := s.sendSignal(s.domainName, workflowExecution, "SignalName", buf.Bytes(), identity) + s.Nil(signalErr) + + // this decision will be ignored as he decision task is already failed + return nil, []*types.Decision{}, nil + } + + workflowComplete = true + return nil, []*types.Decision{{ + DecisionType: types.DecisionTypeCompleteWorkflowExecution.Ptr(), + CompleteWorkflowExecutionDecisionAttributes: &types.CompleteWorkflowExecutionDecisionAttributes{ + Result: []byte("Done."), + }, + }}, nil + } + + poller := &TaskPoller{ + Engine: s.engine, + Domain: s.domainName, + TaskList: taskList, + Identity: identity, + DecisionHandler: dtHandler, + ActivityHandler: nil, + Logger: s.Logger, + T: s.T(), + } + + // first decision to send 101 signals, the last signal will force fail decision and flush buffered events. + _, err := poller.PollAndProcessDecisionTask(false, false) + s.Logger.Info("PollAndProcessDecisionTask", tag.Error(err)) + s.EqualError(err, "EntityNotExistsError{Message: Decision task not found.}") + + // Process signal in decider + _, err = poller.PollAndProcessDecisionTask(false, false) + s.Logger.Info("PollAndProcessDecisionTask", tag.Error(err)) + s.Nil(err) + + s.True(workflowComplete) + s.Equal(101, signalCount) // check that all 101 signals are received. +} + +func (s *IntegrationSuite) TestBufferedEvents() { + id := "integration-buffered-events-test" + wt := "integration-buffered-events-test-type" + tl := "integration-buffered-events-test-tasklist" + identity := "worker1" + signalName := "buffered-signal" + + workflowType := &types.WorkflowType{Name: wt} + taskList := &types.TaskList{Name: tl} + + // Start workflow execution + request := &types.StartWorkflowExecutionRequest{ + RequestID: uuid.New(), + Domain: s.domainName, + WorkflowID: id, + WorkflowType: workflowType, + TaskList: taskList, + Input: nil, + ExecutionStartToCloseTimeoutSeconds: common.Int32Ptr(100), + TaskStartToCloseTimeoutSeconds: common.Int32Ptr(1), + Identity: identity, + } + + we, err0 := s.engine.StartWorkflowExecution(createContext(), request) + s.Nil(err0) + + s.Logger.Info("StartWorkflowExecution", tag.WorkflowRunID(we.RunID)) + + // decider logic + workflowComplete := false + signalSent := false + var signalEvent *types.HistoryEvent + dtHandler := func(execution *types.WorkflowExecution, wt *types.WorkflowType, + previousStartedEventID, startedEventID int64, history *types.History) ([]byte, []*types.Decision, error) { + if !signalSent { + signalSent = true + + // this will create new event when there is in-flight decision task, and the new event will be buffered + err := s.engine.SignalWorkflowExecution(createContext(), + &types.SignalWorkflowExecutionRequest{ + Domain: s.domainName, + WorkflowExecution: &types.WorkflowExecution{ + WorkflowID: id, + }, + SignalName: "buffered-signal", + Input: []byte("buffered-signal-input"), + Identity: identity, + }) + s.NoError(err) + return nil, []*types.Decision{{ + DecisionType: types.DecisionTypeScheduleActivityTask.Ptr(), + ScheduleActivityTaskDecisionAttributes: &types.ScheduleActivityTaskDecisionAttributes{ + ActivityID: "1", + ActivityType: &types.ActivityType{Name: "test-activity-type"}, + TaskList: &types.TaskList{Name: tl}, + Input: []byte("test-input"), + ScheduleToCloseTimeoutSeconds: common.Int32Ptr(100), + ScheduleToStartTimeoutSeconds: common.Int32Ptr(2), + StartToCloseTimeoutSeconds: common.Int32Ptr(50), + HeartbeatTimeoutSeconds: common.Int32Ptr(5), + }, + }}, nil + } else if previousStartedEventID > 0 && signalEvent == nil { + for _, event := range history.Events[previousStartedEventID:] { + if *event.EventType == types.EventTypeWorkflowExecutionSignaled { + signalEvent = event + } + } + } + + workflowComplete = true + return nil, []*types.Decision{{ + DecisionType: types.DecisionTypeCompleteWorkflowExecution.Ptr(), + CompleteWorkflowExecutionDecisionAttributes: &types.CompleteWorkflowExecutionDecisionAttributes{ + Result: []byte("Done."), + }, + }}, nil + } + + poller := &TaskPoller{ + Engine: s.engine, + Domain: s.domainName, + TaskList: taskList, + Identity: identity, + DecisionHandler: dtHandler, + ActivityHandler: nil, + Logger: s.Logger, + T: s.T(), + } + + // first decision, which sends signal and the signal event should be buffered to append after first decision closed + _, err := poller.PollAndProcessDecisionTask(false, false) + s.Logger.Info("PollAndProcessDecisionTask", tag.Error(err)) + s.Nil(err) + + // check history, the signal event should be after the complete decision task + histResp, err := s.engine.GetWorkflowExecutionHistory(createContext(), &types.GetWorkflowExecutionHistoryRequest{ + Domain: s.domainName, + Execution: &types.WorkflowExecution{ + WorkflowID: id, + RunID: we.RunID, + }, + }) + s.NoError(err) + s.NotNil(histResp.History.Events) + s.True(len(histResp.History.Events) >= 6) + s.Equal(histResp.History.Events[3].GetEventType(), types.EventTypeDecisionTaskCompleted) + s.Equal(histResp.History.Events[4].GetEventType(), types.EventTypeActivityTaskScheduled) + s.Equal(histResp.History.Events[5].GetEventType(), types.EventTypeWorkflowExecutionSignaled) + + // Process signal in decider + _, err = poller.PollAndProcessDecisionTask(false, false) + s.Logger.Info("PollAndProcessDecisionTask", tag.Error(err)) + s.Nil(err) + s.NotNil(signalEvent) + s.Equal(signalName, signalEvent.WorkflowExecutionSignaledEventAttributes.SignalName) + s.Equal(identity, signalEvent.WorkflowExecutionSignaledEventAttributes.Identity) + s.True(workflowComplete) +} + +func (s *IntegrationSuite) TestDescribeWorkflowExecution() { + id := "integration-describe-wfe-test" + wt := "integration-describe-wfe-test-type" + tl := "integration-describe-wfe-test-tasklist" + identity := "worker1" + + workflowType := &types.WorkflowType{Name: wt} + taskList := &types.TaskList{Name: tl} + + // Start workflow execution + request := &types.StartWorkflowExecutionRequest{ + RequestID: uuid.New(), + Domain: s.domainName, + WorkflowID: id, + WorkflowType: workflowType, + TaskList: taskList, + Input: nil, + ExecutionStartToCloseTimeoutSeconds: common.Int32Ptr(100), + TaskStartToCloseTimeoutSeconds: common.Int32Ptr(1), + Identity: identity, + } + + we, err0 := s.engine.StartWorkflowExecution(createContext(), request) + s.Nil(err0) + + s.Logger.Info("StartWorkflowExecution", tag.WorkflowRunID(we.RunID)) + + describeWorkflowExecution := func() (*types.DescribeWorkflowExecutionResponse, error) { + return s.engine.DescribeWorkflowExecution(createContext(), &types.DescribeWorkflowExecutionRequest{ + Domain: s.domainName, + Execution: &types.WorkflowExecution{ + WorkflowID: id, + RunID: we.RunID, + }, + }) + } + dweResponse, err := describeWorkflowExecution() + s.Nil(err) + s.True(nil == dweResponse.WorkflowExecutionInfo.CloseTime) + s.Equal(int64(2), dweResponse.WorkflowExecutionInfo.HistoryLength) // WorkflowStarted, DecisionScheduled + s.Equal(dweResponse.WorkflowExecutionInfo.GetStartTime(), dweResponse.WorkflowExecutionInfo.GetExecutionTime()) + + // decider logic + workflowComplete := false + signalSent := false + dtHandler := func(execution *types.WorkflowExecution, wt *types.WorkflowType, + previousStartedEventID, startedEventID int64, history *types.History) ([]byte, []*types.Decision, error) { + if !signalSent { + signalSent = true + + s.NoError(err) + return nil, []*types.Decision{{ + DecisionType: types.DecisionTypeScheduleActivityTask.Ptr(), + ScheduleActivityTaskDecisionAttributes: &types.ScheduleActivityTaskDecisionAttributes{ + ActivityID: "1", + ActivityType: &types.ActivityType{Name: "test-activity-type"}, + TaskList: &types.TaskList{Name: tl}, + Input: []byte("test-input"), + ScheduleToCloseTimeoutSeconds: common.Int32Ptr(100), + ScheduleToStartTimeoutSeconds: common.Int32Ptr(2), + StartToCloseTimeoutSeconds: common.Int32Ptr(50), + HeartbeatTimeoutSeconds: common.Int32Ptr(5), + }, + }}, nil + } + + workflowComplete = true + return nil, []*types.Decision{{ + DecisionType: types.DecisionTypeCompleteWorkflowExecution.Ptr(), + CompleteWorkflowExecutionDecisionAttributes: &types.CompleteWorkflowExecutionDecisionAttributes{ + Result: []byte("Done."), + }, + }}, nil + } + + atHandler := func(execution *types.WorkflowExecution, activityType *types.ActivityType, + ActivityID string, input []byte, taskToken []byte) ([]byte, bool, error) { + return []byte("Activity Result."), false, nil + } + + poller := &TaskPoller{ + Engine: s.engine, + Domain: s.domainName, + TaskList: taskList, + Identity: identity, + DecisionHandler: dtHandler, + ActivityHandler: atHandler, + Logger: s.Logger, + T: s.T(), + } + + // first decision to schedule new activity + _, err = poller.PollAndProcessDecisionTask(false, false) + s.Logger.Info("PollAndProcessDecisionTask", tag.Error(err)) + s.Nil(err) + + dweResponse, err = describeWorkflowExecution() + s.Nil(err) + s.True(nil == dweResponse.WorkflowExecutionInfo.CloseStatus) + s.Equal(int64(5), dweResponse.WorkflowExecutionInfo.HistoryLength) // DecisionStarted, DecisionCompleted, ActivityScheduled + s.Equal(1, len(dweResponse.PendingActivities)) + s.Equal("test-activity-type", dweResponse.PendingActivities[0].ActivityType.GetName()) + s.Equal(int64(0), dweResponse.PendingActivities[0].GetLastHeartbeatTimestamp()) + + // process activity task + err = poller.PollAndProcessActivityTask(false) + + dweResponse, err = describeWorkflowExecution() + s.Nil(err) + s.True(nil == dweResponse.WorkflowExecutionInfo.CloseStatus) + s.Equal(int64(8), dweResponse.WorkflowExecutionInfo.HistoryLength) // ActivityTaskStarted, ActivityTaskCompleted, DecisionTaskScheduled + s.Equal(0, len(dweResponse.PendingActivities)) + + // Process signal in decider + _, err = poller.PollAndProcessDecisionTask(false, false) + s.Nil(err) + s.True(workflowComplete) + + dweResponse, err = describeWorkflowExecution() + s.Nil(err) + s.Equal(types.WorkflowExecutionCloseStatusCompleted, *dweResponse.WorkflowExecutionInfo.CloseStatus) + s.Equal(int64(11), dweResponse.WorkflowExecutionInfo.HistoryLength) // DecisionStarted, DecisionCompleted, WorkflowCompleted +} + +func (s *IntegrationSuite) TestVisibility() { + startTime := time.Now().UnixNano() + + // Start 2 workflow executions + id1 := "integration-visibility-test1" + id2 := "integration-visibility-test2" + wt := "integration-visibility-test-type" + tl := "integration-visibility-test-tasklist" + identity := "worker1" + + workflowType := &types.WorkflowType{} + workflowType.Name = wt + + taskList := &types.TaskList{} + taskList.Name = tl + + startRequest := &types.StartWorkflowExecutionRequest{ + RequestID: uuid.New(), + Domain: s.domainName, + WorkflowID: id1, + WorkflowType: workflowType, + TaskList: taskList, + Input: nil, + ExecutionStartToCloseTimeoutSeconds: common.Int32Ptr(100), + TaskStartToCloseTimeoutSeconds: common.Int32Ptr(10), + Identity: identity, + } + + startResponse, err0 := s.engine.StartWorkflowExecution(createContext(), startRequest) + s.Nil(err0) + + // Now complete one of the executions + dtHandler := func(execution *types.WorkflowExecution, wt *types.WorkflowType, + previousStartedEventID, startedEventID int64, history *types.History) ([]byte, []*types.Decision, error) { + return []byte{}, []*types.Decision{{ + DecisionType: types.DecisionTypeCompleteWorkflowExecution.Ptr(), + CompleteWorkflowExecutionDecisionAttributes: &types.CompleteWorkflowExecutionDecisionAttributes{ + Result: []byte("Done."), + }, + }}, nil + } + + poller := &TaskPoller{ + Engine: s.engine, + Domain: s.domainName, + TaskList: taskList, + Identity: identity, + DecisionHandler: dtHandler, + ActivityHandler: nil, + Logger: s.Logger, + T: s.T(), + } + + _, err1 := poller.PollAndProcessDecisionTask(false, false) + s.Nil(err1) + + // wait until the start workflow is done + var nextToken []byte + historyEventFilterType := types.HistoryEventFilterTypeCloseEvent + for { + historyResponse, historyErr := s.engine.GetWorkflowExecutionHistory(createContext(), &types.GetWorkflowExecutionHistoryRequest{ + Domain: startRequest.Domain, + Execution: &types.WorkflowExecution{ + WorkflowID: startRequest.WorkflowID, + RunID: startResponse.RunID, + }, + WaitForNewEvent: true, + HistoryEventFilterType: &historyEventFilterType, + NextPageToken: nextToken, + }) + s.Nil(historyErr) + if len(historyResponse.NextPageToken) == 0 { + break + } + + nextToken = historyResponse.NextPageToken + } + + startRequest = &types.StartWorkflowExecutionRequest{ + RequestID: uuid.New(), + Domain: s.domainName, + WorkflowID: id2, + WorkflowType: workflowType, + TaskList: taskList, + Input: nil, + ExecutionStartToCloseTimeoutSeconds: common.Int32Ptr(100), + TaskStartToCloseTimeoutSeconds: common.Int32Ptr(10), + Identity: identity, + } + + _, err2 := s.engine.StartWorkflowExecution(createContext(), startRequest) + s.Nil(err2) + + startFilter := &types.StartTimeFilter{} + startFilter.EarliestTime = common.Int64Ptr(startTime) + startFilter.LatestTime = common.Int64Ptr(time.Now().UnixNano()) + + closedCount := 0 + openCount := 0 + + var historyLength int64 + for i := 0; i < 10; i++ { + resp, err3 := s.engine.ListClosedWorkflowExecutions(createContext(), &types.ListClosedWorkflowExecutionsRequest{ + Domain: s.domainName, + MaximumPageSize: 100, + StartTimeFilter: startFilter, + }) + s.Nil(err3) + closedCount = len(resp.Executions) + if closedCount == 1 { + historyLength = resp.Executions[0].HistoryLength + break + } + s.Logger.Info("Closed WorkflowExecution is not yet visible") + time.Sleep(100 * time.Millisecond) + } + s.Equal(1, closedCount) + s.Equal(int64(5), historyLength) + + for i := 0; i < 10; i++ { + resp, err4 := s.engine.ListOpenWorkflowExecutions(createContext(), &types.ListOpenWorkflowExecutionsRequest{ + Domain: s.domainName, + MaximumPageSize: 100, + StartTimeFilter: startFilter, + }) + s.Nil(err4) + openCount = len(resp.Executions) + if openCount == 1 { + break + } + s.Logger.Info("Open WorkflowExecution is not yet visible") + time.Sleep(100 * time.Millisecond) + } + s.Equal(1, openCount) +} + +func (s *IntegrationSuite) TestChildWorkflowExecution() { + parentID := "integration-child-workflow-test-parent" + childID := "integration-child-workflow-test-child" + wtParent := "integration-child-workflow-test-parent-type" + wtChild := "integration-child-workflow-test-child-type" + tlParent := "integration-child-workflow-test-parent-tasklist" + tlChild := "integration-child-workflow-test-child-tasklist" + identity := "worker1" + + parentWorkflowType := &types.WorkflowType{} + parentWorkflowType.Name = wtParent + + childWorkflowType := &types.WorkflowType{} + childWorkflowType.Name = wtChild + + taskListParent := &types.TaskList{} + taskListParent.Name = tlParent + taskListChild := &types.TaskList{} + taskListChild.Name = tlChild + + header := &types.Header{ + Fields: map[string][]byte{"tracing": []byte("sample payload")}, + } + + request := &types.StartWorkflowExecutionRequest{ + RequestID: uuid.New(), + Domain: s.domainName, + WorkflowID: parentID, + WorkflowType: parentWorkflowType, + TaskList: taskListParent, + Input: nil, + Header: header, + ExecutionStartToCloseTimeoutSeconds: common.Int32Ptr(100), + TaskStartToCloseTimeoutSeconds: common.Int32Ptr(1), + Identity: identity, + } + + we, err0 := s.engine.StartWorkflowExecution(createContext(), request) + s.Nil(err0) + s.Logger.Info("StartWorkflowExecution", tag.WorkflowRunID(we.RunID)) + + // decider logic + childComplete := false + childExecutionStarted := false + var startedEvent *types.HistoryEvent + var completedEvent *types.HistoryEvent + + memoInfo, _ := json.Marshal("memo") + memo := &types.Memo{ + Fields: map[string][]byte{ + "Info": memoInfo, + }, + } + attrValBytes, _ := json.Marshal("attrVal") + searchAttr := &types.SearchAttributes{ + IndexedFields: map[string][]byte{ + "CustomKeywordField": attrValBytes, + }, + } + + // Parent Decider Logic + dtHandlerParent := func(execution *types.WorkflowExecution, wt *types.WorkflowType, + previousStartedEventID, startedEventID int64, history *types.History) ([]byte, []*types.Decision, error) { + s.Logger.Info("Processing decision task for ", tag.WorkflowID(execution.WorkflowID)) + + if execution.WorkflowID == parentID { + if !childExecutionStarted { + s.Logger.Info("Starting child execution.") + childExecutionStarted = true + + return nil, []*types.Decision{{ + DecisionType: types.DecisionTypeStartChildWorkflowExecution.Ptr(), + StartChildWorkflowExecutionDecisionAttributes: &types.StartChildWorkflowExecutionDecisionAttributes{ + WorkflowID: childID, + WorkflowType: childWorkflowType, + TaskList: taskListChild, + Input: []byte("child-workflow-input"), + Header: header, + ExecutionStartToCloseTimeoutSeconds: common.Int32Ptr(200), + TaskStartToCloseTimeoutSeconds: common.Int32Ptr(2), + Control: nil, + Memo: memo, + SearchAttributes: searchAttr, + }, + }}, nil + } else if previousStartedEventID > 0 { + for _, event := range history.Events[previousStartedEventID:] { + if *event.EventType == types.EventTypeChildWorkflowExecutionStarted { + startedEvent = event + return nil, []*types.Decision{}, nil + } + + if *event.EventType == types.EventTypeChildWorkflowExecutionCompleted { + completedEvent = event + return nil, []*types.Decision{{ + DecisionType: types.DecisionTypeCompleteWorkflowExecution.Ptr(), + CompleteWorkflowExecutionDecisionAttributes: &types.CompleteWorkflowExecutionDecisionAttributes{ + Result: []byte("Done."), + }, + }}, nil + } + } + } + } + + return nil, nil, nil + } + + var childStartedEvent *types.HistoryEvent + // Child Decider Logic + dtHandlerChild := func(execution *types.WorkflowExecution, wt *types.WorkflowType, + previousStartedEventID, startedEventID int64, history *types.History) ([]byte, []*types.Decision, error) { + if previousStartedEventID <= 0 { + childStartedEvent = history.Events[0] + } + + s.Logger.Info("Processing decision task for Child ", tag.WorkflowID(execution.WorkflowID)) + childComplete = true + return nil, []*types.Decision{{ + DecisionType: types.DecisionTypeCompleteWorkflowExecution.Ptr(), + CompleteWorkflowExecutionDecisionAttributes: &types.CompleteWorkflowExecutionDecisionAttributes{ + Result: []byte("Child Done."), + }, + }}, nil + } + + pollerParent := &TaskPoller{ + Engine: s.engine, + Domain: s.domainName, + TaskList: taskListParent, + Identity: identity, + DecisionHandler: dtHandlerParent, + Logger: s.Logger, + T: s.T(), + } + + pollerChild := &TaskPoller{ + Engine: s.engine, + Domain: s.domainName, + TaskList: taskListChild, + Identity: identity, + DecisionHandler: dtHandlerChild, + Logger: s.Logger, + T: s.T(), + } + + // Make first decision to start child execution + _, err := pollerParent.PollAndProcessDecisionTask(false, false) + s.Logger.Info("PollAndProcessDecisionTask", tag.Error(err)) + s.Nil(err) + s.True(childExecutionStarted) + + // Process ChildExecution Started event and Process Child Execution and complete it + _, err = pollerParent.PollAndProcessDecisionTask(false, false) + s.Logger.Info("PollAndProcessDecisionTask", tag.Error(err)) + s.Nil(err) + + _, err = pollerChild.PollAndProcessDecisionTask(false, false) + s.Logger.Info("PollAndProcessDecisionTask", tag.Error(err)) + s.Nil(err) + s.NotNil(startedEvent) + s.True(childComplete) + s.NotNil(childStartedEvent) + s.Equal(types.EventTypeWorkflowExecutionStarted, childStartedEvent.GetEventType()) + s.Equal(s.domainName, childStartedEvent.WorkflowExecutionStartedEventAttributes.GetParentWorkflowDomain()) + s.Equal(parentID, childStartedEvent.WorkflowExecutionStartedEventAttributes.ParentWorkflowExecution.GetWorkflowID()) + s.Equal(we.GetRunID(), childStartedEvent.WorkflowExecutionStartedEventAttributes.ParentWorkflowExecution.GetRunID()) + s.Equal(startedEvent.ChildWorkflowExecutionStartedEventAttributes.GetInitiatedEventID(), + childStartedEvent.WorkflowExecutionStartedEventAttributes.GetParentInitiatedEventID()) + s.Equal(header, startedEvent.ChildWorkflowExecutionStartedEventAttributes.Header) + s.Equal(header, childStartedEvent.WorkflowExecutionStartedEventAttributes.Header) + s.Equal(memo, childStartedEvent.WorkflowExecutionStartedEventAttributes.GetMemo()) + s.Equal(searchAttr, childStartedEvent.WorkflowExecutionStartedEventAttributes.GetSearchAttributes()) + + // Process ChildExecution completed event and complete parent execution + _, err = pollerParent.PollAndProcessDecisionTask(false, false) + s.Logger.Info("PollAndProcessDecisionTask", tag.Error(err)) + s.Nil(err) + s.NotNil(completedEvent) + completedAttributes := completedEvent.ChildWorkflowExecutionCompletedEventAttributes + s.Empty(completedAttributes.Domain) + s.Equal(childID, completedAttributes.WorkflowExecution.WorkflowID) + s.Equal(wtChild, completedAttributes.WorkflowType.Name) + s.Equal([]byte("Child Done."), completedAttributes.Result) +} + +func (s *IntegrationSuite) TestCronChildWorkflowExecution() { + parentID := "integration-cron-child-workflow-test-parent" + childID := "integration-cron-child-workflow-test-child" + wtParent := "integration-cron-child-workflow-test-parent-type" + wtChild := "integration-cron-child-workflow-test-child-type" + tlParent := "integration-cron-child-workflow-test-parent-tasklist" + tlChild := "integration-cron-child-workflow-test-child-tasklist" + identity := "worker1" + + cronSchedule := "@every 3s" + targetBackoffDuration := time.Second * 3 + backoffDurationTolerance := time.Second + + parentWorkflowType := &types.WorkflowType{} + parentWorkflowType.Name = wtParent + + childWorkflowType := &types.WorkflowType{} + childWorkflowType.Name = wtChild + + taskListParent := &types.TaskList{} + taskListParent.Name = tlParent + taskListChild := &types.TaskList{} + taskListChild.Name = tlChild + + request := &types.StartWorkflowExecutionRequest{ + RequestID: uuid.New(), + Domain: s.domainName, + WorkflowID: parentID, + WorkflowType: parentWorkflowType, + TaskList: taskListParent, + Input: nil, + ExecutionStartToCloseTimeoutSeconds: common.Int32Ptr(100), + TaskStartToCloseTimeoutSeconds: common.Int32Ptr(1), + Identity: identity, + } + + startParentWorkflowTS := time.Now() + we, err0 := s.engine.StartWorkflowExecution(createContext(), request) + s.Nil(err0) + s.Logger.Info("StartWorkflowExecution", tag.WorkflowRunID(we.RunID)) + + // decider logic + childExecutionStarted := false + var terminatedEvent *types.HistoryEvent + var startChildWorkflowTS time.Time + // Parent Decider Logic + dtHandlerParent := func(execution *types.WorkflowExecution, wt *types.WorkflowType, + previousStartedEventID, startedEventID int64, history *types.History) ([]byte, []*types.Decision, error) { + s.Logger.Info("Processing decision task for ", tag.WorkflowID(execution.WorkflowID)) + + if !childExecutionStarted { + s.Logger.Info("Starting child execution.") + childExecutionStarted = true + startChildWorkflowTS = time.Now() + return nil, []*types.Decision{{ + DecisionType: types.DecisionTypeStartChildWorkflowExecution.Ptr(), + StartChildWorkflowExecutionDecisionAttributes: &types.StartChildWorkflowExecutionDecisionAttributes{ + WorkflowID: childID, + WorkflowType: childWorkflowType, + TaskList: taskListChild, + Input: nil, + ExecutionStartToCloseTimeoutSeconds: common.Int32Ptr(200), + TaskStartToCloseTimeoutSeconds: common.Int32Ptr(2), + Control: nil, + CronSchedule: cronSchedule, + }, + }}, nil + } + for _, event := range history.Events[previousStartedEventID:] { + if *event.EventType == types.EventTypeChildWorkflowExecutionTerminated { + terminatedEvent = event + return nil, []*types.Decision{{ + DecisionType: types.DecisionTypeCompleteWorkflowExecution.Ptr(), + CompleteWorkflowExecutionDecisionAttributes: &types.CompleteWorkflowExecutionDecisionAttributes{ + Result: []byte("Done."), + }, + }}, nil + } + } + return nil, nil, nil + } + + // Child Decider Logic + dtHandlerChild := func(execution *types.WorkflowExecution, wt *types.WorkflowType, + previousStartedEventID, startedEventID int64, history *types.History) ([]byte, []*types.Decision, error) { + + s.Logger.Info("Processing decision task for Child ", tag.WorkflowID(execution.WorkflowID)) + return nil, []*types.Decision{{ + DecisionType: types.DecisionTypeCompleteWorkflowExecution.Ptr(), + CompleteWorkflowExecutionDecisionAttributes: &types.CompleteWorkflowExecutionDecisionAttributes{}, + }}, nil + } + + pollerParent := &TaskPoller{ + Engine: s.engine, + Domain: s.domainName, + TaskList: taskListParent, + Identity: identity, + DecisionHandler: dtHandlerParent, + Logger: s.Logger, + T: s.T(), + } + + pollerChild := &TaskPoller{ + Engine: s.engine, + Domain: s.domainName, + TaskList: taskListChild, + Identity: identity, + DecisionHandler: dtHandlerChild, + Logger: s.Logger, + T: s.T(), + } + + // Make first decision to start child execution + _, err := pollerParent.PollAndProcessDecisionTask(false, false) + s.Logger.Info("PollAndProcessDecisionTask", tag.Error(err)) + s.Nil(err) + s.True(childExecutionStarted) + + // Process ChildExecution Started event + _, err = pollerParent.PollAndProcessDecisionTask(false, false) + s.Logger.Info("PollAndProcessDecisionTask", tag.Error(err)) + s.Nil(err) + + startFilter := &types.StartTimeFilter{} + startFilter.EarliestTime = common.Int64Ptr(startChildWorkflowTS.UnixNano()) + for i := 0; i < 2; i++ { + // Sleep some time before checking the open executions. + // This will not cost extra time as the polling for first decision task will be blocked for 3 seconds. + time.Sleep(2 * time.Second) + startFilter.LatestTime = common.Int64Ptr(time.Now().UnixNano()) + resp, err := s.engine.ListOpenWorkflowExecutions(createContext(), &types.ListOpenWorkflowExecutionsRequest{ + Domain: s.domainName, + MaximumPageSize: 100, + StartTimeFilter: startFilter, + ExecutionFilter: &types.WorkflowExecutionFilter{ + WorkflowID: childID, + }, + }) + s.Nil(err) + s.Equal(1, len(resp.GetExecutions())) + + _, err = pollerChild.PollAndProcessDecisionTask(false, false) + s.Logger.Info("PollAndProcessDecisionTask", tag.Error(err)) + s.Nil(err) + + backoffDuration := time.Now().Sub(startChildWorkflowTS) + s.True(backoffDuration < targetBackoffDuration+backoffDurationTolerance) + startChildWorkflowTS = time.Now() + } + + // terminate the childworkflow + terminateErr := s.engine.TerminateWorkflowExecution(createContext(), &types.TerminateWorkflowExecutionRequest{ + Domain: s.domainName, + WorkflowExecution: &types.WorkflowExecution{ + WorkflowID: childID, + }, + }) + s.Nil(terminateErr) + + // Process ChildExecution terminated event and complete parent execution + _, err = pollerParent.PollAndProcessDecisionTask(false, false) + s.Logger.Info("PollAndProcessDecisionTask", tag.Error(err)) + s.Nil(err) + s.NotNil(terminatedEvent) + terminatedAttributes := terminatedEvent.ChildWorkflowExecutionTerminatedEventAttributes + s.Empty(terminatedAttributes.Domain) + s.Equal(childID, terminatedAttributes.WorkflowExecution.WorkflowID) + s.Equal(wtChild, terminatedAttributes.WorkflowType.Name) + + startFilter.EarliestTime = common.Int64Ptr(startParentWorkflowTS.UnixNano()) + startFilter.LatestTime = common.Int64Ptr(time.Now().UnixNano()) + var closedExecutions []*types.WorkflowExecutionInfo + for i := 0; i < 10; i++ { + resp, err := s.engine.ListClosedWorkflowExecutions(createContext(), &types.ListClosedWorkflowExecutionsRequest{ + Domain: s.domainName, + MaximumPageSize: 100, + StartTimeFilter: startFilter, + }) + s.Nil(err) + if len(resp.GetExecutions()) == 4 { + closedExecutions = resp.GetExecutions() + break + } + time.Sleep(200 * time.Millisecond) + } + s.NotNil(closedExecutions) + sort.Slice(closedExecutions, func(i, j int) bool { + return closedExecutions[i].GetStartTime() < closedExecutions[j].GetStartTime() + }) + //The first parent is not the cron workflow, only verify child workflow with cron schedule + lastExecution := closedExecutions[1] + for i := 2; i != 4; i++ { + executionInfo := closedExecutions[i] + // Round up the time precision to seconds + expectedBackoff := executionInfo.GetExecutionTime()/1000000000 - lastExecution.GetExecutionTime()/1000000000 + // The execution time calculate based on last execution close time + // However, the current execution time is based on the current start time + // This code is to remove the diff between current start time and last execution close time + // TODO: Remove this line once we unify the time source. + executionTimeDiff := executionInfo.GetStartTime()/1000000000 - lastExecution.GetCloseTime()/1000000000 + // The backoff between any two executions should be multiplier of the target backoff duration which is 3 in this test + s.Equal(int64(0), int64(expectedBackoff-executionTimeDiff)/1000000000%(targetBackoffDuration.Nanoseconds()/1000000000)) + lastExecution = executionInfo + } +} + +func (s *IntegrationSuite) TestWorkflowTimeout() { + startTime := time.Now().UnixNano() + + id := "integration-workflow-timeout" + wt := "integration-workflow-timeout-type" + tl := "integration-workflow-timeout-tasklist" + identity := "worker1" + + workflowType := &types.WorkflowType{} + workflowType.Name = wt + + taskList := &types.TaskList{} + taskList.Name = tl + + request := &types.StartWorkflowExecutionRequest{ + RequestID: uuid.New(), + Domain: s.domainName, + WorkflowID: id, + WorkflowType: workflowType, + TaskList: taskList, + Input: nil, + ExecutionStartToCloseTimeoutSeconds: common.Int32Ptr(1), + TaskStartToCloseTimeoutSeconds: common.Int32Ptr(1), + Identity: identity, + } + + we, err0 := s.engine.StartWorkflowExecution(createContext(), request) + s.Nil(err0) + + s.Logger.Info("StartWorkflowExecution", tag.WorkflowRunID(we.RunID)) + + workflowComplete := false + +GetHistoryLoop: + for i := 0; i < 10; i++ { + historyResponse, err := s.engine.GetWorkflowExecutionHistory(createContext(), &types.GetWorkflowExecutionHistoryRequest{ + Domain: s.domainName, + Execution: &types.WorkflowExecution{ + WorkflowID: id, + RunID: we.RunID, + }, + }) + s.Nil(err) + history := historyResponse.History + + lastEvent := history.Events[len(history.Events)-1] + if *lastEvent.EventType != types.EventTypeWorkflowExecutionTimedOut { + s.Logger.Warn("Execution not timedout yet.") + time.Sleep(200 * time.Millisecond) + continue GetHistoryLoop + } + + timeoutEventAttributes := lastEvent.WorkflowExecutionTimedOutEventAttributes + s.Equal(types.TimeoutTypeStartToClose, *timeoutEventAttributes.TimeoutType) + workflowComplete = true + break GetHistoryLoop + } + s.True(workflowComplete) + + startFilter := &types.StartTimeFilter{} + startFilter.EarliestTime = common.Int64Ptr(startTime) + startFilter.LatestTime = common.Int64Ptr(time.Now().UnixNano()) + + closedCount := 0 +ListClosedLoop: + for i := 0; i < 10; i++ { + resp, err3 := s.engine.ListClosedWorkflowExecutions(createContext(), &types.ListClosedWorkflowExecutionsRequest{ + Domain: s.domainName, + MaximumPageSize: 100, + StartTimeFilter: startFilter, + }) + s.Nil(err3) + closedCount = len(resp.Executions) + if closedCount == 0 { + s.Logger.Info("Closed WorkflowExecution is not yet visibile") + time.Sleep(1000 * time.Millisecond) + continue ListClosedLoop + } + break ListClosedLoop + } + s.Equal(1, closedCount) +} + +func (s *IntegrationSuite) TestDecisionTaskFailed() { + id := "integration-decisiontask-failed-test" + wt := "integration-decisiontask-failed-test-type" + tl := "integration-decisiontask-failed-test-tasklist" + identity := "worker1" + activityName := "activity_type1" + + workflowType := &types.WorkflowType{} + workflowType.Name = wt + + taskList := &types.TaskList{} + taskList.Name = tl + + // Start workflow execution + request := &types.StartWorkflowExecutionRequest{ + RequestID: uuid.New(), + Domain: s.domainName, + WorkflowID: id, + WorkflowType: workflowType, + TaskList: taskList, + Input: nil, + ExecutionStartToCloseTimeoutSeconds: common.Int32Ptr(100), + TaskStartToCloseTimeoutSeconds: common.Int32Ptr(10), + Identity: identity, + } + + we, err0 := s.engine.StartWorkflowExecution(createContext(), request) + s.Nil(err0) + s.Logger.Info("StartWorkflowExecution", tag.WorkflowRunID(we.RunID)) + + workflowExecution := &types.WorkflowExecution{ + WorkflowID: id, + RunID: we.RunID, + } + + // decider logic + workflowComplete := false + activityScheduled := false + activityData := int32(1) + failureCount := 10 + signalCount := 0 + sendSignal := false + lastDecisionTimestamp := int64(0) + //var signalEvent *types.HistoryEvent + dtHandler := func(execution *types.WorkflowExecution, wt *types.WorkflowType, + previousStartedEventID, startedEventID int64, history *types.History) ([]byte, []*types.Decision, error) { + // Count signals + for _, event := range history.Events[previousStartedEventID:] { + if event.GetEventType() == types.EventTypeWorkflowExecutionSignaled { + signalCount++ + } + } + // Some signals received on this decision + if signalCount == 1 { + return nil, []*types.Decision{}, nil + } + + // Send signals during decision + if sendSignal { + s.sendSignal(s.domainName, workflowExecution, "signalC", nil, identity) + s.sendSignal(s.domainName, workflowExecution, "signalD", nil, identity) + s.sendSignal(s.domainName, workflowExecution, "signalE", nil, identity) + sendSignal = false + } + + if !activityScheduled { + activityScheduled = true + buf := new(bytes.Buffer) + s.Nil(binary.Write(buf, binary.LittleEndian, activityData)) + + return nil, []*types.Decision{{ + DecisionType: types.DecisionTypeScheduleActivityTask.Ptr(), + ScheduleActivityTaskDecisionAttributes: &types.ScheduleActivityTaskDecisionAttributes{ + ActivityID: "1", + ActivityType: &types.ActivityType{Name: activityName}, + TaskList: &types.TaskList{Name: tl}, + Input: buf.Bytes(), + ScheduleToCloseTimeoutSeconds: common.Int32Ptr(100), + ScheduleToStartTimeoutSeconds: common.Int32Ptr(2), + StartToCloseTimeoutSeconds: common.Int32Ptr(50), + HeartbeatTimeoutSeconds: common.Int32Ptr(5), + }, + }}, nil + } else if failureCount > 0 { + // Otherwise decrement failureCount and keep failing decisions + failureCount-- + return nil, nil, errors.New("Decider Panic") + } + + workflowComplete = true + time.Sleep(time.Second) + s.Logger.Warn(fmt.Sprintf("PrevStarted: %v, StartedEventID: %v, Size: %v", previousStartedEventID, startedEventID, + len(history.Events))) + lastDecisionEvent := history.Events[startedEventID-1] + s.Equal(types.EventTypeDecisionTaskStarted, lastDecisionEvent.GetEventType()) + lastDecisionTimestamp = lastDecisionEvent.GetTimestamp() + return nil, []*types.Decision{{ + DecisionType: types.DecisionTypeCompleteWorkflowExecution.Ptr(), + CompleteWorkflowExecutionDecisionAttributes: &types.CompleteWorkflowExecutionDecisionAttributes{ + Result: []byte("Done."), + }, + }}, nil + } + + // activity handler + atHandler := func(execution *types.WorkflowExecution, activityType *types.ActivityType, + ActivityID string, input []byte, taskToken []byte) ([]byte, bool, error) { + + return []byte("Activity Result."), false, nil + } + + poller := &TaskPoller{ + Engine: s.engine, + Domain: s.domainName, + TaskList: taskList, + Identity: identity, + DecisionHandler: dtHandler, + ActivityHandler: atHandler, + Logger: s.Logger, + T: s.T(), + } + + // Make first decision to schedule activity + _, err := poller.PollAndProcessDecisionTask(false, false) + s.Logger.Info("PollAndProcessDecisionTask", tag.Error(err)) + s.Nil(err) + + // process activity + err = poller.PollAndProcessActivityTask(false) + s.Logger.Info("PollAndProcessActivityTask", tag.Error(err)) + s.Nil(err) + + // fail decision 5 times + for i := 0; i < 5; i++ { + _, err := poller.PollAndProcessDecisionTaskWithAttempt(false, false, false, false, int64(i)) + s.Nil(err) + } + + err = s.sendSignal(s.domainName, workflowExecution, "signalA", nil, identity) + s.Nil(err, "failed to send signal to execution") + + // process signal + _, err = poller.PollAndProcessDecisionTask(false, false) + s.Logger.Info("PollAndProcessDecisionTask", tag.Error(err)) + s.Nil(err) + s.Equal(1, signalCount) + + // send another signal to trigger decision + err = s.sendSignal(s.domainName, workflowExecution, "signalB", nil, identity) + s.Nil(err, "failed to send signal to execution") + + // fail decision 2 more times + for i := 0; i < 2; i++ { + _, err := poller.PollAndProcessDecisionTaskWithAttempt(false, false, false, false, int64(i)) + s.Nil(err) + } + s.Equal(3, signalCount) + + // now send a signal during failed decision + sendSignal = true + _, err = poller.PollAndProcessDecisionTaskWithAttempt(false, false, false, false, int64(2)) + s.Nil(err) + s.Equal(4, signalCount) + + // fail decision 1 more times + for i := 0; i < 2; i++ { + _, err := poller.PollAndProcessDecisionTaskWithAttempt(false, false, false, false, int64(i)) + s.Nil(err) + } + s.Equal(12, signalCount) + + // Make complete workflow decision + _, err = poller.PollAndProcessDecisionTaskWithAttempt(true, false, false, false, int64(2)) + s.Logger.Info("pollAndProcessDecisionTask", tag.Error(err)) + s.Nil(err) + s.True(workflowComplete) + s.Equal(16, signalCount) + + events := s.getHistory(s.domainName, workflowExecution) + var lastEvent *types.HistoryEvent + var lastDecisionStartedEvent *types.HistoryEvent + lastIdx := 0 + for i, e := range events { + if e.GetEventType() == types.EventTypeDecisionTaskStarted { + lastDecisionStartedEvent = e + lastIdx = i + } + lastEvent = e + } + s.Equal(types.EventTypeWorkflowExecutionCompleted, lastEvent.GetEventType()) + s.Logger.Info(fmt.Sprintf("Last Decision Time: %v, Last Decision History Timestamp: %v, Complete Timestamp: %v", + time.Unix(0, lastDecisionTimestamp), time.Unix(0, lastDecisionStartedEvent.GetTimestamp()), + time.Unix(0, lastEvent.GetTimestamp()))) + s.Equal(lastDecisionTimestamp, lastDecisionStartedEvent.GetTimestamp()) + s.True(time.Duration(lastEvent.GetTimestamp()-lastDecisionTimestamp) >= time.Second) + + s.Equal(2, len(events)-lastIdx-1) + decisionCompletedEvent := events[lastIdx+1] + workflowCompletedEvent := events[lastIdx+2] + s.Equal(types.EventTypeDecisionTaskCompleted, decisionCompletedEvent.GetEventType()) + s.Equal(types.EventTypeWorkflowExecutionCompleted, workflowCompletedEvent.GetEventType()) +} + +func (s *IntegrationSuite) TestDescribeTaskList() { + WorkflowID := "integration-get-poller-history" + workflowTypeName := "integration-get-poller-history-type" + tasklistName := "integration-get-poller-history-tasklist" + identity := "worker1" + activityName := "activity_type1" + + workflowType := &types.WorkflowType{} + workflowType.Name = workflowTypeName + + taskList := &types.TaskList{} + taskList.Name = tasklistName + + // Start workflow execution + request := &types.StartWorkflowExecutionRequest{ + RequestID: uuid.New(), + Domain: s.domainName, + WorkflowID: WorkflowID, + WorkflowType: workflowType, + TaskList: taskList, + Input: nil, + ExecutionStartToCloseTimeoutSeconds: common.Int32Ptr(100), + TaskStartToCloseTimeoutSeconds: common.Int32Ptr(1), + Identity: identity, + } + + we, err0 := s.engine.StartWorkflowExecution(createContext(), request) + s.Nil(err0) + + s.Logger.Info("StartWorkflowExecution", tag.WorkflowRunID(we.RunID)) + + // decider logic + activityScheduled := false + activityData := int32(1) + // var signalEvent *types.HistoryEvent + dtHandler := func(execution *types.WorkflowExecution, wt *types.WorkflowType, + previousStartedEventID, startedEventID int64, history *types.History) ([]byte, []*types.Decision, error) { + + if !activityScheduled { + activityScheduled = true + buf := new(bytes.Buffer) + s.Nil(binary.Write(buf, binary.LittleEndian, activityData)) + + return nil, []*types.Decision{{ + DecisionType: types.DecisionTypeScheduleActivityTask.Ptr(), + ScheduleActivityTaskDecisionAttributes: &types.ScheduleActivityTaskDecisionAttributes{ + ActivityID: "1", + ActivityType: &types.ActivityType{Name: activityName}, + TaskList: taskList, + Input: buf.Bytes(), + ScheduleToCloseTimeoutSeconds: common.Int32Ptr(100), + ScheduleToStartTimeoutSeconds: common.Int32Ptr(25), + StartToCloseTimeoutSeconds: common.Int32Ptr(50), + HeartbeatTimeoutSeconds: common.Int32Ptr(25), + }, + }}, nil + } + + return nil, []*types.Decision{{ + DecisionType: types.DecisionTypeCompleteWorkflowExecution.Ptr(), + CompleteWorkflowExecutionDecisionAttributes: &types.CompleteWorkflowExecutionDecisionAttributes{ + Result: []byte("Done."), + }, + }}, nil + } + + atHandler := func(execution *types.WorkflowExecution, activityType *types.ActivityType, + ActivityID string, input []byte, taskToken []byte) ([]byte, bool, error) { + return []byte("Activity Result."), false, nil + } + + poller := &TaskPoller{ + Engine: s.engine, + Domain: s.domainName, + TaskList: taskList, + Identity: identity, + DecisionHandler: dtHandler, + ActivityHandler: atHandler, + Logger: s.Logger, + T: s.T(), + } + + // this function poll events from history side + testDescribeTaskList := func(domain string, tasklist *types.TaskList, tasklistType types.TaskListType) []*types.PollerInfo { + responseInner, errInner := s.engine.DescribeTaskList(createContext(), &types.DescribeTaskListRequest{ + Domain: domain, + TaskList: taskList, + TaskListType: &tasklistType, + }) + + s.Nil(errInner) + return responseInner.Pollers + } + + before := time.Now() + + // when no one polling on the tasklist (activity or decition), there shall be no poller information + pollerInfos := testDescribeTaskList(s.domainName, taskList, types.TaskListTypeActivity) + s.Empty(pollerInfos) + pollerInfos = testDescribeTaskList(s.domainName, taskList, types.TaskListTypeDecision) + s.Empty(pollerInfos) + + _, errDecision := poller.PollAndProcessDecisionTask(false, false) + s.Nil(errDecision) + pollerInfos = testDescribeTaskList(s.domainName, taskList, types.TaskListTypeActivity) + s.Empty(pollerInfos) + pollerInfos = testDescribeTaskList(s.domainName, taskList, types.TaskListTypeDecision) + s.Equal(1, len(pollerInfos)) + s.Equal(identity, pollerInfos[0].GetIdentity()) + s.True(time.Unix(0, pollerInfos[0].GetLastAccessTime()).After(before)) + s.NotEmpty(pollerInfos[0].GetLastAccessTime()) + + errActivity := poller.PollAndProcessActivityTask(false) + s.Nil(errActivity) + pollerInfos = testDescribeTaskList(s.domainName, taskList, types.TaskListTypeActivity) + s.Equal(1, len(pollerInfos)) + s.Equal(identity, pollerInfos[0].GetIdentity()) + s.True(time.Unix(0, pollerInfos[0].GetLastAccessTime()).After(before)) + s.NotEmpty(pollerInfos[0].GetLastAccessTime()) + pollerInfos = testDescribeTaskList(s.domainName, taskList, types.TaskListTypeDecision) + s.Equal(1, len(pollerInfos)) + s.Equal(identity, pollerInfos[0].GetIdentity()) + s.True(time.Unix(0, pollerInfos[0].GetLastAccessTime()).After(before)) + s.NotEmpty(pollerInfos[0].GetLastAccessTime()) +} + +func (s *IntegrationSuite) TestTransientDecisionTimeout() { + id := "integration-transient-decision-timeout-test" + wt := "integration-transient-decision-timeout-test-type" + tl := "integration-transient-decision-timeout-test-tasklist" + identity := "worker1" + + workflowType := &types.WorkflowType{} + workflowType.Name = wt + + taskList := &types.TaskList{} + taskList.Name = tl + + // Start workflow execution + request := &types.StartWorkflowExecutionRequest{ + RequestID: uuid.New(), + Domain: s.domainName, + WorkflowID: id, + WorkflowType: workflowType, + TaskList: taskList, + Input: nil, + ExecutionStartToCloseTimeoutSeconds: common.Int32Ptr(100), + TaskStartToCloseTimeoutSeconds: common.Int32Ptr(2), + Identity: identity, + } + + we, err0 := s.engine.StartWorkflowExecution(createContext(), request) + s.Nil(err0) + s.Logger.Info("StartWorkflowExecution", tag.WorkflowRunID(we.RunID)) + + workflowExecution := &types.WorkflowExecution{ + WorkflowID: id, + RunID: we.RunID, + } + + // decider logic + workflowComplete := false + failDecision := true + signalCount := 0 + //var signalEvent *types.HistoryEvent + dtHandler := func(execution *types.WorkflowExecution, wt *types.WorkflowType, + previousStartedEventID, startedEventID int64, history *types.History) ([]byte, []*types.Decision, error) { + if failDecision { + failDecision = false + return nil, nil, errors.New("Decider Panic") + } + + // Count signals + for _, event := range history.Events[previousStartedEventID:] { + if event.GetEventType() == types.EventTypeWorkflowExecutionSignaled { + signalCount++ + } + } + + workflowComplete = true + return nil, []*types.Decision{{ + DecisionType: types.DecisionTypeCompleteWorkflowExecution.Ptr(), + CompleteWorkflowExecutionDecisionAttributes: &types.CompleteWorkflowExecutionDecisionAttributes{ + Result: []byte("Done."), + }, + }}, nil + } + + poller := &TaskPoller{ + Engine: s.engine, + Domain: s.domainName, + TaskList: taskList, + Identity: identity, + DecisionHandler: dtHandler, + ActivityHandler: nil, + Logger: s.Logger, + T: s.T(), + } + + // First decision immediately fails and schedules a transient decision + _, err := poller.PollAndProcessDecisionTask(false, false) + s.Logger.Info("PollAndProcessDecisionTask", tag.Error(err)) + s.Nil(err) + + // Now send a signal when transient decision is scheduled + err = s.sendSignal(s.domainName, workflowExecution, "signalA", nil, identity) + s.Nil(err, "failed to send signal to execution") + + // Drop decision task to cause a Decision Timeout + _, err = poller.PollAndProcessDecisionTask(false, true) + s.Logger.Info("PollAndProcessDecisionTask", tag.Error(err)) + s.Nil(err) + + // Now process signal and complete workflow execution + _, err = poller.PollAndProcessDecisionTaskWithAttempt(true, false, false, false, int64(1)) + s.Logger.Info("PollAndProcessDecisionTask", tag.Error(err)) + s.Nil(err) + + s.Equal(1, signalCount) + s.True(workflowComplete) +} + +func (s *IntegrationSuite) TestNoTransientDecisionAfterFlushBufferedEvents() { + id := "integration-no-transient-decision-after-flush-buffered-events-test" + wt := "integration-no-transient-decision-after-flush-buffered-events-test-type" + tl := "integration-no-transient-decision-after-flush-buffered-events-test-tasklist" + identity := "worker1" + + workflowType := &types.WorkflowType{Name: wt} + taskList := &types.TaskList{Name: tl} + + // Start workflow execution + request := &types.StartWorkflowExecutionRequest{ + RequestID: uuid.New(), + Domain: s.domainName, + WorkflowID: id, + WorkflowType: workflowType, + TaskList: taskList, + Input: nil, + ExecutionStartToCloseTimeoutSeconds: common.Int32Ptr(100), + TaskStartToCloseTimeoutSeconds: common.Int32Ptr(20), + Identity: identity, + } + + we, err0 := s.engine.StartWorkflowExecution(createContext(), request) + s.Nil(err0) + + s.Logger.Info("StartWorkflowExecution", tag.WorkflowRunID(we.RunID)) + + // decider logic + workflowComplete := false + continueAsNewAndSignal := false + dtHandler := func(execution *types.WorkflowExecution, wt *types.WorkflowType, + previousStartedEventID, startedEventID int64, history *types.History) ([]byte, []*types.Decision, error) { + + if !continueAsNewAndSignal { + continueAsNewAndSignal = true + // this will create new event when there is in-flight decision task, and the new event will be buffered + err := s.engine.SignalWorkflowExecution(createContext(), + &types.SignalWorkflowExecutionRequest{ + Domain: s.domainName, + WorkflowExecution: &types.WorkflowExecution{ + WorkflowID: id, + }, + SignalName: "buffered-signal-1", + Input: []byte("buffered-signal-input"), + Identity: identity, + }) + s.NoError(err) + + return nil, []*types.Decision{{ + DecisionType: types.DecisionTypeContinueAsNewWorkflowExecution.Ptr(), + ContinueAsNewWorkflowExecutionDecisionAttributes: &types.ContinueAsNewWorkflowExecutionDecisionAttributes{ + WorkflowType: workflowType, + TaskList: taskList, + Input: nil, + ExecutionStartToCloseTimeoutSeconds: common.Int32Ptr(1000), + TaskStartToCloseTimeoutSeconds: common.Int32Ptr(100), + }, + }}, nil + } + + workflowComplete = true + return nil, []*types.Decision{{ + DecisionType: types.DecisionTypeCompleteWorkflowExecution.Ptr(), + CompleteWorkflowExecutionDecisionAttributes: &types.CompleteWorkflowExecutionDecisionAttributes{ + Result: []byte("Done."), + }, + }}, nil + } + + poller := &TaskPoller{ + Engine: s.engine, + Domain: s.domainName, + TaskList: taskList, + Identity: identity, + DecisionHandler: dtHandler, + Logger: s.Logger, + T: s.T(), + } + + // fist decision, this try to do a continue as new but there is a buffered event, + // so it will fail and create a new decision + _, err := poller.PollAndProcessDecisionTask(false, false) + s.Logger.Info("pollAndProcessDecisionTask", tag.Error(err)) + s.Nil(err) + + // second decision, which will complete the workflow + // this expect the decision to have attempt == 0 + _, err = poller.PollAndProcessDecisionTaskWithAttempt(true, false, false, false, 0) + s.Logger.Info("pollAndProcessDecisionTask", tag.Error(err)) + s.Nil(err) + + s.True(workflowComplete) +} + +func (s *IntegrationSuite) TestRelayDecisionTimeout() { + id := "integration-relay-decision-timeout-test" + wt := "integration-relay-decision-timeout-test-type" + tl := "integration-relay-decision-timeout-test-tasklist" + identity := "worker1" + + workflowType := &types.WorkflowType{} + workflowType.Name = wt + + taskList := &types.TaskList{} + taskList.Name = tl + + // Start workflow execution + request := &types.StartWorkflowExecutionRequest{ + RequestID: uuid.New(), + Domain: s.domainName, + WorkflowID: id, + WorkflowType: workflowType, + TaskList: taskList, + Input: nil, + ExecutionStartToCloseTimeoutSeconds: common.Int32Ptr(100), + TaskStartToCloseTimeoutSeconds: common.Int32Ptr(2), + Identity: identity, + } + + we, err0 := s.engine.StartWorkflowExecution(createContext(), request) + s.Nil(err0) + s.Logger.Info("StartWorkflowExecution", tag.WorkflowRunID(we.RunID)) + + workflowExecution := &types.WorkflowExecution{ + WorkflowID: id, + RunID: we.RunID, + } + + workflowComplete, isFirst := false, true + dtHandler := func(execution *types.WorkflowExecution, wt *types.WorkflowType, + previousStartedEventID, startedEventID int64, history *types.History) ([]byte, []*types.Decision, error) { + if isFirst { + isFirst = false + return nil, []*types.Decision{{ + DecisionType: types.DecisionTypeRecordMarker.Ptr(), + RecordMarkerDecisionAttributes: &types.RecordMarkerDecisionAttributes{ + MarkerName: "test-marker", + }, + }}, nil + } + workflowComplete = true + return nil, []*types.Decision{{ + DecisionType: types.DecisionTypeCompleteWorkflowExecution.Ptr(), + CompleteWorkflowExecutionDecisionAttributes: &types.CompleteWorkflowExecutionDecisionAttributes{}, + }}, nil + } + + poller := &TaskPoller{ + Engine: s.engine, + Domain: s.domainName, + TaskList: taskList, + Identity: identity, + DecisionHandler: dtHandler, + ActivityHandler: nil, + Logger: s.Logger, + T: s.T(), + } + + // First decision task complete with a marker decision, and request to relay decision (immediately return a new decision task) + _, newTask, err := poller.PollAndProcessDecisionTaskWithAttemptAndRetryAndForceNewDecision( + false, + false, + false, + false, + 0, + 3, + true, + nil) + s.Logger.Info("PollAndProcessDecisionTask", tag.Error(err)) + s.Nil(err) + s.NotNil(newTask) + s.NotNil(newTask.DecisionTask) + + time.Sleep(time.Second * 2) // wait 2s for relay decision to timeout + decisionTaskTimeout := false + for i := 0; i < 3; i++ { + events := s.getHistory(s.domainName, workflowExecution) + if len(events) >= 8 { + s.Equal(types.EventTypeDecisionTaskTimedOut, events[7].GetEventType()) + s.Equal(types.TimeoutTypeStartToClose, events[7].DecisionTaskTimedOutEventAttributes.GetTimeoutType()) + decisionTaskTimeout = true + break + } + time.Sleep(time.Second) + } + // verify relay decision task timeout + s.True(decisionTaskTimeout) + + // Now complete workflow + _, err = poller.PollAndProcessDecisionTaskWithAttempt(true, false, false, false, int64(1)) + s.Logger.Info("PollAndProcessDecisionTask", tag.Error(err)) + s.Nil(err) + + s.True(workflowComplete) +} + +func (s *IntegrationSuite) TestTaskProcessingProtectionForRateLimitError() { + id := "integration-task-processing-protection-for-rate-limit-error-test" + wt := "integration-task-processing-protection-for-rate-limit-error-test-type" + tl := "integration-task-processing-protection-for-rate-limit-error-test-tasklist" + identity := "worker1" + + workflowType := &types.WorkflowType{} + workflowType.Name = wt + + taskList := &types.TaskList{} + taskList.Name = tl + + // Start workflow execution + request := &types.StartWorkflowExecutionRequest{ + RequestID: uuid.New(), + Domain: s.domainName, + WorkflowID: id, + WorkflowType: workflowType, + TaskList: taskList, + Input: nil, + ExecutionStartToCloseTimeoutSeconds: common.Int32Ptr(601), + TaskStartToCloseTimeoutSeconds: common.Int32Ptr(600), + Identity: identity, + } + + we, err0 := s.engine.StartWorkflowExecution(createContext(), request) + s.Nil(err0) + + s.Logger.Info("StartWorkflowExecution", tag.WorkflowRunID(we.RunID)) + workflowExecution := &types.WorkflowExecution{ + WorkflowID: id, + RunID: we.RunID, + } + + // decider logic + workflowComplete := false + signalCount := 0 + createUserTimer := false + dtHandler := func(execution *types.WorkflowExecution, wt *types.WorkflowType, + previousStartedEventID, startedEventID int64, h *types.History) ([]byte, []*types.Decision, error) { + + if !createUserTimer { + createUserTimer = true + + return nil, []*types.Decision{{ + DecisionType: types.DecisionTypeStartTimer.Ptr(), + StartTimerDecisionAttributes: &types.StartTimerDecisionAttributes{ + TimerID: "timer-id-1", + StartToFireTimeoutSeconds: common.Int64Ptr(5), + }, + }}, nil + } + + // Count signals + for _, event := range h.Events[previousStartedEventID:] { + if event.GetEventType() == types.EventTypeWorkflowExecutionSignaled { + signalCount++ + } + } + + workflowComplete = true + return nil, []*types.Decision{{ + DecisionType: types.DecisionTypeCompleteWorkflowExecution.Ptr(), + CompleteWorkflowExecutionDecisionAttributes: &types.CompleteWorkflowExecutionDecisionAttributes{ + Result: []byte("Done."), + }, + }}, nil + } + + poller := &TaskPoller{ + Engine: s.engine, + Domain: s.domainName, + TaskList: taskList, + Identity: identity, + DecisionHandler: dtHandler, + ActivityHandler: nil, + Logger: s.Logger, + T: s.T(), + } + + // Process first decision to create user timer + _, err := poller.PollAndProcessDecisionTask(false, false) + s.Logger.Info("PollAndProcessDecisionTask", tag.Error(err)) + s.Nil(err) + + // Send one signal to create a new decision + buf := new(bytes.Buffer) + binary.Write(buf, binary.LittleEndian, 0) + s.Nil(s.sendSignal(s.domainName, workflowExecution, "SignalName", buf.Bytes(), identity)) + + // Drop decision to cause all events to be buffered from now on + _, err = poller.PollAndProcessDecisionTask(false, true) + s.Logger.Info("PollAndProcessDecisionTask", tag.Error(err)) + s.Nil(err) + + // Buffered 100 Signals + for i := 1; i < 101; i++ { + buf := new(bytes.Buffer) + binary.Write(buf, binary.LittleEndian, i) + s.Nil(s.sendSignal(s.domainName, workflowExecution, "SignalName", buf.Bytes(), identity)) + } + + // 101 signal, which will fail the decision + buf = new(bytes.Buffer) + binary.Write(buf, binary.LittleEndian, 101) + signalErr := s.sendSignal(s.domainName, workflowExecution, "SignalName", buf.Bytes(), identity) + s.Nil(signalErr) + + // Process signal in decider + _, err = poller.PollAndProcessDecisionTaskWithAttempt(true, false, false, false, 0) + s.Logger.Info("pollAndProcessDecisionTask", tag.Error(err)) + s.Nil(err) + + s.True(workflowComplete) + s.Equal(102, signalCount) +} + +func (s *IntegrationSuite) TestStickyTimeout_NonTransientDecision() { + id := "integration-sticky-timeout-non-transient-decision" + wt := "integration-sticky-timeout-non-transient-decision-type" + tl := "integration-sticky-timeout-non-transient-decision-tasklist" + stl := "integration-sticky-timeout-non-transient-decision-tasklist-sticky" + identity := "worker1" + + workflowType := &types.WorkflowType{} + workflowType.Name = wt + + taskList := &types.TaskList{} + taskList.Name = tl + + stickyTaskList := &types.TaskList{} + stickyTaskList.Name = stl + stickyScheduleToStartTimeoutSeconds := common.Int32Ptr(2) + + // Start workflow execution + request := &types.StartWorkflowExecutionRequest{ + RequestID: uuid.New(), + Domain: s.domainName, + WorkflowID: id, + WorkflowType: workflowType, + TaskList: taskList, + Input: nil, + ExecutionStartToCloseTimeoutSeconds: common.Int32Ptr(100), + TaskStartToCloseTimeoutSeconds: common.Int32Ptr(1), + Identity: identity, + } + + we, err0 := s.engine.StartWorkflowExecution(createContext(), request) + s.Nil(err0) + + s.Logger.Info("StartWorkflowExecution", tag.WorkflowRunID(we.RunID)) + workflowExecution := &types.WorkflowExecution{ + WorkflowID: id, + RunID: we.RunID, + } + + // decider logic + localActivityDone := false + failureCount := 5 + dtHandler := func(execution *types.WorkflowExecution, wt *types.WorkflowType, + previousStartedEventID, startedEventID int64, history *types.History) ([]byte, []*types.Decision, error) { + + if !localActivityDone { + localActivityDone = true + + return nil, []*types.Decision{{ + DecisionType: types.DecisionTypeRecordMarker.Ptr(), + RecordMarkerDecisionAttributes: &types.RecordMarkerDecisionAttributes{ + MarkerName: "local activity marker", + Details: []byte("local activity data"), + }, + }}, nil + } + + if failureCount > 0 { + // send a signal on third failure to be buffered, forcing a non-transient decision when buffer is flushed + /*if failureCount == 3 { + err := s.engine.SignalWorkflowExecution(createContext(), &types.SignalWorkflowExecutionRequest{ + Domain: s.domainName, + WorkflowExecution: workflowExecution, + SignalName: common.StringPtr("signalB"), + Input: []byte("signal input"), + Identity: identity, + RequestID: uuid.New(), + }) + s.Nil(err) + }*/ + failureCount-- + return nil, nil, errors.New("non deterministic error") + } + + return nil, []*types.Decision{{ + DecisionType: types.DecisionTypeCompleteWorkflowExecution.Ptr(), + CompleteWorkflowExecutionDecisionAttributes: &types.CompleteWorkflowExecutionDecisionAttributes{ + Result: []byte("Done."), + }, + }}, nil + } + + poller := &TaskPoller{ + Engine: s.engine, + Domain: s.domainName, + TaskList: taskList, + Identity: identity, + DecisionHandler: dtHandler, + Logger: s.Logger, + T: s.T(), + StickyTaskList: stickyTaskList, + StickyScheduleToStartTimeoutSeconds: stickyScheduleToStartTimeoutSeconds, + } + + _, err := poller.PollAndProcessDecisionTaskWithAttempt(false, false, false, true, int64(0)) + s.Logger.Info("PollAndProcessDecisionTask", tag.Error(err)) + s.Nil(err) + + err = s.engine.SignalWorkflowExecution(createContext(), &types.SignalWorkflowExecutionRequest{ + Domain: s.domainName, + WorkflowExecution: workflowExecution, + SignalName: "signalA", + Input: []byte("signal input"), + Identity: identity, + RequestID: uuid.New(), + }) + + // Wait for decision timeout + stickyTimeout := false +WaitForStickyTimeoutLoop: + for i := 0; i < 10; i++ { + events := s.getHistory(s.domainName, workflowExecution) + for _, event := range events { + if event.GetEventType() == types.EventTypeDecisionTaskTimedOut { + s.Equal(types.TimeoutTypeScheduleToStart, event.DecisionTaskTimedOutEventAttributes.GetTimeoutType()) + stickyTimeout = true + break WaitForStickyTimeoutLoop + } + } + time.Sleep(time.Second) + } + s.True(stickyTimeout, "Decision not timed out.") + + for i := 0; i < 3; i++ { + _, err = poller.PollAndProcessDecisionTaskWithAttempt(true, false, false, true, int64(i)) + s.Logger.Info("PollAndProcessDecisionTask", tag.Error(err)) + s.Nil(err) + } + + err = s.engine.SignalWorkflowExecution(createContext(), &types.SignalWorkflowExecutionRequest{ + Domain: s.domainName, + WorkflowExecution: workflowExecution, + SignalName: "signalB", + Input: []byte("signal input"), + Identity: identity, + RequestID: uuid.New(), + }) + s.Nil(err) + + for i := 0; i < 2; i++ { + _, err = poller.PollAndProcessDecisionTaskWithAttempt(true, false, false, true, int64(i)) + s.Logger.Info("PollAndProcessDecisionTask", tag.Error(err)) + s.Nil(err) + } + + decisionTaskFailed := false + events := s.getHistory(s.domainName, workflowExecution) + for _, event := range events { + if event.GetEventType() == types.EventTypeDecisionTaskFailed { + decisionTaskFailed = true + break + } + } + s.True(decisionTaskFailed) + + // Complete workflow execution + _, err = poller.PollAndProcessDecisionTaskWithAttempt(true, false, false, true, int64(2)) + + // Assert for single decision task failed and workflow completion + failedDecisions := 0 + workflowComplete := false + events = s.getHistory(s.domainName, workflowExecution) + for _, event := range events { + switch event.GetEventType() { + case types.EventTypeDecisionTaskFailed: + failedDecisions++ + case types.EventTypeWorkflowExecutionCompleted: + workflowComplete = true + } + } + s.True(workflowComplete, "Workflow not complete") + s.Equal(2, failedDecisions, "Mismatched failed decision count") +} + +func (s *IntegrationSuite) TestStickyTasklistResetThenTimeout() { + id := "integration-reset-sticky-fire-schedule-to-start-timeout" + wt := "integration-reset-sticky-fire-schedule-to-start-timeout-type" + tl := "integration-reset-sticky-fire-schedule-to-start-timeout-tasklist" + stl := "integration-reset-sticky-fire-schedule-to-start-timeout-tasklist-sticky" + identity := "worker1" + + workflowType := &types.WorkflowType{} + workflowType.Name = wt + + taskList := &types.TaskList{} + taskList.Name = tl + + stickyTaskList := &types.TaskList{} + stickyTaskList.Name = stl + stickyScheduleToStartTimeoutSeconds := common.Int32Ptr(2) + + // Start workflow execution + request := &types.StartWorkflowExecutionRequest{ + RequestID: uuid.New(), + Domain: s.domainName, + WorkflowID: id, + WorkflowType: workflowType, + TaskList: taskList, + Input: nil, + ExecutionStartToCloseTimeoutSeconds: common.Int32Ptr(100), + TaskStartToCloseTimeoutSeconds: common.Int32Ptr(1), + Identity: identity, + } + + we, err0 := s.engine.StartWorkflowExecution(createContext(), request) + s.Nil(err0) + + s.Logger.Info("StartWorkflowExecution", tag.WorkflowRunID(we.RunID)) + workflowExecution := &types.WorkflowExecution{ + WorkflowID: id, + RunID: we.RunID, + } + + // decider logic + localActivityDone := false + failureCount := 5 + dtHandler := func(execution *types.WorkflowExecution, wt *types.WorkflowType, + previousStartedEventID, startedEventID int64, history *types.History) ([]byte, []*types.Decision, error) { + + if !localActivityDone { + localActivityDone = true + + return nil, []*types.Decision{{ + DecisionType: types.DecisionTypeRecordMarker.Ptr(), + RecordMarkerDecisionAttributes: &types.RecordMarkerDecisionAttributes{ + MarkerName: "local activity marker", + Details: []byte("local activity data"), + }, + }}, nil + } + + if failureCount > 0 { + failureCount-- + return nil, nil, errors.New("non deterministic error") + } + + return nil, []*types.Decision{{ + DecisionType: types.DecisionTypeCompleteWorkflowExecution.Ptr(), + CompleteWorkflowExecutionDecisionAttributes: &types.CompleteWorkflowExecutionDecisionAttributes{ + Result: []byte("Done."), + }, + }}, nil + } + + poller := &TaskPoller{ + Engine: s.engine, + Domain: s.domainName, + TaskList: taskList, + Identity: identity, + DecisionHandler: dtHandler, + Logger: s.Logger, + T: s.T(), + StickyTaskList: stickyTaskList, + StickyScheduleToStartTimeoutSeconds: stickyScheduleToStartTimeoutSeconds, + } + + _, err := poller.PollAndProcessDecisionTaskWithAttempt(false, false, false, true, int64(0)) + s.Logger.Info("PollAndProcessDecisionTask: %v", tag.Error(err)) + s.Nil(err) + + err = s.engine.SignalWorkflowExecution(createContext(), &types.SignalWorkflowExecutionRequest{ + Domain: s.domainName, + WorkflowExecution: workflowExecution, + SignalName: "signalA", + Input: []byte("signal input"), + Identity: identity, + RequestID: uuid.New(), + }) + + //Reset sticky tasklist before sticky decision task starts + s.engine.ResetStickyTaskList(createContext(), &types.ResetStickyTaskListRequest{ + Domain: s.domainName, + Execution: workflowExecution, + }) + + // Wait for decision timeout + stickyTimeout := false +WaitForStickyTimeoutLoop: + for i := 0; i < 10; i++ { + events := s.getHistory(s.domainName, workflowExecution) + for _, event := range events { + if event.GetEventType() == types.EventTypeDecisionTaskTimedOut { + s.Equal(types.TimeoutTypeScheduleToStart, event.DecisionTaskTimedOutEventAttributes.GetTimeoutType()) + stickyTimeout = true + break WaitForStickyTimeoutLoop + } + } + time.Sleep(time.Second) + } + s.True(stickyTimeout, "Decision not timed out.") + + for i := 0; i < 3; i++ { + _, err = poller.PollAndProcessDecisionTaskWithAttempt(true, false, false, true, int64(i)) + s.Logger.Info("PollAndProcessDecisionTask: %v", tag.Error(err)) + s.Nil(err) + } + + err = s.engine.SignalWorkflowExecution(createContext(), &types.SignalWorkflowExecutionRequest{ + Domain: s.domainName, + WorkflowExecution: workflowExecution, + SignalName: "signalB", + Input: []byte("signal input"), + Identity: identity, + RequestID: uuid.New(), + }) + s.Nil(err) + + for i := 0; i < 2; i++ { + _, err = poller.PollAndProcessDecisionTaskWithAttempt(true, false, false, true, int64(i)) + s.Logger.Info("PollAndProcessDecisionTask: %v", tag.Error(err)) + s.Nil(err) + } + + decisionTaskFailed := false + events := s.getHistory(s.domainName, workflowExecution) + for _, event := range events { + if event.GetEventType() == types.EventTypeDecisionTaskFailed { + decisionTaskFailed = true + break + } + } + s.True(decisionTaskFailed) + + // Complete workflow execution + _, err = poller.PollAndProcessDecisionTaskWithAttempt(true, false, false, true, int64(2)) + + // Assert for single decision task failed and workflow completion + failedDecisions := 0 + workflowComplete := false + events = s.getHistory(s.domainName, workflowExecution) + for _, event := range events { + switch event.GetEventType() { + case types.EventTypeDecisionTaskFailed: + failedDecisions++ + case types.EventTypeWorkflowExecutionCompleted: + workflowComplete = true + } + } + s.True(workflowComplete, "Workflow not complete") + s.Equal(2, failedDecisions, "Mismatched failed decision count") +} + +func (s *IntegrationSuite) TestBufferedEventsOutOfOrder() { + id := "integration-buffered-events-out-of-order-test" + wt := "integration-buffered-events-out-of-order-test-type" + tl := "integration-buffered-events-out-of-order-test-tasklist" + identity := "worker1" + + workflowType := &types.WorkflowType{Name: wt} + taskList := &types.TaskList{Name: tl} + + // Start workflow execution + request := &types.StartWorkflowExecutionRequest{ + RequestID: uuid.New(), + Domain: s.domainName, + WorkflowID: id, + WorkflowType: workflowType, + TaskList: taskList, + Input: nil, + ExecutionStartToCloseTimeoutSeconds: common.Int32Ptr(100), + TaskStartToCloseTimeoutSeconds: common.Int32Ptr(20), + Identity: identity, + } + + we, err0 := s.engine.StartWorkflowExecution(createContext(), request) + s.Nil(err0) + + s.Logger.Info("StartWorkflowExecution", tag.WorkflowRunID(we.RunID)) + workflowExecution := &types.WorkflowExecution{ + WorkflowID: id, + RunID: we.RunID, + } + + // decider logic + workflowComplete := false + firstDecision := false + secondDecision := false + dtHandler := func(execution *types.WorkflowExecution, wt *types.WorkflowType, + previousStartedEventID, startedEventID int64, history *types.History) ([]byte, []*types.Decision, error) { + + s.Logger.Info(fmt.Sprintf("Decider called: first: %v, second: %v, complete: %v\n", firstDecision, secondDecision, workflowComplete)) + + if !firstDecision { + firstDecision = true + return nil, []*types.Decision{{ + DecisionType: types.DecisionTypeRecordMarker.Ptr(), + RecordMarkerDecisionAttributes: &types.RecordMarkerDecisionAttributes{ + MarkerName: "some random marker name", + Details: []byte("some random marker details"), + }, + }, { + DecisionType: types.DecisionTypeScheduleActivityTask.Ptr(), + ScheduleActivityTaskDecisionAttributes: &types.ScheduleActivityTaskDecisionAttributes{ + ActivityID: "Activity-1", + ActivityType: &types.ActivityType{Name: "ActivityType"}, + Domain: s.domainName, + TaskList: taskList, + Input: []byte("some random activity input"), + ScheduleToCloseTimeoutSeconds: common.Int32Ptr(100), + ScheduleToStartTimeoutSeconds: common.Int32Ptr(100), + StartToCloseTimeoutSeconds: common.Int32Ptr(100), + HeartbeatTimeoutSeconds: common.Int32Ptr(100), + }, + }}, nil + } + + if !secondDecision { + secondDecision = true + return nil, []*types.Decision{{ + DecisionType: types.DecisionTypeRecordMarker.Ptr(), + RecordMarkerDecisionAttributes: &types.RecordMarkerDecisionAttributes{ + MarkerName: "some random marker name", + Details: []byte("some random marker details"), + }, + }}, nil + } + + workflowComplete = true + return nil, []*types.Decision{{ + DecisionType: types.DecisionTypeCompleteWorkflowExecution.Ptr(), + CompleteWorkflowExecutionDecisionAttributes: &types.CompleteWorkflowExecutionDecisionAttributes{ + Result: []byte("Done."), + }, + }}, nil + } + // activity handler + atHandler := func(execution *types.WorkflowExecution, activityType *types.ActivityType, + ActivityID string, input []byte, taskToken []byte) ([]byte, bool, error) { + return []byte("Activity Result."), false, nil + } + + poller := &TaskPoller{ + Engine: s.engine, + Domain: s.domainName, + TaskList: taskList, + Identity: identity, + DecisionHandler: dtHandler, + ActivityHandler: atHandler, + Logger: s.Logger, + T: s.T(), + } + + // first decision, which will schedule an activity and add marker + _, task, err := poller.PollAndProcessDecisionTaskWithAttemptAndRetryAndForceNewDecision( + true, + false, + false, + false, + int64(0), + 1, + true, + nil) + s.Logger.Info("pollAndProcessDecisionTask", tag.Error(err)) + s.Nil(err) + + // This will cause activity start and complete to be buffered + err = poller.PollAndProcessActivityTask(false) + s.Logger.Info("pollAndProcessActivityTask", tag.Error(err)) + s.Nil(err) + + // second decision, completes another local activity and forces flush of buffered activity events + newDecisionTask := task.GetDecisionTask() + s.NotNil(newDecisionTask) + task, err = poller.HandlePartialDecision(newDecisionTask) + s.Logger.Info("pollAndProcessDecisionTask", tag.Error(err)) + s.Nil(err) + s.NotNil(task) + + // third decision, which will close workflow + newDecisionTask = task.GetDecisionTask() + s.NotNil(newDecisionTask) + task, err = poller.HandlePartialDecision(newDecisionTask) + s.Logger.Info("pollAndProcessDecisionTask", tag.Error(err)) + s.Nil(err) + s.Nil(task.DecisionTask) + + events := s.getHistory(s.domainName, workflowExecution) + var scheduleEvent, startedEvent, completedEvent *types.HistoryEvent + for _, event := range events { + switch event.GetEventType() { + case types.EventTypeActivityTaskScheduled: + scheduleEvent = event + case types.EventTypeActivityTaskStarted: + startedEvent = event + case types.EventTypeActivityTaskCompleted: + completedEvent = event + } + } + + s.NotNil(scheduleEvent) + s.NotNil(startedEvent) + s.NotNil(completedEvent) + s.True(startedEvent.GetEventID() < completedEvent.GetEventID()) + s.Equal(scheduleEvent.GetEventID(), startedEvent.ActivityTaskStartedEventAttributes.GetScheduledEventID()) + s.Equal(scheduleEvent.GetEventID(), completedEvent.ActivityTaskCompletedEventAttributes.GetScheduledEventID()) + s.Equal(startedEvent.GetEventID(), completedEvent.ActivityTaskCompletedEventAttributes.GetStartedEventID()) + s.True(workflowComplete) +} + +type startFunc func() (*types.StartWorkflowExecutionResponse, error) + +func (s *IntegrationSuite) TestStartWithMemo() { + id := "integration-start-with-memo-test" + wt := "integration-start-with-memo-test-type" + tl := "integration-start-with-memo-test-tasklist" + identity := "worker1" + + workflowType := &types.WorkflowType{} + workflowType.Name = wt + + taskList := &types.TaskList{} + taskList.Name = tl + + memoInfo, _ := json.Marshal(id) + memo := &types.Memo{ + Fields: map[string][]byte{ + "Info": memoInfo, + }, + } + + request := &types.StartWorkflowExecutionRequest{ + RequestID: uuid.New(), + Domain: s.domainName, + WorkflowID: id, + WorkflowType: workflowType, + TaskList: taskList, + Input: nil, + ExecutionStartToCloseTimeoutSeconds: common.Int32Ptr(100), + TaskStartToCloseTimeoutSeconds: common.Int32Ptr(1), + Identity: identity, + Memo: memo, + } + + fn := func() (*types.StartWorkflowExecutionResponse, error) { + return s.engine.StartWorkflowExecution(createContext(), request) + } + s.startWithMemoHelper(fn, id, taskList, memo) +} + +func (s *IntegrationSuite) TestSignalWithStartWithMemo() { + id := "integration-signal-with-start-with-memo-test" + wt := "integration-signal-with-start-with-memo-test-type" + tl := "integration-signal-with-start-with-memo-test-tasklist" + identity := "worker1" + + workflowType := &types.WorkflowType{} + workflowType.Name = wt + + taskList := &types.TaskList{} + taskList.Name = tl + + memoInfo, _ := json.Marshal(id) + memo := &types.Memo{ + Fields: map[string][]byte{ + "Info": memoInfo, + }, + } + + signalName := "my signal" + signalInput := []byte("my signal input.") + request := &types.SignalWithStartWorkflowExecutionRequest{ + RequestID: uuid.New(), + Domain: s.domainName, + WorkflowID: id, + WorkflowType: workflowType, + TaskList: taskList, + Input: nil, + ExecutionStartToCloseTimeoutSeconds: common.Int32Ptr(100), + TaskStartToCloseTimeoutSeconds: common.Int32Ptr(1), + SignalName: signalName, + SignalInput: signalInput, + Identity: identity, + Memo: memo, + } + + fn := func() (*types.StartWorkflowExecutionResponse, error) { + return s.engine.SignalWithStartWorkflowExecution(createContext(), request) + } + s.startWithMemoHelper(fn, id, taskList, memo) +} + +func (s *IntegrationSuite) TestCancelTimer() { + id := "integration-cancel-timer-test" + wt := "integration-cancel-timer-test-type" + tl := "integration-cancel-timer-test-tasklist" + identity := "worker1" + + workflowType := &types.WorkflowType{} + workflowType.Name = wt + + taskList := &types.TaskList{} + taskList.Name = tl + + request := &types.StartWorkflowExecutionRequest{ + RequestID: uuid.New(), + Domain: s.domainName, + WorkflowID: id, + WorkflowType: workflowType, + TaskList: taskList, + Input: nil, + ExecutionStartToCloseTimeoutSeconds: common.Int32Ptr(100), + TaskStartToCloseTimeoutSeconds: common.Int32Ptr(1000), + Identity: identity, + } + + creatResp, err0 := s.engine.StartWorkflowExecution(createContext(), request) + s.Nil(err0) + workflowExecution := &types.WorkflowExecution{ + WorkflowID: id, + RunID: creatResp.GetRunID(), + } + + TimerID := "1" + timerScheduled := false + signalDelivered := false + timerCancelled := false + workflowComplete := false + timer := int64(2000) + dtHandler := func(execution *types.WorkflowExecution, wt *types.WorkflowType, + previousStartedEventID, startedEventID int64, history *types.History) ([]byte, []*types.Decision, error) { + + if !timerScheduled { + timerScheduled = true + return nil, []*types.Decision{{ + DecisionType: types.DecisionTypeStartTimer.Ptr(), + StartTimerDecisionAttributes: &types.StartTimerDecisionAttributes{ + TimerID: TimerID, + StartToFireTimeoutSeconds: common.Int64Ptr(timer), + }, + }}, nil + } + + resp, err := s.engine.GetWorkflowExecutionHistory(createContext(), &types.GetWorkflowExecutionHistoryRequest{ + Domain: s.domainName, + Execution: workflowExecution, + MaximumPageSize: 200, + }) + s.Nil(err) + for _, event := range resp.History.Events { + switch event.GetEventType() { + case types.EventTypeWorkflowExecutionSignaled: + signalDelivered = true + case types.EventTypeTimerCanceled: + timerCancelled = true + } + } + + if !signalDelivered { + s.Fail("should receive a signal") + } + + if !timerCancelled { + return nil, []*types.Decision{{ + DecisionType: types.DecisionTypeCancelTimer.Ptr(), + CancelTimerDecisionAttributes: &types.CancelTimerDecisionAttributes{ + TimerID: TimerID, + }, + }}, nil + } + + workflowComplete = true + return nil, []*types.Decision{{ + DecisionType: types.DecisionTypeCompleteWorkflowExecution.Ptr(), + CompleteWorkflowExecutionDecisionAttributes: &types.CompleteWorkflowExecutionDecisionAttributes{ + Result: []byte("Done."), + }, + }}, nil + } + + poller := &TaskPoller{ + Engine: s.engine, + Domain: s.domainName, + TaskList: taskList, + Identity: identity, + DecisionHandler: dtHandler, + ActivityHandler: nil, + Logger: s.Logger, + T: s.T(), + } + + // schedule the timer + _, err := poller.PollAndProcessDecisionTask(false, false) + s.Logger.Info("PollAndProcessDecisionTask: completed") + s.Nil(err) + + s.Nil(s.sendSignal(s.domainName, workflowExecution, "random signal name", []byte("random signal payload"), identity)) + + // receive the signal & cancel the timer + _, err = poller.PollAndProcessDecisionTask(false, false) + s.Logger.Info("PollAndProcessDecisionTask: completed") + s.Nil(err) + + s.Nil(s.sendSignal(s.domainName, workflowExecution, "random signal name", []byte("random signal payload"), identity)) + // complete the workflow + _, err = poller.PollAndProcessDecisionTask(false, false) + s.Logger.Info("PollAndProcessDecisionTask: completed") + s.Nil(err) + + s.True(workflowComplete) + + resp, err := s.engine.GetWorkflowExecutionHistory(createContext(), &types.GetWorkflowExecutionHistoryRequest{ + Domain: s.domainName, + Execution: workflowExecution, + MaximumPageSize: 200, + }) + s.Nil(err) + for _, event := range resp.History.Events { + switch event.GetEventType() { + case types.EventTypeWorkflowExecutionSignaled: + signalDelivered = true + case types.EventTypeTimerCanceled: + timerCancelled = true + case types.EventTypeTimerFired: + s.Fail("timer got fired") + } + } +} + +func (s *IntegrationSuite) TestCancelTimer_CancelFiredAndBuffered() { + id := "integration-cancel-timer-fired-and-buffered-test" + wt := "integration-cancel-timer-fired-and-buffered-test-type" + tl := "integration-cancel-timer-fired-and-buffered-test-tasklist" + identity := "worker1" + + workflowType := &types.WorkflowType{} + workflowType.Name = wt + + taskList := &types.TaskList{} + taskList.Name = tl + + request := &types.StartWorkflowExecutionRequest{ + RequestID: uuid.New(), + Domain: s.domainName, + WorkflowID: id, + WorkflowType: workflowType, + TaskList: taskList, + Input: nil, + ExecutionStartToCloseTimeoutSeconds: common.Int32Ptr(100), + TaskStartToCloseTimeoutSeconds: common.Int32Ptr(1000), + Identity: identity, + } + + creatResp, err0 := s.engine.StartWorkflowExecution(createContext(), request) + s.Nil(err0) + workflowExecution := &types.WorkflowExecution{ + WorkflowID: id, + RunID: creatResp.GetRunID(), + } + + TimerID := 1 + timerScheduled := false + signalDelivered := false + timerCancelled := false + workflowComplete := false + timer := int64(4) + dtHandler := func(execution *types.WorkflowExecution, wt *types.WorkflowType, + previousStartedEventID, startedEventID int64, history *types.History) ([]byte, []*types.Decision, error) { + + if !timerScheduled { + timerScheduled = true + return nil, []*types.Decision{{ + DecisionType: types.DecisionTypeStartTimer.Ptr(), + StartTimerDecisionAttributes: &types.StartTimerDecisionAttributes{ + TimerID: fmt.Sprintf("%v", TimerID), + StartToFireTimeoutSeconds: common.Int64Ptr(timer), + }, + }}, nil + } + + resp, err := s.engine.GetWorkflowExecutionHistory(createContext(), &types.GetWorkflowExecutionHistoryRequest{ + Domain: s.domainName, + Execution: workflowExecution, + MaximumPageSize: 200, + }) + s.Nil(err) + for _, event := range resp.History.Events { + switch event.GetEventType() { + case types.EventTypeWorkflowExecutionSignaled: + signalDelivered = true + case types.EventTypeTimerCanceled: + timerCancelled = true + } + } + + if !signalDelivered { + s.Fail("should receive a signal") + } + + if !timerCancelled { + time.Sleep(time.Duration(2*timer) * time.Second) + return nil, []*types.Decision{{ + DecisionType: types.DecisionTypeCancelTimer.Ptr(), + CancelTimerDecisionAttributes: &types.CancelTimerDecisionAttributes{ + TimerID: fmt.Sprintf("%v", TimerID), + }, + }}, nil + } + + workflowComplete = true + return nil, []*types.Decision{{ + DecisionType: types.DecisionTypeCompleteWorkflowExecution.Ptr(), + CompleteWorkflowExecutionDecisionAttributes: &types.CompleteWorkflowExecutionDecisionAttributes{ + Result: []byte("Done."), + }, + }}, nil + } + + poller := &TaskPoller{ + Engine: s.engine, + Domain: s.domainName, + TaskList: taskList, + Identity: identity, + DecisionHandler: dtHandler, + ActivityHandler: nil, + Logger: s.Logger, + T: s.T(), + } + + // schedule the timer + _, err := poller.PollAndProcessDecisionTask(false, false) + s.Logger.Info("PollAndProcessDecisionTask: completed") + s.Nil(err) + + s.Nil(s.sendSignal(s.domainName, workflowExecution, "random signal name", []byte("random signal payload"), identity)) + + // receive the signal & cancel the timer + _, err = poller.PollAndProcessDecisionTask(false, false) + s.Logger.Info("PollAndProcessDecisionTask: completed") + s.Nil(err) + + s.Nil(s.sendSignal(s.domainName, workflowExecution, "random signal name", []byte("random signal payload"), identity)) + // complete the workflow + _, err = poller.PollAndProcessDecisionTask(false, false) + s.Logger.Info("PollAndProcessDecisionTask: completed") + s.Nil(err) + + s.True(workflowComplete) + + resp, err := s.engine.GetWorkflowExecutionHistory(createContext(), &types.GetWorkflowExecutionHistoryRequest{ + Domain: s.domainName, + Execution: workflowExecution, + MaximumPageSize: 200, + }) + s.Nil(err) + for _, event := range resp.History.Events { + switch event.GetEventType() { + case types.EventTypeWorkflowExecutionSignaled: + signalDelivered = true + case types.EventTypeTimerCanceled: + timerCancelled = true + case types.EventTypeTimerFired: + s.Fail("timer got fired") + } + } +} + +// helper function for TestStartWithMemo and TestSignalWithStartWithMemo to reduce duplicate code +func (s *IntegrationSuite) startWithMemoHelper(startFn startFunc, id string, taskList *types.TaskList, memo *types.Memo) { + identity := "worker1" + + we, err0 := startFn() + s.Nil(err0) + + s.Logger.Info("StartWorkflowExecution: response", tag.WorkflowRunID(we.RunID)) + + dtHandler := func(execution *types.WorkflowExecution, wt *types.WorkflowType, + previousStartedEventID, startedEventID int64, history *types.History) ([]byte, []*types.Decision, error) { + return []byte(strconv.Itoa(1)), []*types.Decision{{ + DecisionType: types.DecisionTypeCompleteWorkflowExecution.Ptr(), + CompleteWorkflowExecutionDecisionAttributes: &types.CompleteWorkflowExecutionDecisionAttributes{ + Result: []byte("Done."), + }, + }}, nil + } + + poller := &TaskPoller{ + Engine: s.engine, + Domain: s.domainName, + TaskList: taskList, + Identity: identity, + DecisionHandler: dtHandler, + Logger: s.Logger, + T: s.T(), + } + + // verify open visibility + var openExecutionInfo *types.WorkflowExecutionInfo + for i := 0; i < 10; i++ { + resp, err1 := s.engine.ListOpenWorkflowExecutions(createContext(), &types.ListOpenWorkflowExecutionsRequest{ + Domain: s.domainName, + MaximumPageSize: 100, + StartTimeFilter: &types.StartTimeFilter{ + EarliestTime: common.Int64Ptr(0), + LatestTime: common.Int64Ptr(time.Now().UnixNano()), + }, + ExecutionFilter: &types.WorkflowExecutionFilter{ + WorkflowID: id, + }, + }) + s.Nil(err1) + if len(resp.Executions) == 1 { + openExecutionInfo = resp.Executions[0] + break + } + s.Logger.Info("Open WorkflowExecution is not yet visible") + time.Sleep(100 * time.Millisecond) + } + s.NotNil(openExecutionInfo) + s.Equal(memo, openExecutionInfo.Memo) + + // make progress of workflow + _, err := poller.PollAndProcessDecisionTask(false, false) + s.Logger.Info("PollAndProcessDecisionTask: %v", tag.Error(err)) + s.Nil(err) + + // verify history + execution := &types.WorkflowExecution{ + WorkflowID: id, + RunID: we.RunID, + } + historyResponse, historyErr := s.engine.GetWorkflowExecutionHistory(createContext(), &types.GetWorkflowExecutionHistoryRequest{ + Domain: s.domainName, + Execution: execution, + }) + s.Nil(historyErr) + history := historyResponse.History + firstEvent := history.Events[0] + s.Equal(types.EventTypeWorkflowExecutionStarted, firstEvent.GetEventType()) + startdEventAttributes := firstEvent.WorkflowExecutionStartedEventAttributes + s.Equal(memo, startdEventAttributes.Memo) + + // verify DescribeWorkflowExecution result + descRequest := &types.DescribeWorkflowExecutionRequest{ + Domain: s.domainName, + Execution: execution, + } + descResp, err := s.engine.DescribeWorkflowExecution(createContext(), descRequest) + s.Nil(err) + s.Equal(memo, descResp.WorkflowExecutionInfo.Memo) + + // verify closed visibility + var closedExecutionInfo *types.WorkflowExecutionInfo + for i := 0; i < 10; i++ { + resp, err1 := s.engine.ListClosedWorkflowExecutions(createContext(), &types.ListClosedWorkflowExecutionsRequest{ + Domain: s.domainName, + MaximumPageSize: 100, + StartTimeFilter: &types.StartTimeFilter{ + EarliestTime: common.Int64Ptr(0), + LatestTime: common.Int64Ptr(time.Now().UnixNano()), + }, + ExecutionFilter: &types.WorkflowExecutionFilter{ + WorkflowID: id, + }, + }) + s.Nil(err1) + if len(resp.Executions) == 1 { + closedExecutionInfo = resp.Executions[0] + break + } + s.Logger.Info("Closed WorkflowExecution is not yet visible") + time.Sleep(100 * time.Millisecond) + } + s.NotNil(closedExecutionInfo) + s.Equal(memo, closedExecutionInfo.Memo) +} + +func (s *IntegrationSuite) sendSignal(domainName string, execution *types.WorkflowExecution, signalName string, + input []byte, identity string) error { + return s.engine.SignalWorkflowExecution(createContext(), &types.SignalWorkflowExecutionRequest{ + Domain: domainName, + WorkflowExecution: execution, + SignalName: signalName, + Input: input, + Identity: identity, + }) +} diff --git a/host/integration_test.go b/host/integration_test.go index 357fc3947cf..4a0a556e0d7 100644 --- a/host/integration_test.go +++ b/host/integration_test.go @@ -21,53 +21,12 @@ package host import ( - "bytes" - "encoding/binary" - "encoding/json" - "errors" "flag" - "fmt" - "math" - "sort" - "strconv" "testing" - "time" - "github.com/pborman/uuid" - log "github.com/sirupsen/logrus" - "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" - - "github.com/uber/cadence/common" - "github.com/uber/cadence/common/log/tag" - "github.com/uber/cadence/common/types" - cadencehistory "github.com/uber/cadence/service/history" - "github.com/uber/cadence/service/history/execution" - "github.com/uber/cadence/service/matching" -) - -type ( - IntegrationSuite struct { - // override suite.Suite.Assertions with require.Assertions; this means that s.NotNil(nil) will stop the test, - // not merely log an error - *require.Assertions - IntegrationBase - } ) -func (s *IntegrationSuite) SetupSuite() { - s.setupSuite() -} - -func (s *IntegrationSuite) TearDownSuite() { - s.tearDownSuite() -} - -func (s *IntegrationSuite) SetupTest() { - // Have to define our overridden assertions in the test setup. If we did it earlier, s.T() will return nil - s.Assertions = require.New(s.T()) -} - func TestIntegrationSuite(t *testing.T) { flag.Parse() @@ -87,4013 +46,3 @@ func TestIntegrationSuite(t *testing.T) { suite.Run(t, s) } -func (s *IntegrationSuite) TestStartWorkflowExecution() { - id := "integration-start-workflow-test" - wt := "integration-start-workflow-test-type" - tl := "integration-start-workflow-test-tasklist" - identity := "worker1" - - workflowType := &types.WorkflowType{} - workflowType.Name = wt - - taskList := &types.TaskList{} - taskList.Name = tl - - request := &types.StartWorkflowExecutionRequest{ - RequestID: uuid.New(), - Domain: s.domainName, - WorkflowID: id, - WorkflowType: workflowType, - TaskList: taskList, - Input: nil, - ExecutionStartToCloseTimeoutSeconds: common.Int32Ptr(100), - TaskStartToCloseTimeoutSeconds: common.Int32Ptr(1), - Identity: identity, - } - - we0, err0 := s.engine.StartWorkflowExecution(createContext(), request) - s.Nil(err0) - - we1, err1 := s.engine.StartWorkflowExecution(createContext(), request) - s.Nil(err1) - s.Equal(we0.RunID, we1.RunID) - - newRequest := &types.StartWorkflowExecutionRequest{ - RequestID: uuid.New(), - Domain: s.domainName, - WorkflowID: id, - WorkflowType: workflowType, - TaskList: taskList, - Input: nil, - ExecutionStartToCloseTimeoutSeconds: common.Int32Ptr(100), - TaskStartToCloseTimeoutSeconds: common.Int32Ptr(1), - Identity: identity, - } - we2, err2 := s.engine.StartWorkflowExecution(createContext(), newRequest) - s.NotNil(err2) - s.IsType(&types.WorkflowExecutionAlreadyStartedError{}, err2) - log.Infof("Unable to start workflow execution: %v", err2.Error()) - s.Nil(we2) -} - -func (s *IntegrationSuite) TestStartWorkflowExecution_IDReusePolicy() { - id := "integration-start-workflow-id-reuse-test" - wt := "integration-start-workflow-id-reuse-type" - tl := "integration-start-workflow-id-reuse-tasklist" - identity := "worker1" - - workflowType := &types.WorkflowType{} - workflowType.Name = wt - - taskList := &types.TaskList{} - taskList.Name = tl - - createStartRequest := func(policy types.WorkflowIDReusePolicy) *types.StartWorkflowExecutionRequest { - return &types.StartWorkflowExecutionRequest{ - RequestID: uuid.New(), - Domain: s.domainName, - WorkflowID: id, - WorkflowType: workflowType, - TaskList: taskList, - Input: nil, - ExecutionStartToCloseTimeoutSeconds: common.Int32Ptr(100), - TaskStartToCloseTimeoutSeconds: common.Int32Ptr(1), - Identity: identity, - WorkflowIDReusePolicy: &policy, - } - } - - request := createStartRequest(types.WorkflowIDReusePolicyAllowDuplicateFailedOnly) - we, err := s.engine.StartWorkflowExecution(createContext(), request) - s.Nil(err) - - // Test policies when workflow is running - policies := []types.WorkflowIDReusePolicy{ - types.WorkflowIDReusePolicyAllowDuplicateFailedOnly, - types.WorkflowIDReusePolicyAllowDuplicate, - types.WorkflowIDReusePolicyRejectDuplicate, - } - for _, policy := range policies { - newRequest := createStartRequest(policy) - we1, err1 := s.engine.StartWorkflowExecution(createContext(), newRequest) - s.Error(err1) - s.IsType(&types.WorkflowExecutionAlreadyStartedError{}, err1) - s.Nil(we1) - } - - // Test TerminateIfRunning policy when workflow is running - policy := types.WorkflowIDReusePolicyTerminateIfRunning - newRequest := createStartRequest(policy) - we1, err1 := s.engine.StartWorkflowExecution(createContext(), newRequest) - s.NoError(err1) - s.NotEqual(we.GetRunID(), we1.GetRunID()) - // verify terminate status - executionTerminated := false -GetHistoryLoop: - for i := 0; i < 10; i++ { - historyResponse, err := s.engine.GetWorkflowExecutionHistory(createContext(), &types.GetWorkflowExecutionHistoryRequest{ - Domain: s.domainName, - Execution: &types.WorkflowExecution{ - WorkflowID: id, - RunID: we.RunID, - }, - }) - s.Nil(err) - history := historyResponse.History - - lastEvent := history.Events[len(history.Events)-1] - if lastEvent.GetEventType() != types.EventTypeWorkflowExecutionTerminated { - s.Logger.Warn("Execution not terminated yet.") - time.Sleep(100 * time.Millisecond) - continue GetHistoryLoop - } - - terminateEventAttributes := lastEvent.WorkflowExecutionTerminatedEventAttributes - s.Equal(cadencehistory.TerminateIfRunningReason, terminateEventAttributes.GetReason()) - s.Equal(fmt.Sprintf(cadencehistory.TerminateIfRunningDetailsTemplate, we1.GetRunID()), string(terminateEventAttributes.Details)) - s.Equal(execution.IdentityHistoryService, terminateEventAttributes.GetIdentity()) - executionTerminated = true - break GetHistoryLoop - } - s.True(executionTerminated) - - // Terminate current workflow execution - err = s.engine.TerminateWorkflowExecution(createContext(), &types.TerminateWorkflowExecutionRequest{ - Domain: s.domainName, - WorkflowExecution: &types.WorkflowExecution{ - WorkflowID: id, - RunID: we1.RunID, - }, - Reason: "kill workflow", - Identity: identity, - }) - s.Nil(err) - - // test policy AllowDuplicateFailedOnly - policy = types.WorkflowIDReusePolicyAllowDuplicateFailedOnly - newRequest = createStartRequest(policy) - we2, err2 := s.engine.StartWorkflowExecution(createContext(), newRequest) - s.NoError(err2) - s.NotEqual(we1.GetRunID(), we2.GetRunID()) - // complete workflow instead of terminate - dtHandler := func(execution *types.WorkflowExecution, wt *types.WorkflowType, - previousStartedEventID, startedEventID int64, history *types.History) ([]byte, []*types.Decision, error) { - return []byte(strconv.Itoa(0)), []*types.Decision{{ - DecisionType: types.DecisionTypeCompleteWorkflowExecution.Ptr(), - CompleteWorkflowExecutionDecisionAttributes: &types.CompleteWorkflowExecutionDecisionAttributes{ - Result: []byte("Done."), - }, - }}, nil - } - poller := &TaskPoller{ - Engine: s.engine, - Domain: s.domainName, - TaskList: taskList, - Identity: identity, - DecisionHandler: dtHandler, - Logger: s.Logger, - T: s.T(), - } - _, err = poller.PollAndProcessDecisionTask(false, false) - s.Logger.Info("PollAndProcessDecisionTask", tag.Error(err)) - s.Nil(err) - // duplicate requests - we3, err3 := s.engine.StartWorkflowExecution(createContext(), newRequest) - s.NoError(err3) - s.Equal(we2.GetRunID(), we3.GetRunID()) - // new request, same policy - newRequest = createStartRequest(policy) - we3, err3 = s.engine.StartWorkflowExecution(createContext(), newRequest) - s.Error(err3) - s.IsType(&types.WorkflowExecutionAlreadyStartedError{}, err3) - s.Nil(we3) - - // test policy RejectDuplicate - policy = types.WorkflowIDReusePolicyRejectDuplicate - newRequest = createStartRequest(policy) - we3, err3 = s.engine.StartWorkflowExecution(createContext(), newRequest) - s.Error(err3) - s.IsType(&types.WorkflowExecutionAlreadyStartedError{}, err3) - s.Nil(we3) - - // test policy AllowDuplicate - policy = types.WorkflowIDReusePolicyAllowDuplicate - newRequest = createStartRequest(policy) - we4, err4 := s.engine.StartWorkflowExecution(createContext(), newRequest) - s.NoError(err4) - s.NotEqual(we3.GetRunID(), we4.GetRunID()) - - // complete workflow - _, err = poller.PollAndProcessDecisionTask(false, false) - s.Logger.Info("PollAndProcessDecisionTask", tag.Error(err)) - s.Nil(err) - - // test policy TerminateIfRunning - policy = types.WorkflowIDReusePolicyTerminateIfRunning - newRequest = createStartRequest(policy) - we5, err5 := s.engine.StartWorkflowExecution(createContext(), newRequest) - s.NoError(err5) - s.NotEqual(we4.GetRunID(), we5.GetRunID()) -} - -func (s *IntegrationSuite) TestTerminateWorkflow() { - id := "integration-terminate-workflow-test" - wt := "integration-terminate-workflow-test-type" - tl := "integration-terminate-workflow-test-tasklist" - identity := "worker1" - activityName := "activity_type1" - - workflowType := &types.WorkflowType{} - workflowType.Name = wt - - taskList := &types.TaskList{} - taskList.Name = tl - - request := &types.StartWorkflowExecutionRequest{ - RequestID: uuid.New(), - Domain: s.domainName, - WorkflowID: id, - WorkflowType: workflowType, - TaskList: taskList, - Input: nil, - ExecutionStartToCloseTimeoutSeconds: common.Int32Ptr(100), - TaskStartToCloseTimeoutSeconds: common.Int32Ptr(1), - Identity: identity, - } - - we, err0 := s.engine.StartWorkflowExecution(createContext(), request) - s.Nil(err0) - - s.Logger.Info("StartWorkflowExecution", tag.WorkflowRunID(we.RunID)) - - activityCount := int32(1) - activityCounter := int32(0) - dtHandler := func(execution *types.WorkflowExecution, wt *types.WorkflowType, - previousStartedEventID, startedEventID int64, history *types.History) ([]byte, []*types.Decision, error) { - if activityCounter < activityCount { - activityCounter++ - buf := new(bytes.Buffer) - s.Nil(binary.Write(buf, binary.LittleEndian, activityCounter)) - - return []byte(strconv.Itoa(int(activityCounter))), []*types.Decision{{ - DecisionType: types.DecisionTypeScheduleActivityTask.Ptr(), - ScheduleActivityTaskDecisionAttributes: &types.ScheduleActivityTaskDecisionAttributes{ - ActivityID: strconv.Itoa(int(activityCounter)), - ActivityType: &types.ActivityType{Name: activityName}, - TaskList: &types.TaskList{Name: tl}, - Input: buf.Bytes(), - ScheduleToCloseTimeoutSeconds: common.Int32Ptr(100), - ScheduleToStartTimeoutSeconds: common.Int32Ptr(10), - StartToCloseTimeoutSeconds: common.Int32Ptr(50), - HeartbeatTimeoutSeconds: common.Int32Ptr(5), - }, - }}, nil - } - - return []byte(strconv.Itoa(int(activityCounter))), []*types.Decision{{ - DecisionType: types.DecisionTypeCompleteWorkflowExecution.Ptr(), - CompleteWorkflowExecutionDecisionAttributes: &types.CompleteWorkflowExecutionDecisionAttributes{ - Result: []byte("Done."), - }, - }}, nil - } - - atHandler := func(execution *types.WorkflowExecution, activityType *types.ActivityType, - ActivityID string, input []byte, taskToken []byte) ([]byte, bool, error) { - - return []byte("Activity Result."), false, nil - } - - poller := &TaskPoller{ - Engine: s.engine, - Domain: s.domainName, - TaskList: taskList, - Identity: identity, - DecisionHandler: dtHandler, - ActivityHandler: atHandler, - Logger: s.Logger, - T: s.T(), - } - - _, err := poller.PollAndProcessDecisionTask(false, false) - s.Logger.Info("PollAndProcessDecisionTask", tag.Error(err)) - s.Nil(err) - - terminateReason := "terminate reason." - terminateDetails := []byte("terminate details.") - err = s.engine.TerminateWorkflowExecution(createContext(), &types.TerminateWorkflowExecutionRequest{ - Domain: s.domainName, - WorkflowExecution: &types.WorkflowExecution{ - WorkflowID: id, - RunID: we.RunID, - }, - Reason: terminateReason, - Details: terminateDetails, - Identity: identity, - }) - s.Nil(err) - - executionTerminated := false -GetHistoryLoop: - for i := 0; i < 10; i++ { - historyResponse, err := s.engine.GetWorkflowExecutionHistory(createContext(), &types.GetWorkflowExecutionHistoryRequest{ - Domain: s.domainName, - Execution: &types.WorkflowExecution{ - WorkflowID: id, - RunID: we.RunID, - }, - }) - s.Nil(err) - history := historyResponse.History - - lastEvent := history.Events[len(history.Events)-1] - if *lastEvent.EventType != types.EventTypeWorkflowExecutionTerminated { - s.Logger.Warn("Execution not terminated yet.") - time.Sleep(100 * time.Millisecond) - continue GetHistoryLoop - } - - terminateEventAttributes := lastEvent.WorkflowExecutionTerminatedEventAttributes - s.Equal(terminateReason, terminateEventAttributes.Reason) - s.Equal(terminateDetails, terminateEventAttributes.Details) - s.Equal(identity, terminateEventAttributes.Identity) - executionTerminated = true - break GetHistoryLoop - } - - s.True(executionTerminated) - - newExecutionStarted := false -StartNewExecutionLoop: - for i := 0; i < 10; i++ { - request := &types.StartWorkflowExecutionRequest{ - RequestID: uuid.New(), - Domain: s.domainName, - WorkflowID: id, - WorkflowType: workflowType, - TaskList: taskList, - Input: nil, - ExecutionStartToCloseTimeoutSeconds: common.Int32Ptr(100), - TaskStartToCloseTimeoutSeconds: common.Int32Ptr(1), - Identity: identity, - } - - newExecution, err := s.engine.StartWorkflowExecution(createContext(), request) - if err != nil { - s.Logger.Warn("Start New Execution failed. Error", tag.Error(err)) - time.Sleep(100 * time.Millisecond) - continue StartNewExecutionLoop - } - - s.Logger.Info("New Execution Started with the same ID", tag.WorkflowID(id), - tag.WorkflowRunID(newExecution.RunID)) - newExecutionStarted = true - break StartNewExecutionLoop - } - - s.True(newExecutionStarted) -} - -func (s *IntegrationSuite) TestSequentialWorkflow() { - id := "integration-sequential-workflow-test" - wt := "integration-sequential-workflow-test-type" - tl := "integration-sequential-workflow-test-tasklist" - identity := "worker1" - activityName := "activity_type1" - - workflowType := &types.WorkflowType{} - workflowType.Name = wt - - taskList := &types.TaskList{} - taskList.Name = tl - - request := &types.StartWorkflowExecutionRequest{ - RequestID: uuid.New(), - Domain: s.domainName, - WorkflowID: id, - WorkflowType: workflowType, - TaskList: taskList, - Input: nil, - ExecutionStartToCloseTimeoutSeconds: common.Int32Ptr(100), - TaskStartToCloseTimeoutSeconds: common.Int32Ptr(1), - Identity: identity, - } - - we, err0 := s.engine.StartWorkflowExecution(createContext(), request) - s.Nil(err0) - - s.Logger.Info("StartWorkflowExecution", tag.WorkflowRunID(we.RunID)) - - workflowComplete := false - activityCount := int32(10) - activityCounter := int32(0) - dtHandler := func(execution *types.WorkflowExecution, wt *types.WorkflowType, - previousStartedEventID, startedEventID int64, history *types.History) ([]byte, []*types.Decision, error) { - if activityCounter < activityCount { - activityCounter++ - buf := new(bytes.Buffer) - s.Nil(binary.Write(buf, binary.LittleEndian, activityCounter)) - - return []byte(strconv.Itoa(int(activityCounter))), []*types.Decision{{ - DecisionType: types.DecisionTypeScheduleActivityTask.Ptr(), - ScheduleActivityTaskDecisionAttributes: &types.ScheduleActivityTaskDecisionAttributes{ - ActivityID: strconv.Itoa(int(activityCounter)), - ActivityType: &types.ActivityType{Name: activityName}, - TaskList: &types.TaskList{Name: tl}, - Input: buf.Bytes(), - ScheduleToCloseTimeoutSeconds: common.Int32Ptr(100), - ScheduleToStartTimeoutSeconds: common.Int32Ptr(10), - StartToCloseTimeoutSeconds: common.Int32Ptr(50), - HeartbeatTimeoutSeconds: common.Int32Ptr(5), - }, - }}, nil - } - - workflowComplete = true - return []byte(strconv.Itoa(int(activityCounter))), []*types.Decision{{ - DecisionType: types.DecisionTypeCompleteWorkflowExecution.Ptr(), - CompleteWorkflowExecutionDecisionAttributes: &types.CompleteWorkflowExecutionDecisionAttributes{ - Result: []byte("Done."), - }, - }}, nil - } - - expectedActivity := int32(1) - atHandler := func(execution *types.WorkflowExecution, activityType *types.ActivityType, - ActivityID string, input []byte, taskToken []byte) ([]byte, bool, error) { - s.Equal(id, execution.WorkflowID) - s.Equal(activityName, activityType.Name) - id, _ := strconv.Atoi(ActivityID) - s.Equal(int(expectedActivity), id) - buf := bytes.NewReader(input) - var in int32 - binary.Read(buf, binary.LittleEndian, &in) - s.Equal(expectedActivity, in) - expectedActivity++ - - return []byte("Activity Result."), false, nil - } - - poller := &TaskPoller{ - Engine: s.engine, - Domain: s.domainName, - TaskList: taskList, - Identity: identity, - DecisionHandler: dtHandler, - ActivityHandler: atHandler, - Logger: s.Logger, - T: s.T(), - } - - for i := 0; i < 10; i++ { - _, err := poller.PollAndProcessDecisionTask(false, false) - s.Logger.Info("PollAndProcessDecisionTask", tag.Error(err)) - s.Nil(err) - if i%2 == 0 { - err = poller.PollAndProcessActivityTask(false) - } else { // just for testing respondActivityTaskCompleteByID - err = poller.PollAndProcessActivityTaskWithID(false) - } - s.Logger.Info("PollAndProcessActivityTask", tag.Error(err)) - s.Nil(err) - } - - s.False(workflowComplete) - _, err := poller.PollAndProcessDecisionTask(false, false) - s.Nil(err) - s.True(workflowComplete) -} - -func (s *IntegrationSuite) TestCompleteDecisionTaskAndCreateNewOne() { - id := "integration-complete-decision-create-new-test" - wt := "integration-complete-decision-create-new-test-type" - tl := "integration-complete-decision-create-new-test-tasklist" - identity := "worker1" - - workflowType := &types.WorkflowType{} - workflowType.Name = wt - - taskList := &types.TaskList{} - taskList.Name = tl - - request := &types.StartWorkflowExecutionRequest{ - RequestID: uuid.New(), - Domain: s.domainName, - WorkflowID: id, - WorkflowType: workflowType, - TaskList: taskList, - Input: nil, - ExecutionStartToCloseTimeoutSeconds: common.Int32Ptr(100), - TaskStartToCloseTimeoutSeconds: common.Int32Ptr(1), - Identity: identity, - } - - we, err0 := s.engine.StartWorkflowExecution(createContext(), request) - s.Nil(err0) - - s.Logger.Info("StartWorkflowExecution", tag.WorkflowRunID(we.RunID)) - - decisionCount := 0 - dtHandler := func(execution *types.WorkflowExecution, wt *types.WorkflowType, - previousStartedEventID, startedEventID int64, history *types.History) ([]byte, []*types.Decision, error) { - - if decisionCount < 2 { - decisionCount++ - return nil, []*types.Decision{{ - DecisionType: types.DecisionTypeRecordMarker.Ptr(), - RecordMarkerDecisionAttributes: &types.RecordMarkerDecisionAttributes{ - MarkerName: "test-marker", - }, - }}, nil - } - - return nil, []*types.Decision{{ - DecisionType: types.DecisionTypeCompleteWorkflowExecution.Ptr(), - CompleteWorkflowExecutionDecisionAttributes: &types.CompleteWorkflowExecutionDecisionAttributes{ - Result: []byte("Done."), - }, - }}, nil - } - - poller := &TaskPoller{ - Engine: s.engine, - Domain: s.domainName, - TaskList: taskList, - StickyTaskList: taskList, - Identity: identity, - DecisionHandler: dtHandler, - Logger: s.Logger, - T: s.T(), - } - - _, newTask, err := poller.PollAndProcessDecisionTaskWithAttemptAndRetryAndForceNewDecision( - false, - false, - true, - true, - int64(0), - 1, - true, - nil) - s.Nil(err) - s.NotNil(newTask) - s.NotNil(newTask.DecisionTask) - - s.Equal(int64(3), newTask.DecisionTask.GetPreviousStartedEventID()) - s.Equal(int64(7), newTask.DecisionTask.GetStartedEventID()) - s.Equal(4, len(newTask.DecisionTask.History.Events)) - s.Equal(types.EventTypeDecisionTaskCompleted, newTask.DecisionTask.History.Events[0].GetEventType()) - s.Equal(types.EventTypeMarkerRecorded, newTask.DecisionTask.History.Events[1].GetEventType()) - s.Equal(types.EventTypeDecisionTaskScheduled, newTask.DecisionTask.History.Events[2].GetEventType()) - s.Equal(types.EventTypeDecisionTaskStarted, newTask.DecisionTask.History.Events[3].GetEventType()) -} - -func (s *IntegrationSuite) TestDecisionAndActivityTimeoutsWorkflow() { - id := "integration-timeouts-workflow-test" - wt := "integration-timeouts-workflow-test-type" - tl := "integration-timeouts-workflow-test-tasklist" - identity := "worker1" - activityName := "activity_timer" - - workflowType := &types.WorkflowType{} - workflowType.Name = wt - - taskList := &types.TaskList{} - taskList.Name = tl - - request := &types.StartWorkflowExecutionRequest{ - RequestID: uuid.New(), - Domain: s.domainName, - WorkflowID: id, - WorkflowType: workflowType, - TaskList: taskList, - Input: nil, - ExecutionStartToCloseTimeoutSeconds: common.Int32Ptr(100), - TaskStartToCloseTimeoutSeconds: common.Int32Ptr(1), - Identity: identity, - } - - we, err0 := s.engine.StartWorkflowExecution(createContext(), request) - s.Nil(err0) - - s.Logger.Info("StartWorkflowExecution", tag.WorkflowRunID(we.RunID)) - - workflowComplete := false - activityCount := int32(4) - activityCounter := int32(0) - - dtHandler := func(execution *types.WorkflowExecution, wt *types.WorkflowType, - previousStartedEventID, startedEventID int64, history *types.History) ([]byte, []*types.Decision, error) { - if activityCounter < activityCount { - activityCounter++ - buf := new(bytes.Buffer) - s.Nil(binary.Write(buf, binary.LittleEndian, activityCounter)) - - return []byte(strconv.Itoa(int(activityCounter))), []*types.Decision{{ - DecisionType: types.DecisionTypeScheduleActivityTask.Ptr(), - ScheduleActivityTaskDecisionAttributes: &types.ScheduleActivityTaskDecisionAttributes{ - ActivityID: strconv.Itoa(int(activityCounter)), - ActivityType: &types.ActivityType{Name: activityName}, - TaskList: &types.TaskList{Name: tl}, - Input: buf.Bytes(), - ScheduleToCloseTimeoutSeconds: common.Int32Ptr(1), - ScheduleToStartTimeoutSeconds: common.Int32Ptr(1), - StartToCloseTimeoutSeconds: common.Int32Ptr(1), - HeartbeatTimeoutSeconds: common.Int32Ptr(1), - }, - }}, nil - } - - s.Logger.Info("Completing types.") - - workflowComplete = true - return []byte(strconv.Itoa(int(activityCounter))), []*types.Decision{{ - DecisionType: types.DecisionTypeCompleteWorkflowExecution.Ptr(), - CompleteWorkflowExecutionDecisionAttributes: &types.CompleteWorkflowExecutionDecisionAttributes{ - Result: []byte("Done."), - }, - }}, nil - } - - atHandler := func(execution *types.WorkflowExecution, activityType *types.ActivityType, - ActivityID string, input []byte, taskToken []byte) ([]byte, bool, error) { - s.Equal(id, execution.WorkflowID) - s.Equal(activityName, activityType.Name) - s.Logger.Info("Activity ID", tag.WorkflowActivityID(ActivityID)) - return []byte("Activity Result."), false, nil - } - - poller := &TaskPoller{ - Engine: s.engine, - Domain: s.domainName, - TaskList: taskList, - Identity: identity, - DecisionHandler: dtHandler, - ActivityHandler: atHandler, - Logger: s.Logger, - T: s.T(), - } - - for i := 0; i < 8; i++ { - dropDecisionTask := (i%2 == 0) - s.Logger.Info("Calling Decision Task", tag.Counter(i)) - var err error - if dropDecisionTask { - _, err = poller.PollAndProcessDecisionTask(false, true) - } else { - _, err = poller.PollAndProcessDecisionTaskWithAttempt(true, false, false, false, int64(1)) - } - if err != nil { - historyResponse, err := s.engine.GetWorkflowExecutionHistory(createContext(), &types.GetWorkflowExecutionHistoryRequest{ - Domain: s.domainName, - Execution: &types.WorkflowExecution{ - WorkflowID: id, - RunID: we.RunID, - }, - }) - s.Nil(err) - history := historyResponse.History - common.PrettyPrintHistory(history, s.Logger) - } - s.True(err == nil || err == matching.ErrNoTasks, "%v", err) - if !dropDecisionTask { - s.Logger.Info("Calling Activity Task: %d", tag.Counter(i)) - err = poller.PollAndProcessActivityTask(i%4 == 0) - s.True(err == nil || err == matching.ErrNoTasks) - } - } - - s.Logger.Info("Waiting for workflow to complete", tag.WorkflowRunID(we.RunID)) - - s.False(workflowComplete) - _, err := poller.PollAndProcessDecisionTask(false, false) - s.Nil(err) - s.True(workflowComplete) -} - -func (s *IntegrationSuite) TestWorkflowRetry() { - id := "integration-wf-retry-test" - wt := "integration-wf-retry-type" - tl := "integration-wf-retry-tasklist" - identity := "worker1" - - workflowType := &types.WorkflowType{} - workflowType.Name = wt - - taskList := &types.TaskList{} - taskList.Name = tl - - initialIntervalInSeconds := 1 - backoffCoefficient := 1.5 - maximumAttempts := 5 - request := &types.StartWorkflowExecutionRequest{ - RequestID: uuid.New(), - Domain: s.domainName, - WorkflowID: id, - WorkflowType: workflowType, - TaskList: taskList, - Input: nil, - ExecutionStartToCloseTimeoutSeconds: common.Int32Ptr(100), - TaskStartToCloseTimeoutSeconds: common.Int32Ptr(1), - Identity: identity, - RetryPolicy: &types.RetryPolicy{ - InitialIntervalInSeconds: int32(initialIntervalInSeconds), - MaximumAttempts: int32(maximumAttempts), - MaximumIntervalInSeconds: 1, - NonRetriableErrorReasons: []string{"bad-bug"}, - BackoffCoefficient: backoffCoefficient, - ExpirationIntervalInSeconds: 100, - }, - } - - we, err0 := s.engine.StartWorkflowExecution(createContext(), request) - s.Nil(err0) - - s.Logger.Info("StartWorkflowExecution", tag.WorkflowRunID(we.RunID)) - - var executions []*types.WorkflowExecution - - attemptCount := 0 - - dtHandler := func(execution *types.WorkflowExecution, wt *types.WorkflowType, - previousStartedEventID, startedEventID int64, history *types.History) ([]byte, []*types.Decision, error) { - executions = append(executions, execution) - attemptCount++ - if attemptCount == maximumAttempts { - return nil, []*types.Decision{ - { - DecisionType: types.DecisionTypeCompleteWorkflowExecution.Ptr(), - CompleteWorkflowExecutionDecisionAttributes: &types.CompleteWorkflowExecutionDecisionAttributes{ - Result: []byte("succeed-after-retry"), - }, - }}, nil - } - return nil, []*types.Decision{ - { - DecisionType: types.DecisionTypeFailWorkflowExecution.Ptr(), - FailWorkflowExecutionDecisionAttributes: &types.FailWorkflowExecutionDecisionAttributes{ - Reason: common.StringPtr("retryable-error"), - Details: nil, - }, - }}, nil - } - - poller := &TaskPoller{ - Engine: s.engine, - Domain: s.domainName, - TaskList: taskList, - Identity: identity, - DecisionHandler: dtHandler, - Logger: s.Logger, - T: s.T(), - } - - describeWorkflowExecution := func(execution *types.WorkflowExecution) (*types.DescribeWorkflowExecutionResponse, error) { - return s.engine.DescribeWorkflowExecution(createContext(), &types.DescribeWorkflowExecutionRequest{ - Domain: s.domainName, - Execution: execution, - }) - } - - for i := 0; i != maximumAttempts; i++ { - _, err := poller.PollAndProcessDecisionTask(false, false) - s.True(err == nil, err) - events := s.getHistory(s.domainName, executions[i]) - if i == maximumAttempts-1 { - s.Equal(types.EventTypeWorkflowExecutionCompleted, events[len(events)-1].GetEventType()) - } else { - s.Equal(types.EventTypeWorkflowExecutionContinuedAsNew, events[len(events)-1].GetEventType()) - } - s.Equal(int32(i), events[0].GetWorkflowExecutionStartedEventAttributes().GetAttempt()) - - dweResponse, err := describeWorkflowExecution(executions[i]) - s.Nil(err) - backoff := time.Duration(0) - if i > 0 { - backoff = time.Duration(float64(initialIntervalInSeconds)*math.Pow(backoffCoefficient, float64(i-1))) * time.Second - // retry backoff cannot larger than MaximumIntervalInSeconds - if backoff > time.Second { - backoff = time.Second - } - } - expectedExecutionTime := dweResponse.WorkflowExecutionInfo.GetStartTime() + backoff.Nanoseconds() - s.Equal(expectedExecutionTime, dweResponse.WorkflowExecutionInfo.GetExecutionTime()) - } -} - -func (s *IntegrationSuite) TestWorkflowRetryFailures() { - id := "integration-wf-retry-failures-test" - wt := "integration-wf-retry-failures-type" - tl := "integration-wf-retry-failures-tasklist" - identity := "worker1" - - workflowType := &types.WorkflowType{} - workflowType.Name = wt - - taskList := &types.TaskList{} - taskList.Name = tl - - workflowImpl := func(attempts int, errorReason string, executions *[]*types.WorkflowExecution) decisionTaskHandler { - attemptCount := 0 - - dtHandler := func(execution *types.WorkflowExecution, wt *types.WorkflowType, - previousStartedEventID, startedEventID int64, history *types.History) ([]byte, []*types.Decision, error) { - *executions = append(*executions, execution) - attemptCount++ - if attemptCount == attempts { - return nil, []*types.Decision{ - { - DecisionType: types.DecisionTypeCompleteWorkflowExecution.Ptr(), - CompleteWorkflowExecutionDecisionAttributes: &types.CompleteWorkflowExecutionDecisionAttributes{ - Result: []byte("succeed-after-retry"), - }, - }}, nil - } - return nil, []*types.Decision{ - { - DecisionType: types.DecisionTypeFailWorkflowExecution.Ptr(), - FailWorkflowExecutionDecisionAttributes: &types.FailWorkflowExecutionDecisionAttributes{ - //Reason: common.StringPtr("retryable-error"), - Reason: common.StringPtr(errorReason), - Details: nil, - }, - }}, nil - } - - return dtHandler - } - - // Fail using attempt - request := &types.StartWorkflowExecutionRequest{ - RequestID: uuid.New(), - Domain: s.domainName, - WorkflowID: id, - WorkflowType: workflowType, - TaskList: taskList, - Input: nil, - ExecutionStartToCloseTimeoutSeconds: common.Int32Ptr(100), - TaskStartToCloseTimeoutSeconds: common.Int32Ptr(1), - Identity: identity, - RetryPolicy: &types.RetryPolicy{ - InitialIntervalInSeconds: 1, - MaximumAttempts: 3, - MaximumIntervalInSeconds: 1, - NonRetriableErrorReasons: []string{"bad-bug"}, - BackoffCoefficient: 1, - ExpirationIntervalInSeconds: 100, - }, - } - - we, err0 := s.engine.StartWorkflowExecution(createContext(), request) - s.Nil(err0) - - s.Logger.Info("StartWorkflowExecution", tag.WorkflowRunID(we.RunID)) - - executions := []*types.WorkflowExecution{} - dtHandler := workflowImpl(5, "retryable-error", &executions) - poller := &TaskPoller{ - Engine: s.engine, - Domain: s.domainName, - TaskList: taskList, - Identity: identity, - DecisionHandler: dtHandler, - Logger: s.Logger, - T: s.T(), - } - - _, err := poller.PollAndProcessDecisionTask(false, false) - s.True(err == nil, err) - events := s.getHistory(s.domainName, executions[0]) - s.Equal(types.EventTypeWorkflowExecutionContinuedAsNew, events[len(events)-1].GetEventType()) - s.Equal(int32(0), events[0].GetWorkflowExecutionStartedEventAttributes().GetAttempt()) - - _, err = poller.PollAndProcessDecisionTask(false, false) - s.True(err == nil, err) - events = s.getHistory(s.domainName, executions[1]) - s.Equal(types.EventTypeWorkflowExecutionContinuedAsNew, events[len(events)-1].GetEventType()) - s.Equal(int32(1), events[0].GetWorkflowExecutionStartedEventAttributes().GetAttempt()) - - _, err = poller.PollAndProcessDecisionTask(false, false) - s.True(err == nil, err) - events = s.getHistory(s.domainName, executions[2]) - s.Equal(types.EventTypeWorkflowExecutionFailed, events[len(events)-1].GetEventType()) - s.Equal(int32(2), events[0].GetWorkflowExecutionStartedEventAttributes().GetAttempt()) - - // Fail error reason - request = &types.StartWorkflowExecutionRequest{ - RequestID: uuid.New(), - Domain: s.domainName, - WorkflowID: id, - WorkflowType: workflowType, - TaskList: taskList, - Input: nil, - ExecutionStartToCloseTimeoutSeconds: common.Int32Ptr(100), - TaskStartToCloseTimeoutSeconds: common.Int32Ptr(1), - Identity: identity, - RetryPolicy: &types.RetryPolicy{ - InitialIntervalInSeconds: 1, - MaximumAttempts: 3, - MaximumIntervalInSeconds: 1, - NonRetriableErrorReasons: []string{"bad-bug"}, - BackoffCoefficient: 1, - ExpirationIntervalInSeconds: 100, - }, - } - - we, err0 = s.engine.StartWorkflowExecution(createContext(), request) - s.Nil(err0) - - s.Logger.Info("StartWorkflowExecution", tag.WorkflowRunID(we.RunID)) - - executions = []*types.WorkflowExecution{} - dtHandler = workflowImpl(5, "bad-bug", &executions) - poller = &TaskPoller{ - Engine: s.engine, - Domain: s.domainName, - TaskList: taskList, - Identity: identity, - DecisionHandler: dtHandler, - Logger: s.Logger, - T: s.T(), - } - - _, err = poller.PollAndProcessDecisionTask(false, false) - s.True(err == nil, err) - events = s.getHistory(s.domainName, executions[0]) - s.Equal(types.EventTypeWorkflowExecutionFailed, events[len(events)-1].GetEventType()) - s.Equal(int32(0), events[0].GetWorkflowExecutionStartedEventAttributes().GetAttempt()) - -} - -func (s *IntegrationSuite) TestCronWorkflow() { - id := "integration-wf-cron-test" - wt := "integration-wf-cron-type" - tl := "integration-wf-cron-tasklist" - identity := "worker1" - cronSchedule := "@every 3s" - - targetBackoffDuration := time.Second * 3 - backoffDurationTolerance := time.Millisecond * 500 - - workflowType := &types.WorkflowType{} - workflowType.Name = wt - - taskList := &types.TaskList{} - taskList.Name = tl - - memo := &types.Memo{ - Fields: map[string][]byte{"memoKey": []byte("memoVal")}, - } - searchAttr := &types.SearchAttributes{ - IndexedFields: map[string][]byte{"CustomKeywordField": []byte(`"1"`)}, - } - - request := &types.StartWorkflowExecutionRequest{ - RequestID: uuid.New(), - Domain: s.domainName, - WorkflowID: id, - WorkflowType: workflowType, - TaskList: taskList, - Input: nil, - ExecutionStartToCloseTimeoutSeconds: common.Int32Ptr(100), - TaskStartToCloseTimeoutSeconds: common.Int32Ptr(1), - Identity: identity, - CronSchedule: cronSchedule, //minimum interval by standard spec is 1m (* * * * *), use non-standard descriptor for short interval for test - Memo: memo, - SearchAttributes: searchAttr, - } - - startWorkflowTS := time.Now() - we, err0 := s.engine.StartWorkflowExecution(createContext(), request) - s.Nil(err0) - - s.Logger.Info("StartWorkflowExecution", tag.WorkflowRunID(we.RunID)) - - var executions []*types.WorkflowExecution - - attemptCount := 0 - - dtHandler := func(execution *types.WorkflowExecution, wt *types.WorkflowType, - previousStartedEventID, startedEventID int64, history *types.History) ([]byte, []*types.Decision, error) { - executions = append(executions, execution) - attemptCount++ - if attemptCount == 2 { - return nil, []*types.Decision{ - { - DecisionType: types.DecisionTypeCompleteWorkflowExecution.Ptr(), - CompleteWorkflowExecutionDecisionAttributes: &types.CompleteWorkflowExecutionDecisionAttributes{ - Result: []byte("cron-test-result"), - }, - }}, nil - } - return nil, []*types.Decision{ - { - DecisionType: types.DecisionTypeFailWorkflowExecution.Ptr(), - FailWorkflowExecutionDecisionAttributes: &types.FailWorkflowExecutionDecisionAttributes{ - Reason: common.StringPtr("cron-test-error"), - Details: nil, - }, - }}, nil - } - - poller := &TaskPoller{ - Engine: s.engine, - Domain: s.domainName, - TaskList: taskList, - Identity: identity, - DecisionHandler: dtHandler, - Logger: s.Logger, - T: s.T(), - } - - startFilter := &types.StartTimeFilter{} - startFilter.EarliestTime = common.Int64Ptr(startWorkflowTS.UnixNano()) - startFilter.LatestTime = common.Int64Ptr(time.Now().UnixNano()) - - // Sleep some time before checking the open executions. - // This will not cost extra time as the polling for first decision task will be blocked for 3 seconds. - time.Sleep(2 * time.Second) - resp, err := s.engine.ListOpenWorkflowExecutions(createContext(), &types.ListOpenWorkflowExecutionsRequest{ - Domain: s.domainName, - MaximumPageSize: 100, - StartTimeFilter: startFilter, - ExecutionFilter: &types.WorkflowExecutionFilter{ - WorkflowID: id, - }, - }) - s.Nil(err) - s.Equal(1, len(resp.GetExecutions())) - executionInfo := resp.GetExecutions()[0] - s.Equal(targetBackoffDuration.Nanoseconds(), executionInfo.GetExecutionTime()-executionInfo.GetStartTime()) - - _, err = poller.PollAndProcessDecisionTask(false, false) - s.True(err == nil, err) - - // Make sure the cron workflow start running at a proper time, in this case 3 seconds after the - // startWorkflowExecution request - backoffDuration := time.Now().Sub(startWorkflowTS) - s.True(backoffDuration > targetBackoffDuration) - s.True(backoffDuration < targetBackoffDuration+backoffDurationTolerance) - - _, err = poller.PollAndProcessDecisionTask(false, false) - s.True(err == nil, err) - - _, err = poller.PollAndProcessDecisionTask(false, false) - s.True(err == nil, err) - - s.Equal(3, attemptCount) - - terminateErr := s.engine.TerminateWorkflowExecution(createContext(), &types.TerminateWorkflowExecutionRequest{ - Domain: s.domainName, - WorkflowExecution: &types.WorkflowExecution{ - WorkflowID: id, - }, - }) - s.NoError(terminateErr) - events := s.getHistory(s.domainName, executions[0]) - lastEvent := events[len(events)-1] - s.Equal(types.EventTypeWorkflowExecutionContinuedAsNew, lastEvent.GetEventType()) - attributes := lastEvent.WorkflowExecutionContinuedAsNewEventAttributes - s.Equal(types.ContinueAsNewInitiatorCronSchedule, attributes.GetInitiator()) - s.Equal("cron-test-error", attributes.GetFailureReason()) - s.Equal(0, len(attributes.GetLastCompletionResult())) - s.Equal(memo, attributes.Memo) - s.Equal(searchAttr, attributes.SearchAttributes) - - events = s.getHistory(s.domainName, executions[1]) - lastEvent = events[len(events)-1] - s.Equal(types.EventTypeWorkflowExecutionContinuedAsNew, lastEvent.GetEventType()) - attributes = lastEvent.WorkflowExecutionContinuedAsNewEventAttributes - s.Equal(types.ContinueAsNewInitiatorCronSchedule, attributes.GetInitiator()) - s.Equal("", attributes.GetFailureReason()) - s.Equal("cron-test-result", string(attributes.GetLastCompletionResult())) - s.Equal(memo, attributes.Memo) - s.Equal(searchAttr, attributes.SearchAttributes) - - events = s.getHistory(s.domainName, executions[2]) - lastEvent = events[len(events)-1] - s.Equal(types.EventTypeWorkflowExecutionContinuedAsNew, lastEvent.GetEventType()) - attributes = lastEvent.WorkflowExecutionContinuedAsNewEventAttributes - s.Equal(types.ContinueAsNewInitiatorCronSchedule, attributes.GetInitiator()) - s.Equal("cron-test-error", attributes.GetFailureReason()) - s.Equal("cron-test-result", string(attributes.GetLastCompletionResult())) - s.Equal(memo, attributes.Memo) - s.Equal(searchAttr, attributes.SearchAttributes) - - startFilter.LatestTime = common.Int64Ptr(time.Now().UnixNano()) - var closedExecutions []*types.WorkflowExecutionInfo - for i := 0; i < 10; i++ { - resp, err := s.engine.ListClosedWorkflowExecutions(createContext(), &types.ListClosedWorkflowExecutionsRequest{ - Domain: s.domainName, - MaximumPageSize: 100, - StartTimeFilter: startFilter, - ExecutionFilter: &types.WorkflowExecutionFilter{ - WorkflowID: id, - }, - }) - s.Nil(err) - if len(resp.GetExecutions()) == 4 { - closedExecutions = resp.GetExecutions() - break - } - time.Sleep(200 * time.Millisecond) - } - s.NotNil(closedExecutions) - dweResponse, err := s.engine.DescribeWorkflowExecution(createContext(), &types.DescribeWorkflowExecutionRequest{ - Domain: s.domainName, - Execution: &types.WorkflowExecution{ - WorkflowID: id, - RunID: we.RunID, - }, - }) - s.Nil(err) - expectedExecutionTime := dweResponse.WorkflowExecutionInfo.GetStartTime() + 3*time.Second.Nanoseconds() - s.Equal(expectedExecutionTime, dweResponse.WorkflowExecutionInfo.GetExecutionTime()) - - sort.Slice(closedExecutions, func(i, j int) bool { - return closedExecutions[i].GetStartTime() < closedExecutions[j].GetStartTime() - }) - lastExecution := closedExecutions[0] - - // TODO https://github.com/uber/cadence/issues/3540 - // the rest assertion can cause transient failure - if s.testClusterConfig.Persistence.SQLDBPluginName == "postgres" { - return - } - for i := 1; i != 4; i++ { - executionInfo := closedExecutions[i] - expectedBackoff := executionInfo.GetExecutionTime() - lastExecution.GetExecutionTime() - // The execution time calculate based on last execution close time - // However, the current execution time is based on the current start time - // This code is to remove the diff between current start time and last execution close time - // TODO: Remove this line once we unify the time source - executionTimeDiff := executionInfo.GetStartTime() - lastExecution.GetCloseTime() - // The backoff between any two executions should be multiplier of the target backoff duration which is 3 in this test - backoffSeconds := int(time.Duration(expectedBackoff-executionTimeDiff).Round(time.Second).Seconds()) - targetBackoffSeconds := int(targetBackoffDuration.Seconds()) - s.Equal( - 0, - backoffSeconds % targetBackoffSeconds, - "Still Flaky?: backoffSeconds: %v ((%v-%v) - (%v-%v)), targetBackoffSeconds: %v", - backoffSeconds, - executionInfo.GetExecutionTime(), - lastExecution.GetExecutionTime(), - executionInfo.GetStartTime(), - lastExecution.GetCloseTime(), - targetBackoffSeconds, - ) - lastExecution = executionInfo - } -} - -func (s *IntegrationSuite) TestCronWorkflowTimeout() { - id := "integration-wf-cron-timeout-test" - wt := "integration-wf-cron-timeout-type" - tl := "integration-wf-cron-timeout-tasklist" - identity := "worker1" - cronSchedule := "@every 3s" - - workflowType := &types.WorkflowType{} - workflowType.Name = wt - - taskList := &types.TaskList{} - taskList.Name = tl - - memo := &types.Memo{ - Fields: map[string][]byte{"memoKey": []byte("memoVal")}, - } - searchAttr := &types.SearchAttributes{ - IndexedFields: map[string][]byte{"CustomKeywordField": []byte(`"1"`)}, - } - - request := &types.StartWorkflowExecutionRequest{ - RequestID: uuid.New(), - Domain: s.domainName, - WorkflowID: id, - WorkflowType: workflowType, - TaskList: taskList, - Input: nil, - ExecutionStartToCloseTimeoutSeconds: common.Int32Ptr(1), // set workflow timeout to 1s - TaskStartToCloseTimeoutSeconds: common.Int32Ptr(1), - Identity: identity, - CronSchedule: cronSchedule, //minimum interval by standard spec is 1m (* * * * *), use non-standard descriptor for short interval for test - Memo: memo, - SearchAttributes: searchAttr, - } - - we, err0 := s.engine.StartWorkflowExecution(createContext(), request) - s.Nil(err0) - - s.Logger.Info("StartWorkflowExecution", tag.WorkflowRunID(we.RunID)) - - var executions []*types.WorkflowExecution - dtHandler := func(execution *types.WorkflowExecution, wt *types.WorkflowType, - previousStartedEventID, startedEventID int64, history *types.History) ([]byte, []*types.Decision, error) { - - executions = append(executions, execution) - return nil, []*types.Decision{ - { - DecisionType: types.DecisionTypeStartTimer.Ptr(), - - StartTimerDecisionAttributes: &types.StartTimerDecisionAttributes{ - TimerID: "timer-id", - StartToFireTimeoutSeconds: common.Int64Ptr(5), - }, - }}, nil - } - - poller := &TaskPoller{ - Engine: s.engine, - Domain: s.domainName, - TaskList: taskList, - Identity: identity, - DecisionHandler: dtHandler, - Logger: s.Logger, - T: s.T(), - } - - _, err := poller.PollAndProcessDecisionTask(false, false) - s.True(err == nil, err) - - time.Sleep(1 * time.Second) // wait for workflow timeout - - // check when workflow timeout, continueAsNew event contains expected fields - events := s.getHistory(s.domainName, executions[0]) - lastEvent := events[len(events)-1] - s.Equal(types.EventTypeWorkflowExecutionContinuedAsNew, lastEvent.GetEventType()) - attributes := lastEvent.WorkflowExecutionContinuedAsNewEventAttributes - s.Equal(types.ContinueAsNewInitiatorCronSchedule, attributes.GetInitiator()) - s.Equal("cadenceInternal:Timeout START_TO_CLOSE", attributes.GetFailureReason()) - s.Equal(memo, attributes.Memo) - s.Equal(searchAttr, attributes.SearchAttributes) - - _, err = poller.PollAndProcessDecisionTask(false, false) - s.True(err == nil, err) - - // check new run contains expected fields - events = s.getHistory(s.domainName, executions[1]) - firstEvent := events[0] - s.Equal(types.EventTypeWorkflowExecutionStarted, firstEvent.GetEventType()) - startAttributes := firstEvent.WorkflowExecutionStartedEventAttributes - s.Equal(memo, startAttributes.Memo) - s.Equal(searchAttr, startAttributes.SearchAttributes) - - // terminate cron - terminateErr := s.engine.TerminateWorkflowExecution(createContext(), &types.TerminateWorkflowExecutionRequest{ - Domain: s.domainName, - WorkflowExecution: &types.WorkflowExecution{ - WorkflowID: id, - }, - }) - s.NoError(terminateErr) -} - -func (s *IntegrationSuite) TestSequential_UserTimers() { - id := "integration-sequential-user-timers-test" - wt := "integration-sequential-user-timers-test-type" - tl := "integration-sequential-user-timers-test-tasklist" - identity := "worker1" - - workflowType := &types.WorkflowType{} - workflowType.Name = wt - - taskList := &types.TaskList{} - taskList.Name = tl - - request := &types.StartWorkflowExecutionRequest{ - RequestID: uuid.New(), - Domain: s.domainName, - WorkflowID: id, - WorkflowType: workflowType, - TaskList: taskList, - Input: nil, - ExecutionStartToCloseTimeoutSeconds: common.Int32Ptr(100), - TaskStartToCloseTimeoutSeconds: common.Int32Ptr(1), - Identity: identity, - } - - we, err0 := s.engine.StartWorkflowExecution(createContext(), request) - s.Nil(err0) - - s.Logger.Info("StartWorkflowExecution", tag.WorkflowRunID(we.RunID)) - - workflowComplete := false - timerCount := int32(4) - timerCounter := int32(0) - dtHandler := func(execution *types.WorkflowExecution, wt *types.WorkflowType, - previousStartedEventID, startedEventID int64, history *types.History) ([]byte, []*types.Decision, error) { - if timerCounter < timerCount { - timerCounter++ - buf := new(bytes.Buffer) - s.Nil(binary.Write(buf, binary.LittleEndian, timerCounter)) - return []byte(strconv.Itoa(int(timerCounter))), []*types.Decision{{ - DecisionType: types.DecisionTypeStartTimer.Ptr(), - StartTimerDecisionAttributes: &types.StartTimerDecisionAttributes{ - TimerID: fmt.Sprintf("timer-id-%d", timerCounter), - StartToFireTimeoutSeconds: common.Int64Ptr(1), - }, - }}, nil - } - - workflowComplete = true - return []byte(strconv.Itoa(int(timerCounter))), []*types.Decision{{ - DecisionType: types.DecisionTypeCompleteWorkflowExecution.Ptr(), - CompleteWorkflowExecutionDecisionAttributes: &types.CompleteWorkflowExecutionDecisionAttributes{ - Result: []byte("Done."), - }, - }}, nil - } - - poller := &TaskPoller{ - Engine: s.engine, - Domain: s.domainName, - TaskList: taskList, - Identity: identity, - DecisionHandler: dtHandler, - ActivityHandler: nil, - Logger: s.Logger, - T: s.T(), - } - - for i := 0; i < 4; i++ { - _, err := poller.PollAndProcessDecisionTask(false, false) - s.Logger.Info("PollAndProcessDecisionTask: completed") - s.Nil(err) - } - - s.False(workflowComplete) - _, err := poller.PollAndProcessDecisionTask(false, false) - s.Nil(err) - s.True(workflowComplete) -} - -func (s *IntegrationSuite) TestRateLimitBufferedEvents() { - id := "integration-rate-limit-buffered-events-test" - wt := "integration-rate-limit-buffered-events-test-type" - tl := "integration-rate-limit-buffered-events-test-tasklist" - identity := "worker1" - - workflowType := &types.WorkflowType{} - workflowType.Name = wt - - taskList := &types.TaskList{} - taskList.Name = tl - - // Start workflow execution - request := &types.StartWorkflowExecutionRequest{ - RequestID: uuid.New(), - Domain: s.domainName, - WorkflowID: id, - WorkflowType: workflowType, - TaskList: taskList, - Input: nil, - ExecutionStartToCloseTimeoutSeconds: common.Int32Ptr(100), - TaskStartToCloseTimeoutSeconds: common.Int32Ptr(10), - Identity: identity, - } - - we, err0 := s.engine.StartWorkflowExecution(createContext(), request) - s.Nil(err0) - - s.Logger.Info("StartWorkflowExecution", tag.WorkflowRunID(we.RunID)) - workflowExecution := &types.WorkflowExecution{ - WorkflowID: id, - RunID: we.RunID, - } - - // decider logic - workflowComplete := false - signalsSent := false - signalCount := 0 - dtHandler := func(execution *types.WorkflowExecution, wt *types.WorkflowType, - previousStartedEventID, startedEventID int64, h *types.History) ([]byte, []*types.Decision, error) { - - // Count signals - for _, event := range h.Events[previousStartedEventID:] { - if event.GetEventType() == types.EventTypeWorkflowExecutionSignaled { - signalCount++ - } - } - - if !signalsSent { - signalsSent = true - // Buffered Signals - for i := 0; i < 100; i++ { - buf := new(bytes.Buffer) - binary.Write(buf, binary.LittleEndian, i) - s.Nil(s.sendSignal(s.domainName, workflowExecution, "SignalName", buf.Bytes(), identity)) - } - - buf := new(bytes.Buffer) - binary.Write(buf, binary.LittleEndian, 101) - signalErr := s.sendSignal(s.domainName, workflowExecution, "SignalName", buf.Bytes(), identity) - s.Nil(signalErr) - - // this decision will be ignored as he decision task is already failed - return nil, []*types.Decision{}, nil - } - - workflowComplete = true - return nil, []*types.Decision{{ - DecisionType: types.DecisionTypeCompleteWorkflowExecution.Ptr(), - CompleteWorkflowExecutionDecisionAttributes: &types.CompleteWorkflowExecutionDecisionAttributes{ - Result: []byte("Done."), - }, - }}, nil - } - - poller := &TaskPoller{ - Engine: s.engine, - Domain: s.domainName, - TaskList: taskList, - Identity: identity, - DecisionHandler: dtHandler, - ActivityHandler: nil, - Logger: s.Logger, - T: s.T(), - } - - // first decision to send 101 signals, the last signal will force fail decision and flush buffered events. - _, err := poller.PollAndProcessDecisionTask(false, false) - s.Logger.Info("PollAndProcessDecisionTask", tag.Error(err)) - s.EqualError(err, "EntityNotExistsError{Message: Decision task not found.}") - - // Process signal in decider - _, err = poller.PollAndProcessDecisionTask(false, false) - s.Logger.Info("PollAndProcessDecisionTask", tag.Error(err)) - s.Nil(err) - - s.True(workflowComplete) - s.Equal(101, signalCount) // check that all 101 signals are received. -} - -func (s *IntegrationSuite) TestBufferedEvents() { - id := "integration-buffered-events-test" - wt := "integration-buffered-events-test-type" - tl := "integration-buffered-events-test-tasklist" - identity := "worker1" - signalName := "buffered-signal" - - workflowType := &types.WorkflowType{Name: wt} - taskList := &types.TaskList{Name: tl} - - // Start workflow execution - request := &types.StartWorkflowExecutionRequest{ - RequestID: uuid.New(), - Domain: s.domainName, - WorkflowID: id, - WorkflowType: workflowType, - TaskList: taskList, - Input: nil, - ExecutionStartToCloseTimeoutSeconds: common.Int32Ptr(100), - TaskStartToCloseTimeoutSeconds: common.Int32Ptr(1), - Identity: identity, - } - - we, err0 := s.engine.StartWorkflowExecution(createContext(), request) - s.Nil(err0) - - s.Logger.Info("StartWorkflowExecution", tag.WorkflowRunID(we.RunID)) - - // decider logic - workflowComplete := false - signalSent := false - var signalEvent *types.HistoryEvent - dtHandler := func(execution *types.WorkflowExecution, wt *types.WorkflowType, - previousStartedEventID, startedEventID int64, history *types.History) ([]byte, []*types.Decision, error) { - if !signalSent { - signalSent = true - - // this will create new event when there is in-flight decision task, and the new event will be buffered - err := s.engine.SignalWorkflowExecution(createContext(), - &types.SignalWorkflowExecutionRequest{ - Domain: s.domainName, - WorkflowExecution: &types.WorkflowExecution{ - WorkflowID: id, - }, - SignalName: "buffered-signal", - Input: []byte("buffered-signal-input"), - Identity: identity, - }) - s.NoError(err) - return nil, []*types.Decision{{ - DecisionType: types.DecisionTypeScheduleActivityTask.Ptr(), - ScheduleActivityTaskDecisionAttributes: &types.ScheduleActivityTaskDecisionAttributes{ - ActivityID: "1", - ActivityType: &types.ActivityType{Name: "test-activity-type"}, - TaskList: &types.TaskList{Name: tl}, - Input: []byte("test-input"), - ScheduleToCloseTimeoutSeconds: common.Int32Ptr(100), - ScheduleToStartTimeoutSeconds: common.Int32Ptr(2), - StartToCloseTimeoutSeconds: common.Int32Ptr(50), - HeartbeatTimeoutSeconds: common.Int32Ptr(5), - }, - }}, nil - } else if previousStartedEventID > 0 && signalEvent == nil { - for _, event := range history.Events[previousStartedEventID:] { - if *event.EventType == types.EventTypeWorkflowExecutionSignaled { - signalEvent = event - } - } - } - - workflowComplete = true - return nil, []*types.Decision{{ - DecisionType: types.DecisionTypeCompleteWorkflowExecution.Ptr(), - CompleteWorkflowExecutionDecisionAttributes: &types.CompleteWorkflowExecutionDecisionAttributes{ - Result: []byte("Done."), - }, - }}, nil - } - - poller := &TaskPoller{ - Engine: s.engine, - Domain: s.domainName, - TaskList: taskList, - Identity: identity, - DecisionHandler: dtHandler, - ActivityHandler: nil, - Logger: s.Logger, - T: s.T(), - } - - // first decision, which sends signal and the signal event should be buffered to append after first decision closed - _, err := poller.PollAndProcessDecisionTask(false, false) - s.Logger.Info("PollAndProcessDecisionTask", tag.Error(err)) - s.Nil(err) - - // check history, the signal event should be after the complete decision task - histResp, err := s.engine.GetWorkflowExecutionHistory(createContext(), &types.GetWorkflowExecutionHistoryRequest{ - Domain: s.domainName, - Execution: &types.WorkflowExecution{ - WorkflowID: id, - RunID: we.RunID, - }, - }) - s.NoError(err) - s.NotNil(histResp.History.Events) - s.True(len(histResp.History.Events) >= 6) - s.Equal(histResp.History.Events[3].GetEventType(), types.EventTypeDecisionTaskCompleted) - s.Equal(histResp.History.Events[4].GetEventType(), types.EventTypeActivityTaskScheduled) - s.Equal(histResp.History.Events[5].GetEventType(), types.EventTypeWorkflowExecutionSignaled) - - // Process signal in decider - _, err = poller.PollAndProcessDecisionTask(false, false) - s.Logger.Info("PollAndProcessDecisionTask", tag.Error(err)) - s.Nil(err) - s.NotNil(signalEvent) - s.Equal(signalName, signalEvent.WorkflowExecutionSignaledEventAttributes.SignalName) - s.Equal(identity, signalEvent.WorkflowExecutionSignaledEventAttributes.Identity) - s.True(workflowComplete) -} - -func (s *IntegrationSuite) TestDescribeWorkflowExecution() { - id := "integration-describe-wfe-test" - wt := "integration-describe-wfe-test-type" - tl := "integration-describe-wfe-test-tasklist" - identity := "worker1" - - workflowType := &types.WorkflowType{Name: wt} - taskList := &types.TaskList{Name: tl} - - // Start workflow execution - request := &types.StartWorkflowExecutionRequest{ - RequestID: uuid.New(), - Domain: s.domainName, - WorkflowID: id, - WorkflowType: workflowType, - TaskList: taskList, - Input: nil, - ExecutionStartToCloseTimeoutSeconds: common.Int32Ptr(100), - TaskStartToCloseTimeoutSeconds: common.Int32Ptr(1), - Identity: identity, - } - - we, err0 := s.engine.StartWorkflowExecution(createContext(), request) - s.Nil(err0) - - s.Logger.Info("StartWorkflowExecution", tag.WorkflowRunID(we.RunID)) - - describeWorkflowExecution := func() (*types.DescribeWorkflowExecutionResponse, error) { - return s.engine.DescribeWorkflowExecution(createContext(), &types.DescribeWorkflowExecutionRequest{ - Domain: s.domainName, - Execution: &types.WorkflowExecution{ - WorkflowID: id, - RunID: we.RunID, - }, - }) - } - dweResponse, err := describeWorkflowExecution() - s.Nil(err) - s.True(nil == dweResponse.WorkflowExecutionInfo.CloseTime) - s.Equal(int64(2), dweResponse.WorkflowExecutionInfo.HistoryLength) // WorkflowStarted, DecisionScheduled - s.Equal(dweResponse.WorkflowExecutionInfo.GetStartTime(), dweResponse.WorkflowExecutionInfo.GetExecutionTime()) - - // decider logic - workflowComplete := false - signalSent := false - dtHandler := func(execution *types.WorkflowExecution, wt *types.WorkflowType, - previousStartedEventID, startedEventID int64, history *types.History) ([]byte, []*types.Decision, error) { - if !signalSent { - signalSent = true - - s.NoError(err) - return nil, []*types.Decision{{ - DecisionType: types.DecisionTypeScheduleActivityTask.Ptr(), - ScheduleActivityTaskDecisionAttributes: &types.ScheduleActivityTaskDecisionAttributes{ - ActivityID: "1", - ActivityType: &types.ActivityType{Name: "test-activity-type"}, - TaskList: &types.TaskList{Name: tl}, - Input: []byte("test-input"), - ScheduleToCloseTimeoutSeconds: common.Int32Ptr(100), - ScheduleToStartTimeoutSeconds: common.Int32Ptr(2), - StartToCloseTimeoutSeconds: common.Int32Ptr(50), - HeartbeatTimeoutSeconds: common.Int32Ptr(5), - }, - }}, nil - } - - workflowComplete = true - return nil, []*types.Decision{{ - DecisionType: types.DecisionTypeCompleteWorkflowExecution.Ptr(), - CompleteWorkflowExecutionDecisionAttributes: &types.CompleteWorkflowExecutionDecisionAttributes{ - Result: []byte("Done."), - }, - }}, nil - } - - atHandler := func(execution *types.WorkflowExecution, activityType *types.ActivityType, - ActivityID string, input []byte, taskToken []byte) ([]byte, bool, error) { - return []byte("Activity Result."), false, nil - } - - poller := &TaskPoller{ - Engine: s.engine, - Domain: s.domainName, - TaskList: taskList, - Identity: identity, - DecisionHandler: dtHandler, - ActivityHandler: atHandler, - Logger: s.Logger, - T: s.T(), - } - - // first decision to schedule new activity - _, err = poller.PollAndProcessDecisionTask(false, false) - s.Logger.Info("PollAndProcessDecisionTask", tag.Error(err)) - s.Nil(err) - - dweResponse, err = describeWorkflowExecution() - s.Nil(err) - s.True(nil == dweResponse.WorkflowExecutionInfo.CloseStatus) - s.Equal(int64(5), dweResponse.WorkflowExecutionInfo.HistoryLength) // DecisionStarted, DecisionCompleted, ActivityScheduled - s.Equal(1, len(dweResponse.PendingActivities)) - s.Equal("test-activity-type", dweResponse.PendingActivities[0].ActivityType.GetName()) - s.Equal(int64(0), dweResponse.PendingActivities[0].GetLastHeartbeatTimestamp()) - - // process activity task - err = poller.PollAndProcessActivityTask(false) - - dweResponse, err = describeWorkflowExecution() - s.Nil(err) - s.True(nil == dweResponse.WorkflowExecutionInfo.CloseStatus) - s.Equal(int64(8), dweResponse.WorkflowExecutionInfo.HistoryLength) // ActivityTaskStarted, ActivityTaskCompleted, DecisionTaskScheduled - s.Equal(0, len(dweResponse.PendingActivities)) - - // Process signal in decider - _, err = poller.PollAndProcessDecisionTask(false, false) - s.Nil(err) - s.True(workflowComplete) - - dweResponse, err = describeWorkflowExecution() - s.Nil(err) - s.Equal(types.WorkflowExecutionCloseStatusCompleted, *dweResponse.WorkflowExecutionInfo.CloseStatus) - s.Equal(int64(11), dweResponse.WorkflowExecutionInfo.HistoryLength) // DecisionStarted, DecisionCompleted, WorkflowCompleted -} - -func (s *IntegrationSuite) TestVisibility() { - startTime := time.Now().UnixNano() - - // Start 2 workflow executions - id1 := "integration-visibility-test1" - id2 := "integration-visibility-test2" - wt := "integration-visibility-test-type" - tl := "integration-visibility-test-tasklist" - identity := "worker1" - - workflowType := &types.WorkflowType{} - workflowType.Name = wt - - taskList := &types.TaskList{} - taskList.Name = tl - - startRequest := &types.StartWorkflowExecutionRequest{ - RequestID: uuid.New(), - Domain: s.domainName, - WorkflowID: id1, - WorkflowType: workflowType, - TaskList: taskList, - Input: nil, - ExecutionStartToCloseTimeoutSeconds: common.Int32Ptr(100), - TaskStartToCloseTimeoutSeconds: common.Int32Ptr(10), - Identity: identity, - } - - startResponse, err0 := s.engine.StartWorkflowExecution(createContext(), startRequest) - s.Nil(err0) - - // Now complete one of the executions - dtHandler := func(execution *types.WorkflowExecution, wt *types.WorkflowType, - previousStartedEventID, startedEventID int64, history *types.History) ([]byte, []*types.Decision, error) { - return []byte{}, []*types.Decision{{ - DecisionType: types.DecisionTypeCompleteWorkflowExecution.Ptr(), - CompleteWorkflowExecutionDecisionAttributes: &types.CompleteWorkflowExecutionDecisionAttributes{ - Result: []byte("Done."), - }, - }}, nil - } - - poller := &TaskPoller{ - Engine: s.engine, - Domain: s.domainName, - TaskList: taskList, - Identity: identity, - DecisionHandler: dtHandler, - ActivityHandler: nil, - Logger: s.Logger, - T: s.T(), - } - - _, err1 := poller.PollAndProcessDecisionTask(false, false) - s.Nil(err1) - - // wait until the start workflow is done - var nextToken []byte - historyEventFilterType := types.HistoryEventFilterTypeCloseEvent - for { - historyResponse, historyErr := s.engine.GetWorkflowExecutionHistory(createContext(), &types.GetWorkflowExecutionHistoryRequest{ - Domain: startRequest.Domain, - Execution: &types.WorkflowExecution{ - WorkflowID: startRequest.WorkflowID, - RunID: startResponse.RunID, - }, - WaitForNewEvent: true, - HistoryEventFilterType: &historyEventFilterType, - NextPageToken: nextToken, - }) - s.Nil(historyErr) - if len(historyResponse.NextPageToken) == 0 { - break - } - - nextToken = historyResponse.NextPageToken - } - - startRequest = &types.StartWorkflowExecutionRequest{ - RequestID: uuid.New(), - Domain: s.domainName, - WorkflowID: id2, - WorkflowType: workflowType, - TaskList: taskList, - Input: nil, - ExecutionStartToCloseTimeoutSeconds: common.Int32Ptr(100), - TaskStartToCloseTimeoutSeconds: common.Int32Ptr(10), - Identity: identity, - } - - _, err2 := s.engine.StartWorkflowExecution(createContext(), startRequest) - s.Nil(err2) - - startFilter := &types.StartTimeFilter{} - startFilter.EarliestTime = common.Int64Ptr(startTime) - startFilter.LatestTime = common.Int64Ptr(time.Now().UnixNano()) - - closedCount := 0 - openCount := 0 - - var historyLength int64 - for i := 0; i < 10; i++ { - resp, err3 := s.engine.ListClosedWorkflowExecutions(createContext(), &types.ListClosedWorkflowExecutionsRequest{ - Domain: s.domainName, - MaximumPageSize: 100, - StartTimeFilter: startFilter, - }) - s.Nil(err3) - closedCount = len(resp.Executions) - if closedCount == 1 { - historyLength = resp.Executions[0].HistoryLength - break - } - s.Logger.Info("Closed WorkflowExecution is not yet visible") - time.Sleep(100 * time.Millisecond) - } - s.Equal(1, closedCount) - s.Equal(int64(5), historyLength) - - for i := 0; i < 10; i++ { - resp, err4 := s.engine.ListOpenWorkflowExecutions(createContext(), &types.ListOpenWorkflowExecutionsRequest{ - Domain: s.domainName, - MaximumPageSize: 100, - StartTimeFilter: startFilter, - }) - s.Nil(err4) - openCount = len(resp.Executions) - if openCount == 1 { - break - } - s.Logger.Info("Open WorkflowExecution is not yet visible") - time.Sleep(100 * time.Millisecond) - } - s.Equal(1, openCount) -} - -func (s *IntegrationSuite) TestChildWorkflowExecution() { - parentID := "integration-child-workflow-test-parent" - childID := "integration-child-workflow-test-child" - wtParent := "integration-child-workflow-test-parent-type" - wtChild := "integration-child-workflow-test-child-type" - tlParent := "integration-child-workflow-test-parent-tasklist" - tlChild := "integration-child-workflow-test-child-tasklist" - identity := "worker1" - - parentWorkflowType := &types.WorkflowType{} - parentWorkflowType.Name = wtParent - - childWorkflowType := &types.WorkflowType{} - childWorkflowType.Name = wtChild - - taskListParent := &types.TaskList{} - taskListParent.Name = tlParent - taskListChild := &types.TaskList{} - taskListChild.Name = tlChild - - header := &types.Header{ - Fields: map[string][]byte{"tracing": []byte("sample payload")}, - } - - request := &types.StartWorkflowExecutionRequest{ - RequestID: uuid.New(), - Domain: s.domainName, - WorkflowID: parentID, - WorkflowType: parentWorkflowType, - TaskList: taskListParent, - Input: nil, - Header: header, - ExecutionStartToCloseTimeoutSeconds: common.Int32Ptr(100), - TaskStartToCloseTimeoutSeconds: common.Int32Ptr(1), - Identity: identity, - } - - we, err0 := s.engine.StartWorkflowExecution(createContext(), request) - s.Nil(err0) - s.Logger.Info("StartWorkflowExecution", tag.WorkflowRunID(we.RunID)) - - // decider logic - childComplete := false - childExecutionStarted := false - var startedEvent *types.HistoryEvent - var completedEvent *types.HistoryEvent - - memoInfo, _ := json.Marshal("memo") - memo := &types.Memo{ - Fields: map[string][]byte{ - "Info": memoInfo, - }, - } - attrValBytes, _ := json.Marshal("attrVal") - searchAttr := &types.SearchAttributes{ - IndexedFields: map[string][]byte{ - "CustomKeywordField": attrValBytes, - }, - } - - // Parent Decider Logic - dtHandlerParent := func(execution *types.WorkflowExecution, wt *types.WorkflowType, - previousStartedEventID, startedEventID int64, history *types.History) ([]byte, []*types.Decision, error) { - s.Logger.Info("Processing decision task for ", tag.WorkflowID(execution.WorkflowID)) - - if execution.WorkflowID == parentID { - if !childExecutionStarted { - s.Logger.Info("Starting child execution.") - childExecutionStarted = true - - return nil, []*types.Decision{{ - DecisionType: types.DecisionTypeStartChildWorkflowExecution.Ptr(), - StartChildWorkflowExecutionDecisionAttributes: &types.StartChildWorkflowExecutionDecisionAttributes{ - WorkflowID: childID, - WorkflowType: childWorkflowType, - TaskList: taskListChild, - Input: []byte("child-workflow-input"), - Header: header, - ExecutionStartToCloseTimeoutSeconds: common.Int32Ptr(200), - TaskStartToCloseTimeoutSeconds: common.Int32Ptr(2), - Control: nil, - Memo: memo, - SearchAttributes: searchAttr, - }, - }}, nil - } else if previousStartedEventID > 0 { - for _, event := range history.Events[previousStartedEventID:] { - if *event.EventType == types.EventTypeChildWorkflowExecutionStarted { - startedEvent = event - return nil, []*types.Decision{}, nil - } - - if *event.EventType == types.EventTypeChildWorkflowExecutionCompleted { - completedEvent = event - return nil, []*types.Decision{{ - DecisionType: types.DecisionTypeCompleteWorkflowExecution.Ptr(), - CompleteWorkflowExecutionDecisionAttributes: &types.CompleteWorkflowExecutionDecisionAttributes{ - Result: []byte("Done."), - }, - }}, nil - } - } - } - } - - return nil, nil, nil - } - - var childStartedEvent *types.HistoryEvent - // Child Decider Logic - dtHandlerChild := func(execution *types.WorkflowExecution, wt *types.WorkflowType, - previousStartedEventID, startedEventID int64, history *types.History) ([]byte, []*types.Decision, error) { - if previousStartedEventID <= 0 { - childStartedEvent = history.Events[0] - } - - s.Logger.Info("Processing decision task for Child ", tag.WorkflowID(execution.WorkflowID)) - childComplete = true - return nil, []*types.Decision{{ - DecisionType: types.DecisionTypeCompleteWorkflowExecution.Ptr(), - CompleteWorkflowExecutionDecisionAttributes: &types.CompleteWorkflowExecutionDecisionAttributes{ - Result: []byte("Child Done."), - }, - }}, nil - } - - pollerParent := &TaskPoller{ - Engine: s.engine, - Domain: s.domainName, - TaskList: taskListParent, - Identity: identity, - DecisionHandler: dtHandlerParent, - Logger: s.Logger, - T: s.T(), - } - - pollerChild := &TaskPoller{ - Engine: s.engine, - Domain: s.domainName, - TaskList: taskListChild, - Identity: identity, - DecisionHandler: dtHandlerChild, - Logger: s.Logger, - T: s.T(), - } - - // Make first decision to start child execution - _, err := pollerParent.PollAndProcessDecisionTask(false, false) - s.Logger.Info("PollAndProcessDecisionTask", tag.Error(err)) - s.Nil(err) - s.True(childExecutionStarted) - - // Process ChildExecution Started event and Process Child Execution and complete it - _, err = pollerParent.PollAndProcessDecisionTask(false, false) - s.Logger.Info("PollAndProcessDecisionTask", tag.Error(err)) - s.Nil(err) - - _, err = pollerChild.PollAndProcessDecisionTask(false, false) - s.Logger.Info("PollAndProcessDecisionTask", tag.Error(err)) - s.Nil(err) - s.NotNil(startedEvent) - s.True(childComplete) - s.NotNil(childStartedEvent) - s.Equal(types.EventTypeWorkflowExecutionStarted, childStartedEvent.GetEventType()) - s.Equal(s.domainName, childStartedEvent.WorkflowExecutionStartedEventAttributes.GetParentWorkflowDomain()) - s.Equal(parentID, childStartedEvent.WorkflowExecutionStartedEventAttributes.ParentWorkflowExecution.GetWorkflowID()) - s.Equal(we.GetRunID(), childStartedEvent.WorkflowExecutionStartedEventAttributes.ParentWorkflowExecution.GetRunID()) - s.Equal(startedEvent.ChildWorkflowExecutionStartedEventAttributes.GetInitiatedEventID(), - childStartedEvent.WorkflowExecutionStartedEventAttributes.GetParentInitiatedEventID()) - s.Equal(header, startedEvent.ChildWorkflowExecutionStartedEventAttributes.Header) - s.Equal(header, childStartedEvent.WorkflowExecutionStartedEventAttributes.Header) - s.Equal(memo, childStartedEvent.WorkflowExecutionStartedEventAttributes.GetMemo()) - s.Equal(searchAttr, childStartedEvent.WorkflowExecutionStartedEventAttributes.GetSearchAttributes()) - - // Process ChildExecution completed event and complete parent execution - _, err = pollerParent.PollAndProcessDecisionTask(false, false) - s.Logger.Info("PollAndProcessDecisionTask", tag.Error(err)) - s.Nil(err) - s.NotNil(completedEvent) - completedAttributes := completedEvent.ChildWorkflowExecutionCompletedEventAttributes - s.Empty(completedAttributes.Domain) - s.Equal(childID, completedAttributes.WorkflowExecution.WorkflowID) - s.Equal(wtChild, completedAttributes.WorkflowType.Name) - s.Equal([]byte("Child Done."), completedAttributes.Result) -} - -func (s *IntegrationSuite) TestCronChildWorkflowExecution() { - parentID := "integration-cron-child-workflow-test-parent" - childID := "integration-cron-child-workflow-test-child" - wtParent := "integration-cron-child-workflow-test-parent-type" - wtChild := "integration-cron-child-workflow-test-child-type" - tlParent := "integration-cron-child-workflow-test-parent-tasklist" - tlChild := "integration-cron-child-workflow-test-child-tasklist" - identity := "worker1" - - cronSchedule := "@every 3s" - targetBackoffDuration := time.Second * 3 - backoffDurationTolerance := time.Second - - parentWorkflowType := &types.WorkflowType{} - parentWorkflowType.Name = wtParent - - childWorkflowType := &types.WorkflowType{} - childWorkflowType.Name = wtChild - - taskListParent := &types.TaskList{} - taskListParent.Name = tlParent - taskListChild := &types.TaskList{} - taskListChild.Name = tlChild - - request := &types.StartWorkflowExecutionRequest{ - RequestID: uuid.New(), - Domain: s.domainName, - WorkflowID: parentID, - WorkflowType: parentWorkflowType, - TaskList: taskListParent, - Input: nil, - ExecutionStartToCloseTimeoutSeconds: common.Int32Ptr(100), - TaskStartToCloseTimeoutSeconds: common.Int32Ptr(1), - Identity: identity, - } - - startParentWorkflowTS := time.Now() - we, err0 := s.engine.StartWorkflowExecution(createContext(), request) - s.Nil(err0) - s.Logger.Info("StartWorkflowExecution", tag.WorkflowRunID(we.RunID)) - - // decider logic - childExecutionStarted := false - var terminatedEvent *types.HistoryEvent - var startChildWorkflowTS time.Time - // Parent Decider Logic - dtHandlerParent := func(execution *types.WorkflowExecution, wt *types.WorkflowType, - previousStartedEventID, startedEventID int64, history *types.History) ([]byte, []*types.Decision, error) { - s.Logger.Info("Processing decision task for ", tag.WorkflowID(execution.WorkflowID)) - - if !childExecutionStarted { - s.Logger.Info("Starting child execution.") - childExecutionStarted = true - startChildWorkflowTS = time.Now() - return nil, []*types.Decision{{ - DecisionType: types.DecisionTypeStartChildWorkflowExecution.Ptr(), - StartChildWorkflowExecutionDecisionAttributes: &types.StartChildWorkflowExecutionDecisionAttributes{ - WorkflowID: childID, - WorkflowType: childWorkflowType, - TaskList: taskListChild, - Input: nil, - ExecutionStartToCloseTimeoutSeconds: common.Int32Ptr(200), - TaskStartToCloseTimeoutSeconds: common.Int32Ptr(2), - Control: nil, - CronSchedule: cronSchedule, - }, - }}, nil - } - for _, event := range history.Events[previousStartedEventID:] { - if *event.EventType == types.EventTypeChildWorkflowExecutionTerminated { - terminatedEvent = event - return nil, []*types.Decision{{ - DecisionType: types.DecisionTypeCompleteWorkflowExecution.Ptr(), - CompleteWorkflowExecutionDecisionAttributes: &types.CompleteWorkflowExecutionDecisionAttributes{ - Result: []byte("Done."), - }, - }}, nil - } - } - return nil, nil, nil - } - - // Child Decider Logic - dtHandlerChild := func(execution *types.WorkflowExecution, wt *types.WorkflowType, - previousStartedEventID, startedEventID int64, history *types.History) ([]byte, []*types.Decision, error) { - - s.Logger.Info("Processing decision task for Child ", tag.WorkflowID(execution.WorkflowID)) - return nil, []*types.Decision{{ - DecisionType: types.DecisionTypeCompleteWorkflowExecution.Ptr(), - CompleteWorkflowExecutionDecisionAttributes: &types.CompleteWorkflowExecutionDecisionAttributes{}, - }}, nil - } - - pollerParent := &TaskPoller{ - Engine: s.engine, - Domain: s.domainName, - TaskList: taskListParent, - Identity: identity, - DecisionHandler: dtHandlerParent, - Logger: s.Logger, - T: s.T(), - } - - pollerChild := &TaskPoller{ - Engine: s.engine, - Domain: s.domainName, - TaskList: taskListChild, - Identity: identity, - DecisionHandler: dtHandlerChild, - Logger: s.Logger, - T: s.T(), - } - - // Make first decision to start child execution - _, err := pollerParent.PollAndProcessDecisionTask(false, false) - s.Logger.Info("PollAndProcessDecisionTask", tag.Error(err)) - s.Nil(err) - s.True(childExecutionStarted) - - // Process ChildExecution Started event - _, err = pollerParent.PollAndProcessDecisionTask(false, false) - s.Logger.Info("PollAndProcessDecisionTask", tag.Error(err)) - s.Nil(err) - - startFilter := &types.StartTimeFilter{} - startFilter.EarliestTime = common.Int64Ptr(startChildWorkflowTS.UnixNano()) - for i := 0; i < 2; i++ { - // Sleep some time before checking the open executions. - // This will not cost extra time as the polling for first decision task will be blocked for 3 seconds. - time.Sleep(2 * time.Second) - startFilter.LatestTime = common.Int64Ptr(time.Now().UnixNano()) - resp, err := s.engine.ListOpenWorkflowExecutions(createContext(), &types.ListOpenWorkflowExecutionsRequest{ - Domain: s.domainName, - MaximumPageSize: 100, - StartTimeFilter: startFilter, - ExecutionFilter: &types.WorkflowExecutionFilter{ - WorkflowID: childID, - }, - }) - s.Nil(err) - s.Equal(1, len(resp.GetExecutions())) - - _, err = pollerChild.PollAndProcessDecisionTask(false, false) - s.Logger.Info("PollAndProcessDecisionTask", tag.Error(err)) - s.Nil(err) - - backoffDuration := time.Now().Sub(startChildWorkflowTS) - s.True(backoffDuration < targetBackoffDuration+backoffDurationTolerance) - startChildWorkflowTS = time.Now() - } - - // terminate the childworkflow - terminateErr := s.engine.TerminateWorkflowExecution(createContext(), &types.TerminateWorkflowExecutionRequest{ - Domain: s.domainName, - WorkflowExecution: &types.WorkflowExecution{ - WorkflowID: childID, - }, - }) - s.Nil(terminateErr) - - // Process ChildExecution terminated event and complete parent execution - _, err = pollerParent.PollAndProcessDecisionTask(false, false) - s.Logger.Info("PollAndProcessDecisionTask", tag.Error(err)) - s.Nil(err) - s.NotNil(terminatedEvent) - terminatedAttributes := terminatedEvent.ChildWorkflowExecutionTerminatedEventAttributes - s.Empty(terminatedAttributes.Domain) - s.Equal(childID, terminatedAttributes.WorkflowExecution.WorkflowID) - s.Equal(wtChild, terminatedAttributes.WorkflowType.Name) - - startFilter.EarliestTime = common.Int64Ptr(startParentWorkflowTS.UnixNano()) - startFilter.LatestTime = common.Int64Ptr(time.Now().UnixNano()) - var closedExecutions []*types.WorkflowExecutionInfo - for i := 0; i < 10; i++ { - resp, err := s.engine.ListClosedWorkflowExecutions(createContext(), &types.ListClosedWorkflowExecutionsRequest{ - Domain: s.domainName, - MaximumPageSize: 100, - StartTimeFilter: startFilter, - }) - s.Nil(err) - if len(resp.GetExecutions()) == 4 { - closedExecutions = resp.GetExecutions() - break - } - time.Sleep(200 * time.Millisecond) - } - s.NotNil(closedExecutions) - sort.Slice(closedExecutions, func(i, j int) bool { - return closedExecutions[i].GetStartTime() < closedExecutions[j].GetStartTime() - }) - //The first parent is not the cron workflow, only verify child workflow with cron schedule - lastExecution := closedExecutions[1] - for i := 2; i != 4; i++ { - executionInfo := closedExecutions[i] - // Round up the time precision to seconds - expectedBackoff := executionInfo.GetExecutionTime()/1000000000 - lastExecution.GetExecutionTime()/1000000000 - // The execution time calculate based on last execution close time - // However, the current execution time is based on the current start time - // This code is to remove the diff between current start time and last execution close time - // TODO: Remove this line once we unify the time source. - executionTimeDiff := executionInfo.GetStartTime()/1000000000 - lastExecution.GetCloseTime()/1000000000 - // The backoff between any two executions should be multiplier of the target backoff duration which is 3 in this test - s.Equal(int64(0), int64(expectedBackoff-executionTimeDiff)/1000000000%(targetBackoffDuration.Nanoseconds()/1000000000)) - lastExecution = executionInfo - } -} - -func (s *IntegrationSuite) TestWorkflowTimeout() { - startTime := time.Now().UnixNano() - - id := "integration-workflow-timeout" - wt := "integration-workflow-timeout-type" - tl := "integration-workflow-timeout-tasklist" - identity := "worker1" - - workflowType := &types.WorkflowType{} - workflowType.Name = wt - - taskList := &types.TaskList{} - taskList.Name = tl - - request := &types.StartWorkflowExecutionRequest{ - RequestID: uuid.New(), - Domain: s.domainName, - WorkflowID: id, - WorkflowType: workflowType, - TaskList: taskList, - Input: nil, - ExecutionStartToCloseTimeoutSeconds: common.Int32Ptr(1), - TaskStartToCloseTimeoutSeconds: common.Int32Ptr(1), - Identity: identity, - } - - we, err0 := s.engine.StartWorkflowExecution(createContext(), request) - s.Nil(err0) - - s.Logger.Info("StartWorkflowExecution", tag.WorkflowRunID(we.RunID)) - - workflowComplete := false - -GetHistoryLoop: - for i := 0; i < 10; i++ { - historyResponse, err := s.engine.GetWorkflowExecutionHistory(createContext(), &types.GetWorkflowExecutionHistoryRequest{ - Domain: s.domainName, - Execution: &types.WorkflowExecution{ - WorkflowID: id, - RunID: we.RunID, - }, - }) - s.Nil(err) - history := historyResponse.History - - lastEvent := history.Events[len(history.Events)-1] - if *lastEvent.EventType != types.EventTypeWorkflowExecutionTimedOut { - s.Logger.Warn("Execution not timedout yet.") - time.Sleep(200 * time.Millisecond) - continue GetHistoryLoop - } - - timeoutEventAttributes := lastEvent.WorkflowExecutionTimedOutEventAttributes - s.Equal(types.TimeoutTypeStartToClose, *timeoutEventAttributes.TimeoutType) - workflowComplete = true - break GetHistoryLoop - } - s.True(workflowComplete) - - startFilter := &types.StartTimeFilter{} - startFilter.EarliestTime = common.Int64Ptr(startTime) - startFilter.LatestTime = common.Int64Ptr(time.Now().UnixNano()) - - closedCount := 0 -ListClosedLoop: - for i := 0; i < 10; i++ { - resp, err3 := s.engine.ListClosedWorkflowExecutions(createContext(), &types.ListClosedWorkflowExecutionsRequest{ - Domain: s.domainName, - MaximumPageSize: 100, - StartTimeFilter: startFilter, - }) - s.Nil(err3) - closedCount = len(resp.Executions) - if closedCount == 0 { - s.Logger.Info("Closed WorkflowExecution is not yet visibile") - time.Sleep(1000 * time.Millisecond) - continue ListClosedLoop - } - break ListClosedLoop - } - s.Equal(1, closedCount) -} - -func (s *IntegrationSuite) TestDecisionTaskFailed() { - id := "integration-decisiontask-failed-test" - wt := "integration-decisiontask-failed-test-type" - tl := "integration-decisiontask-failed-test-tasklist" - identity := "worker1" - activityName := "activity_type1" - - workflowType := &types.WorkflowType{} - workflowType.Name = wt - - taskList := &types.TaskList{} - taskList.Name = tl - - // Start workflow execution - request := &types.StartWorkflowExecutionRequest{ - RequestID: uuid.New(), - Domain: s.domainName, - WorkflowID: id, - WorkflowType: workflowType, - TaskList: taskList, - Input: nil, - ExecutionStartToCloseTimeoutSeconds: common.Int32Ptr(100), - TaskStartToCloseTimeoutSeconds: common.Int32Ptr(10), - Identity: identity, - } - - we, err0 := s.engine.StartWorkflowExecution(createContext(), request) - s.Nil(err0) - s.Logger.Info("StartWorkflowExecution", tag.WorkflowRunID(we.RunID)) - - workflowExecution := &types.WorkflowExecution{ - WorkflowID: id, - RunID: we.RunID, - } - - // decider logic - workflowComplete := false - activityScheduled := false - activityData := int32(1) - failureCount := 10 - signalCount := 0 - sendSignal := false - lastDecisionTimestamp := int64(0) - //var signalEvent *types.HistoryEvent - dtHandler := func(execution *types.WorkflowExecution, wt *types.WorkflowType, - previousStartedEventID, startedEventID int64, history *types.History) ([]byte, []*types.Decision, error) { - // Count signals - for _, event := range history.Events[previousStartedEventID:] { - if event.GetEventType() == types.EventTypeWorkflowExecutionSignaled { - signalCount++ - } - } - // Some signals received on this decision - if signalCount == 1 { - return nil, []*types.Decision{}, nil - } - - // Send signals during decision - if sendSignal { - s.sendSignal(s.domainName, workflowExecution, "signalC", nil, identity) - s.sendSignal(s.domainName, workflowExecution, "signalD", nil, identity) - s.sendSignal(s.domainName, workflowExecution, "signalE", nil, identity) - sendSignal = false - } - - if !activityScheduled { - activityScheduled = true - buf := new(bytes.Buffer) - s.Nil(binary.Write(buf, binary.LittleEndian, activityData)) - - return nil, []*types.Decision{{ - DecisionType: types.DecisionTypeScheduleActivityTask.Ptr(), - ScheduleActivityTaskDecisionAttributes: &types.ScheduleActivityTaskDecisionAttributes{ - ActivityID: "1", - ActivityType: &types.ActivityType{Name: activityName}, - TaskList: &types.TaskList{Name: tl}, - Input: buf.Bytes(), - ScheduleToCloseTimeoutSeconds: common.Int32Ptr(100), - ScheduleToStartTimeoutSeconds: common.Int32Ptr(2), - StartToCloseTimeoutSeconds: common.Int32Ptr(50), - HeartbeatTimeoutSeconds: common.Int32Ptr(5), - }, - }}, nil - } else if failureCount > 0 { - // Otherwise decrement failureCount and keep failing decisions - failureCount-- - return nil, nil, errors.New("Decider Panic") - } - - workflowComplete = true - time.Sleep(time.Second) - s.Logger.Warn(fmt.Sprintf("PrevStarted: %v, StartedEventID: %v, Size: %v", previousStartedEventID, startedEventID, - len(history.Events))) - lastDecisionEvent := history.Events[startedEventID-1] - s.Equal(types.EventTypeDecisionTaskStarted, lastDecisionEvent.GetEventType()) - lastDecisionTimestamp = lastDecisionEvent.GetTimestamp() - return nil, []*types.Decision{{ - DecisionType: types.DecisionTypeCompleteWorkflowExecution.Ptr(), - CompleteWorkflowExecutionDecisionAttributes: &types.CompleteWorkflowExecutionDecisionAttributes{ - Result: []byte("Done."), - }, - }}, nil - } - - // activity handler - atHandler := func(execution *types.WorkflowExecution, activityType *types.ActivityType, - ActivityID string, input []byte, taskToken []byte) ([]byte, bool, error) { - - return []byte("Activity Result."), false, nil - } - - poller := &TaskPoller{ - Engine: s.engine, - Domain: s.domainName, - TaskList: taskList, - Identity: identity, - DecisionHandler: dtHandler, - ActivityHandler: atHandler, - Logger: s.Logger, - T: s.T(), - } - - // Make first decision to schedule activity - _, err := poller.PollAndProcessDecisionTask(false, false) - s.Logger.Info("PollAndProcessDecisionTask", tag.Error(err)) - s.Nil(err) - - // process activity - err = poller.PollAndProcessActivityTask(false) - s.Logger.Info("PollAndProcessActivityTask", tag.Error(err)) - s.Nil(err) - - // fail decision 5 times - for i := 0; i < 5; i++ { - _, err := poller.PollAndProcessDecisionTaskWithAttempt(false, false, false, false, int64(i)) - s.Nil(err) - } - - err = s.sendSignal(s.domainName, workflowExecution, "signalA", nil, identity) - s.Nil(err, "failed to send signal to execution") - - // process signal - _, err = poller.PollAndProcessDecisionTask(false, false) - s.Logger.Info("PollAndProcessDecisionTask", tag.Error(err)) - s.Nil(err) - s.Equal(1, signalCount) - - // send another signal to trigger decision - err = s.sendSignal(s.domainName, workflowExecution, "signalB", nil, identity) - s.Nil(err, "failed to send signal to execution") - - // fail decision 2 more times - for i := 0; i < 2; i++ { - _, err := poller.PollAndProcessDecisionTaskWithAttempt(false, false, false, false, int64(i)) - s.Nil(err) - } - s.Equal(3, signalCount) - - // now send a signal during failed decision - sendSignal = true - _, err = poller.PollAndProcessDecisionTaskWithAttempt(false, false, false, false, int64(2)) - s.Nil(err) - s.Equal(4, signalCount) - - // fail decision 1 more times - for i := 0; i < 2; i++ { - _, err := poller.PollAndProcessDecisionTaskWithAttempt(false, false, false, false, int64(i)) - s.Nil(err) - } - s.Equal(12, signalCount) - - // Make complete workflow decision - _, err = poller.PollAndProcessDecisionTaskWithAttempt(true, false, false, false, int64(2)) - s.Logger.Info("pollAndProcessDecisionTask", tag.Error(err)) - s.Nil(err) - s.True(workflowComplete) - s.Equal(16, signalCount) - - events := s.getHistory(s.domainName, workflowExecution) - var lastEvent *types.HistoryEvent - var lastDecisionStartedEvent *types.HistoryEvent - lastIdx := 0 - for i, e := range events { - if e.GetEventType() == types.EventTypeDecisionTaskStarted { - lastDecisionStartedEvent = e - lastIdx = i - } - lastEvent = e - } - s.Equal(types.EventTypeWorkflowExecutionCompleted, lastEvent.GetEventType()) - s.Logger.Info(fmt.Sprintf("Last Decision Time: %v, Last Decision History Timestamp: %v, Complete Timestamp: %v", - time.Unix(0, lastDecisionTimestamp), time.Unix(0, lastDecisionStartedEvent.GetTimestamp()), - time.Unix(0, lastEvent.GetTimestamp()))) - s.Equal(lastDecisionTimestamp, lastDecisionStartedEvent.GetTimestamp()) - s.True(time.Duration(lastEvent.GetTimestamp()-lastDecisionTimestamp) >= time.Second) - - s.Equal(2, len(events)-lastIdx-1) - decisionCompletedEvent := events[lastIdx+1] - workflowCompletedEvent := events[lastIdx+2] - s.Equal(types.EventTypeDecisionTaskCompleted, decisionCompletedEvent.GetEventType()) - s.Equal(types.EventTypeWorkflowExecutionCompleted, workflowCompletedEvent.GetEventType()) -} - -func (s *IntegrationSuite) TestDescribeTaskList() { - WorkflowID := "integration-get-poller-history" - workflowTypeName := "integration-get-poller-history-type" - tasklistName := "integration-get-poller-history-tasklist" - identity := "worker1" - activityName := "activity_type1" - - workflowType := &types.WorkflowType{} - workflowType.Name = workflowTypeName - - taskList := &types.TaskList{} - taskList.Name = tasklistName - - // Start workflow execution - request := &types.StartWorkflowExecutionRequest{ - RequestID: uuid.New(), - Domain: s.domainName, - WorkflowID: WorkflowID, - WorkflowType: workflowType, - TaskList: taskList, - Input: nil, - ExecutionStartToCloseTimeoutSeconds: common.Int32Ptr(100), - TaskStartToCloseTimeoutSeconds: common.Int32Ptr(1), - Identity: identity, - } - - we, err0 := s.engine.StartWorkflowExecution(createContext(), request) - s.Nil(err0) - - s.Logger.Info("StartWorkflowExecution", tag.WorkflowRunID(we.RunID)) - - // decider logic - activityScheduled := false - activityData := int32(1) - // var signalEvent *types.HistoryEvent - dtHandler := func(execution *types.WorkflowExecution, wt *types.WorkflowType, - previousStartedEventID, startedEventID int64, history *types.History) ([]byte, []*types.Decision, error) { - - if !activityScheduled { - activityScheduled = true - buf := new(bytes.Buffer) - s.Nil(binary.Write(buf, binary.LittleEndian, activityData)) - - return nil, []*types.Decision{{ - DecisionType: types.DecisionTypeScheduleActivityTask.Ptr(), - ScheduleActivityTaskDecisionAttributes: &types.ScheduleActivityTaskDecisionAttributes{ - ActivityID: "1", - ActivityType: &types.ActivityType{Name: activityName}, - TaskList: taskList, - Input: buf.Bytes(), - ScheduleToCloseTimeoutSeconds: common.Int32Ptr(100), - ScheduleToStartTimeoutSeconds: common.Int32Ptr(25), - StartToCloseTimeoutSeconds: common.Int32Ptr(50), - HeartbeatTimeoutSeconds: common.Int32Ptr(25), - }, - }}, nil - } - - return nil, []*types.Decision{{ - DecisionType: types.DecisionTypeCompleteWorkflowExecution.Ptr(), - CompleteWorkflowExecutionDecisionAttributes: &types.CompleteWorkflowExecutionDecisionAttributes{ - Result: []byte("Done."), - }, - }}, nil - } - - atHandler := func(execution *types.WorkflowExecution, activityType *types.ActivityType, - ActivityID string, input []byte, taskToken []byte) ([]byte, bool, error) { - return []byte("Activity Result."), false, nil - } - - poller := &TaskPoller{ - Engine: s.engine, - Domain: s.domainName, - TaskList: taskList, - Identity: identity, - DecisionHandler: dtHandler, - ActivityHandler: atHandler, - Logger: s.Logger, - T: s.T(), - } - - // this function poll events from history side - testDescribeTaskList := func(domain string, tasklist *types.TaskList, tasklistType types.TaskListType) []*types.PollerInfo { - responseInner, errInner := s.engine.DescribeTaskList(createContext(), &types.DescribeTaskListRequest{ - Domain: domain, - TaskList: taskList, - TaskListType: &tasklistType, - }) - - s.Nil(errInner) - return responseInner.Pollers - } - - before := time.Now() - - // when no one polling on the tasklist (activity or decition), there shall be no poller information - pollerInfos := testDescribeTaskList(s.domainName, taskList, types.TaskListTypeActivity) - s.Empty(pollerInfos) - pollerInfos = testDescribeTaskList(s.domainName, taskList, types.TaskListTypeDecision) - s.Empty(pollerInfos) - - _, errDecision := poller.PollAndProcessDecisionTask(false, false) - s.Nil(errDecision) - pollerInfos = testDescribeTaskList(s.domainName, taskList, types.TaskListTypeActivity) - s.Empty(pollerInfos) - pollerInfos = testDescribeTaskList(s.domainName, taskList, types.TaskListTypeDecision) - s.Equal(1, len(pollerInfos)) - s.Equal(identity, pollerInfos[0].GetIdentity()) - s.True(time.Unix(0, pollerInfos[0].GetLastAccessTime()).After(before)) - s.NotEmpty(pollerInfos[0].GetLastAccessTime()) - - errActivity := poller.PollAndProcessActivityTask(false) - s.Nil(errActivity) - pollerInfos = testDescribeTaskList(s.domainName, taskList, types.TaskListTypeActivity) - s.Equal(1, len(pollerInfos)) - s.Equal(identity, pollerInfos[0].GetIdentity()) - s.True(time.Unix(0, pollerInfos[0].GetLastAccessTime()).After(before)) - s.NotEmpty(pollerInfos[0].GetLastAccessTime()) - pollerInfos = testDescribeTaskList(s.domainName, taskList, types.TaskListTypeDecision) - s.Equal(1, len(pollerInfos)) - s.Equal(identity, pollerInfos[0].GetIdentity()) - s.True(time.Unix(0, pollerInfos[0].GetLastAccessTime()).After(before)) - s.NotEmpty(pollerInfos[0].GetLastAccessTime()) -} - -func (s *IntegrationSuite) TestTransientDecisionTimeout() { - id := "integration-transient-decision-timeout-test" - wt := "integration-transient-decision-timeout-test-type" - tl := "integration-transient-decision-timeout-test-tasklist" - identity := "worker1" - - workflowType := &types.WorkflowType{} - workflowType.Name = wt - - taskList := &types.TaskList{} - taskList.Name = tl - - // Start workflow execution - request := &types.StartWorkflowExecutionRequest{ - RequestID: uuid.New(), - Domain: s.domainName, - WorkflowID: id, - WorkflowType: workflowType, - TaskList: taskList, - Input: nil, - ExecutionStartToCloseTimeoutSeconds: common.Int32Ptr(100), - TaskStartToCloseTimeoutSeconds: common.Int32Ptr(2), - Identity: identity, - } - - we, err0 := s.engine.StartWorkflowExecution(createContext(), request) - s.Nil(err0) - s.Logger.Info("StartWorkflowExecution", tag.WorkflowRunID(we.RunID)) - - workflowExecution := &types.WorkflowExecution{ - WorkflowID: id, - RunID: we.RunID, - } - - // decider logic - workflowComplete := false - failDecision := true - signalCount := 0 - //var signalEvent *types.HistoryEvent - dtHandler := func(execution *types.WorkflowExecution, wt *types.WorkflowType, - previousStartedEventID, startedEventID int64, history *types.History) ([]byte, []*types.Decision, error) { - if failDecision { - failDecision = false - return nil, nil, errors.New("Decider Panic") - } - - // Count signals - for _, event := range history.Events[previousStartedEventID:] { - if event.GetEventType() == types.EventTypeWorkflowExecutionSignaled { - signalCount++ - } - } - - workflowComplete = true - return nil, []*types.Decision{{ - DecisionType: types.DecisionTypeCompleteWorkflowExecution.Ptr(), - CompleteWorkflowExecutionDecisionAttributes: &types.CompleteWorkflowExecutionDecisionAttributes{ - Result: []byte("Done."), - }, - }}, nil - } - - poller := &TaskPoller{ - Engine: s.engine, - Domain: s.domainName, - TaskList: taskList, - Identity: identity, - DecisionHandler: dtHandler, - ActivityHandler: nil, - Logger: s.Logger, - T: s.T(), - } - - // First decision immediately fails and schedules a transient decision - _, err := poller.PollAndProcessDecisionTask(false, false) - s.Logger.Info("PollAndProcessDecisionTask", tag.Error(err)) - s.Nil(err) - - // Now send a signal when transient decision is scheduled - err = s.sendSignal(s.domainName, workflowExecution, "signalA", nil, identity) - s.Nil(err, "failed to send signal to execution") - - // Drop decision task to cause a Decision Timeout - _, err = poller.PollAndProcessDecisionTask(false, true) - s.Logger.Info("PollAndProcessDecisionTask", tag.Error(err)) - s.Nil(err) - - // Now process signal and complete workflow execution - _, err = poller.PollAndProcessDecisionTaskWithAttempt(true, false, false, false, int64(1)) - s.Logger.Info("PollAndProcessDecisionTask", tag.Error(err)) - s.Nil(err) - - s.Equal(1, signalCount) - s.True(workflowComplete) -} - -func (s *IntegrationSuite) TestNoTransientDecisionAfterFlushBufferedEvents() { - id := "integration-no-transient-decision-after-flush-buffered-events-test" - wt := "integration-no-transient-decision-after-flush-buffered-events-test-type" - tl := "integration-no-transient-decision-after-flush-buffered-events-test-tasklist" - identity := "worker1" - - workflowType := &types.WorkflowType{Name: wt} - taskList := &types.TaskList{Name: tl} - - // Start workflow execution - request := &types.StartWorkflowExecutionRequest{ - RequestID: uuid.New(), - Domain: s.domainName, - WorkflowID: id, - WorkflowType: workflowType, - TaskList: taskList, - Input: nil, - ExecutionStartToCloseTimeoutSeconds: common.Int32Ptr(100), - TaskStartToCloseTimeoutSeconds: common.Int32Ptr(20), - Identity: identity, - } - - we, err0 := s.engine.StartWorkflowExecution(createContext(), request) - s.Nil(err0) - - s.Logger.Info("StartWorkflowExecution", tag.WorkflowRunID(we.RunID)) - - // decider logic - workflowComplete := false - continueAsNewAndSignal := false - dtHandler := func(execution *types.WorkflowExecution, wt *types.WorkflowType, - previousStartedEventID, startedEventID int64, history *types.History) ([]byte, []*types.Decision, error) { - - if !continueAsNewAndSignal { - continueAsNewAndSignal = true - // this will create new event when there is in-flight decision task, and the new event will be buffered - err := s.engine.SignalWorkflowExecution(createContext(), - &types.SignalWorkflowExecutionRequest{ - Domain: s.domainName, - WorkflowExecution: &types.WorkflowExecution{ - WorkflowID: id, - }, - SignalName: "buffered-signal-1", - Input: []byte("buffered-signal-input"), - Identity: identity, - }) - s.NoError(err) - - return nil, []*types.Decision{{ - DecisionType: types.DecisionTypeContinueAsNewWorkflowExecution.Ptr(), - ContinueAsNewWorkflowExecutionDecisionAttributes: &types.ContinueAsNewWorkflowExecutionDecisionAttributes{ - WorkflowType: workflowType, - TaskList: taskList, - Input: nil, - ExecutionStartToCloseTimeoutSeconds: common.Int32Ptr(1000), - TaskStartToCloseTimeoutSeconds: common.Int32Ptr(100), - }, - }}, nil - } - - workflowComplete = true - return nil, []*types.Decision{{ - DecisionType: types.DecisionTypeCompleteWorkflowExecution.Ptr(), - CompleteWorkflowExecutionDecisionAttributes: &types.CompleteWorkflowExecutionDecisionAttributes{ - Result: []byte("Done."), - }, - }}, nil - } - - poller := &TaskPoller{ - Engine: s.engine, - Domain: s.domainName, - TaskList: taskList, - Identity: identity, - DecisionHandler: dtHandler, - Logger: s.Logger, - T: s.T(), - } - - // fist decision, this try to do a continue as new but there is a buffered event, - // so it will fail and create a new decision - _, err := poller.PollAndProcessDecisionTask(false, false) - s.Logger.Info("pollAndProcessDecisionTask", tag.Error(err)) - s.Nil(err) - - // second decision, which will complete the workflow - // this expect the decision to have attempt == 0 - _, err = poller.PollAndProcessDecisionTaskWithAttempt(true, false, false, false, 0) - s.Logger.Info("pollAndProcessDecisionTask", tag.Error(err)) - s.Nil(err) - - s.True(workflowComplete) -} - -func (s *IntegrationSuite) TestRelayDecisionTimeout() { - id := "integration-relay-decision-timeout-test" - wt := "integration-relay-decision-timeout-test-type" - tl := "integration-relay-decision-timeout-test-tasklist" - identity := "worker1" - - workflowType := &types.WorkflowType{} - workflowType.Name = wt - - taskList := &types.TaskList{} - taskList.Name = tl - - // Start workflow execution - request := &types.StartWorkflowExecutionRequest{ - RequestID: uuid.New(), - Domain: s.domainName, - WorkflowID: id, - WorkflowType: workflowType, - TaskList: taskList, - Input: nil, - ExecutionStartToCloseTimeoutSeconds: common.Int32Ptr(100), - TaskStartToCloseTimeoutSeconds: common.Int32Ptr(2), - Identity: identity, - } - - we, err0 := s.engine.StartWorkflowExecution(createContext(), request) - s.Nil(err0) - s.Logger.Info("StartWorkflowExecution", tag.WorkflowRunID(we.RunID)) - - workflowExecution := &types.WorkflowExecution{ - WorkflowID: id, - RunID: we.RunID, - } - - workflowComplete, isFirst := false, true - dtHandler := func(execution *types.WorkflowExecution, wt *types.WorkflowType, - previousStartedEventID, startedEventID int64, history *types.History) ([]byte, []*types.Decision, error) { - if isFirst { - isFirst = false - return nil, []*types.Decision{{ - DecisionType: types.DecisionTypeRecordMarker.Ptr(), - RecordMarkerDecisionAttributes: &types.RecordMarkerDecisionAttributes{ - MarkerName: "test-marker", - }, - }}, nil - } - workflowComplete = true - return nil, []*types.Decision{{ - DecisionType: types.DecisionTypeCompleteWorkflowExecution.Ptr(), - CompleteWorkflowExecutionDecisionAttributes: &types.CompleteWorkflowExecutionDecisionAttributes{}, - }}, nil - } - - poller := &TaskPoller{ - Engine: s.engine, - Domain: s.domainName, - TaskList: taskList, - Identity: identity, - DecisionHandler: dtHandler, - ActivityHandler: nil, - Logger: s.Logger, - T: s.T(), - } - - // First decision task complete with a marker decision, and request to relay decision (immediately return a new decision task) - _, newTask, err := poller.PollAndProcessDecisionTaskWithAttemptAndRetryAndForceNewDecision( - false, - false, - false, - false, - 0, - 3, - true, - nil) - s.Logger.Info("PollAndProcessDecisionTask", tag.Error(err)) - s.Nil(err) - s.NotNil(newTask) - s.NotNil(newTask.DecisionTask) - - time.Sleep(time.Second * 2) // wait 2s for relay decision to timeout - decisionTaskTimeout := false - for i := 0; i < 3; i++ { - events := s.getHistory(s.domainName, workflowExecution) - if len(events) >= 8 { - s.Equal(types.EventTypeDecisionTaskTimedOut, events[7].GetEventType()) - s.Equal(types.TimeoutTypeStartToClose, events[7].DecisionTaskTimedOutEventAttributes.GetTimeoutType()) - decisionTaskTimeout = true - break - } - time.Sleep(time.Second) - } - // verify relay decision task timeout - s.True(decisionTaskTimeout) - - // Now complete workflow - _, err = poller.PollAndProcessDecisionTaskWithAttempt(true, false, false, false, int64(1)) - s.Logger.Info("PollAndProcessDecisionTask", tag.Error(err)) - s.Nil(err) - - s.True(workflowComplete) -} - -func (s *IntegrationSuite) TestTaskProcessingProtectionForRateLimitError() { - id := "integration-task-processing-protection-for-rate-limit-error-test" - wt := "integration-task-processing-protection-for-rate-limit-error-test-type" - tl := "integration-task-processing-protection-for-rate-limit-error-test-tasklist" - identity := "worker1" - - workflowType := &types.WorkflowType{} - workflowType.Name = wt - - taskList := &types.TaskList{} - taskList.Name = tl - - // Start workflow execution - request := &types.StartWorkflowExecutionRequest{ - RequestID: uuid.New(), - Domain: s.domainName, - WorkflowID: id, - WorkflowType: workflowType, - TaskList: taskList, - Input: nil, - ExecutionStartToCloseTimeoutSeconds: common.Int32Ptr(601), - TaskStartToCloseTimeoutSeconds: common.Int32Ptr(600), - Identity: identity, - } - - we, err0 := s.engine.StartWorkflowExecution(createContext(), request) - s.Nil(err0) - - s.Logger.Info("StartWorkflowExecution", tag.WorkflowRunID(we.RunID)) - workflowExecution := &types.WorkflowExecution{ - WorkflowID: id, - RunID: we.RunID, - } - - // decider logic - workflowComplete := false - signalCount := 0 - createUserTimer := false - dtHandler := func(execution *types.WorkflowExecution, wt *types.WorkflowType, - previousStartedEventID, startedEventID int64, h *types.History) ([]byte, []*types.Decision, error) { - - if !createUserTimer { - createUserTimer = true - - return nil, []*types.Decision{{ - DecisionType: types.DecisionTypeStartTimer.Ptr(), - StartTimerDecisionAttributes: &types.StartTimerDecisionAttributes{ - TimerID: "timer-id-1", - StartToFireTimeoutSeconds: common.Int64Ptr(5), - }, - }}, nil - } - - // Count signals - for _, event := range h.Events[previousStartedEventID:] { - if event.GetEventType() == types.EventTypeWorkflowExecutionSignaled { - signalCount++ - } - } - - workflowComplete = true - return nil, []*types.Decision{{ - DecisionType: types.DecisionTypeCompleteWorkflowExecution.Ptr(), - CompleteWorkflowExecutionDecisionAttributes: &types.CompleteWorkflowExecutionDecisionAttributes{ - Result: []byte("Done."), - }, - }}, nil - } - - poller := &TaskPoller{ - Engine: s.engine, - Domain: s.domainName, - TaskList: taskList, - Identity: identity, - DecisionHandler: dtHandler, - ActivityHandler: nil, - Logger: s.Logger, - T: s.T(), - } - - // Process first decision to create user timer - _, err := poller.PollAndProcessDecisionTask(false, false) - s.Logger.Info("PollAndProcessDecisionTask", tag.Error(err)) - s.Nil(err) - - // Send one signal to create a new decision - buf := new(bytes.Buffer) - binary.Write(buf, binary.LittleEndian, 0) - s.Nil(s.sendSignal(s.domainName, workflowExecution, "SignalName", buf.Bytes(), identity)) - - // Drop decision to cause all events to be buffered from now on - _, err = poller.PollAndProcessDecisionTask(false, true) - s.Logger.Info("PollAndProcessDecisionTask", tag.Error(err)) - s.Nil(err) - - // Buffered 100 Signals - for i := 1; i < 101; i++ { - buf := new(bytes.Buffer) - binary.Write(buf, binary.LittleEndian, i) - s.Nil(s.sendSignal(s.domainName, workflowExecution, "SignalName", buf.Bytes(), identity)) - } - - // 101 signal, which will fail the decision - buf = new(bytes.Buffer) - binary.Write(buf, binary.LittleEndian, 101) - signalErr := s.sendSignal(s.domainName, workflowExecution, "SignalName", buf.Bytes(), identity) - s.Nil(signalErr) - - // Process signal in decider - _, err = poller.PollAndProcessDecisionTaskWithAttempt(true, false, false, false, 0) - s.Logger.Info("pollAndProcessDecisionTask", tag.Error(err)) - s.Nil(err) - - s.True(workflowComplete) - s.Equal(102, signalCount) -} - -func (s *IntegrationSuite) TestStickyTimeout_NonTransientDecision() { - id := "integration-sticky-timeout-non-transient-decision" - wt := "integration-sticky-timeout-non-transient-decision-type" - tl := "integration-sticky-timeout-non-transient-decision-tasklist" - stl := "integration-sticky-timeout-non-transient-decision-tasklist-sticky" - identity := "worker1" - - workflowType := &types.WorkflowType{} - workflowType.Name = wt - - taskList := &types.TaskList{} - taskList.Name = tl - - stickyTaskList := &types.TaskList{} - stickyTaskList.Name = stl - stickyScheduleToStartTimeoutSeconds := common.Int32Ptr(2) - - // Start workflow execution - request := &types.StartWorkflowExecutionRequest{ - RequestID: uuid.New(), - Domain: s.domainName, - WorkflowID: id, - WorkflowType: workflowType, - TaskList: taskList, - Input: nil, - ExecutionStartToCloseTimeoutSeconds: common.Int32Ptr(100), - TaskStartToCloseTimeoutSeconds: common.Int32Ptr(1), - Identity: identity, - } - - we, err0 := s.engine.StartWorkflowExecution(createContext(), request) - s.Nil(err0) - - s.Logger.Info("StartWorkflowExecution", tag.WorkflowRunID(we.RunID)) - workflowExecution := &types.WorkflowExecution{ - WorkflowID: id, - RunID: we.RunID, - } - - // decider logic - localActivityDone := false - failureCount := 5 - dtHandler := func(execution *types.WorkflowExecution, wt *types.WorkflowType, - previousStartedEventID, startedEventID int64, history *types.History) ([]byte, []*types.Decision, error) { - - if !localActivityDone { - localActivityDone = true - - return nil, []*types.Decision{{ - DecisionType: types.DecisionTypeRecordMarker.Ptr(), - RecordMarkerDecisionAttributes: &types.RecordMarkerDecisionAttributes{ - MarkerName: "local activity marker", - Details: []byte("local activity data"), - }, - }}, nil - } - - if failureCount > 0 { - // send a signal on third failure to be buffered, forcing a non-transient decision when buffer is flushed - /*if failureCount == 3 { - err := s.engine.SignalWorkflowExecution(createContext(), &types.SignalWorkflowExecutionRequest{ - Domain: s.domainName, - WorkflowExecution: workflowExecution, - SignalName: common.StringPtr("signalB"), - Input: []byte("signal input"), - Identity: identity, - RequestID: uuid.New(), - }) - s.Nil(err) - }*/ - failureCount-- - return nil, nil, errors.New("non deterministic error") - } - - return nil, []*types.Decision{{ - DecisionType: types.DecisionTypeCompleteWorkflowExecution.Ptr(), - CompleteWorkflowExecutionDecisionAttributes: &types.CompleteWorkflowExecutionDecisionAttributes{ - Result: []byte("Done."), - }, - }}, nil - } - - poller := &TaskPoller{ - Engine: s.engine, - Domain: s.domainName, - TaskList: taskList, - Identity: identity, - DecisionHandler: dtHandler, - Logger: s.Logger, - T: s.T(), - StickyTaskList: stickyTaskList, - StickyScheduleToStartTimeoutSeconds: stickyScheduleToStartTimeoutSeconds, - } - - _, err := poller.PollAndProcessDecisionTaskWithAttempt(false, false, false, true, int64(0)) - s.Logger.Info("PollAndProcessDecisionTask", tag.Error(err)) - s.Nil(err) - - err = s.engine.SignalWorkflowExecution(createContext(), &types.SignalWorkflowExecutionRequest{ - Domain: s.domainName, - WorkflowExecution: workflowExecution, - SignalName: "signalA", - Input: []byte("signal input"), - Identity: identity, - RequestID: uuid.New(), - }) - - // Wait for decision timeout - stickyTimeout := false -WaitForStickyTimeoutLoop: - for i := 0; i < 10; i++ { - events := s.getHistory(s.domainName, workflowExecution) - for _, event := range events { - if event.GetEventType() == types.EventTypeDecisionTaskTimedOut { - s.Equal(types.TimeoutTypeScheduleToStart, event.DecisionTaskTimedOutEventAttributes.GetTimeoutType()) - stickyTimeout = true - break WaitForStickyTimeoutLoop - } - } - time.Sleep(time.Second) - } - s.True(stickyTimeout, "Decision not timed out.") - - for i := 0; i < 3; i++ { - _, err = poller.PollAndProcessDecisionTaskWithAttempt(true, false, false, true, int64(i)) - s.Logger.Info("PollAndProcessDecisionTask", tag.Error(err)) - s.Nil(err) - } - - err = s.engine.SignalWorkflowExecution(createContext(), &types.SignalWorkflowExecutionRequest{ - Domain: s.domainName, - WorkflowExecution: workflowExecution, - SignalName: "signalB", - Input: []byte("signal input"), - Identity: identity, - RequestID: uuid.New(), - }) - s.Nil(err) - - for i := 0; i < 2; i++ { - _, err = poller.PollAndProcessDecisionTaskWithAttempt(true, false, false, true, int64(i)) - s.Logger.Info("PollAndProcessDecisionTask", tag.Error(err)) - s.Nil(err) - } - - decisionTaskFailed := false - events := s.getHistory(s.domainName, workflowExecution) - for _, event := range events { - if event.GetEventType() == types.EventTypeDecisionTaskFailed { - decisionTaskFailed = true - break - } - } - s.True(decisionTaskFailed) - - // Complete workflow execution - _, err = poller.PollAndProcessDecisionTaskWithAttempt(true, false, false, true, int64(2)) - - // Assert for single decision task failed and workflow completion - failedDecisions := 0 - workflowComplete := false - events = s.getHistory(s.domainName, workflowExecution) - for _, event := range events { - switch event.GetEventType() { - case types.EventTypeDecisionTaskFailed: - failedDecisions++ - case types.EventTypeWorkflowExecutionCompleted: - workflowComplete = true - } - } - s.True(workflowComplete, "Workflow not complete") - s.Equal(2, failedDecisions, "Mismatched failed decision count") -} - -func (s *IntegrationSuite) TestStickyTasklistResetThenTimeout() { - id := "integration-reset-sticky-fire-schedule-to-start-timeout" - wt := "integration-reset-sticky-fire-schedule-to-start-timeout-type" - tl := "integration-reset-sticky-fire-schedule-to-start-timeout-tasklist" - stl := "integration-reset-sticky-fire-schedule-to-start-timeout-tasklist-sticky" - identity := "worker1" - - workflowType := &types.WorkflowType{} - workflowType.Name = wt - - taskList := &types.TaskList{} - taskList.Name = tl - - stickyTaskList := &types.TaskList{} - stickyTaskList.Name = stl - stickyScheduleToStartTimeoutSeconds := common.Int32Ptr(2) - - // Start workflow execution - request := &types.StartWorkflowExecutionRequest{ - RequestID: uuid.New(), - Domain: s.domainName, - WorkflowID: id, - WorkflowType: workflowType, - TaskList: taskList, - Input: nil, - ExecutionStartToCloseTimeoutSeconds: common.Int32Ptr(100), - TaskStartToCloseTimeoutSeconds: common.Int32Ptr(1), - Identity: identity, - } - - we, err0 := s.engine.StartWorkflowExecution(createContext(), request) - s.Nil(err0) - - s.Logger.Info("StartWorkflowExecution", tag.WorkflowRunID(we.RunID)) - workflowExecution := &types.WorkflowExecution{ - WorkflowID: id, - RunID: we.RunID, - } - - // decider logic - localActivityDone := false - failureCount := 5 - dtHandler := func(execution *types.WorkflowExecution, wt *types.WorkflowType, - previousStartedEventID, startedEventID int64, history *types.History) ([]byte, []*types.Decision, error) { - - if !localActivityDone { - localActivityDone = true - - return nil, []*types.Decision{{ - DecisionType: types.DecisionTypeRecordMarker.Ptr(), - RecordMarkerDecisionAttributes: &types.RecordMarkerDecisionAttributes{ - MarkerName: "local activity marker", - Details: []byte("local activity data"), - }, - }}, nil - } - - if failureCount > 0 { - failureCount-- - return nil, nil, errors.New("non deterministic error") - } - - return nil, []*types.Decision{{ - DecisionType: types.DecisionTypeCompleteWorkflowExecution.Ptr(), - CompleteWorkflowExecutionDecisionAttributes: &types.CompleteWorkflowExecutionDecisionAttributes{ - Result: []byte("Done."), - }, - }}, nil - } - - poller := &TaskPoller{ - Engine: s.engine, - Domain: s.domainName, - TaskList: taskList, - Identity: identity, - DecisionHandler: dtHandler, - Logger: s.Logger, - T: s.T(), - StickyTaskList: stickyTaskList, - StickyScheduleToStartTimeoutSeconds: stickyScheduleToStartTimeoutSeconds, - } - - _, err := poller.PollAndProcessDecisionTaskWithAttempt(false, false, false, true, int64(0)) - s.Logger.Info("PollAndProcessDecisionTask: %v", tag.Error(err)) - s.Nil(err) - - err = s.engine.SignalWorkflowExecution(createContext(), &types.SignalWorkflowExecutionRequest{ - Domain: s.domainName, - WorkflowExecution: workflowExecution, - SignalName: "signalA", - Input: []byte("signal input"), - Identity: identity, - RequestID: uuid.New(), - }) - - //Reset sticky tasklist before sticky decision task starts - s.engine.ResetStickyTaskList(createContext(), &types.ResetStickyTaskListRequest{ - Domain: s.domainName, - Execution: workflowExecution, - }) - - // Wait for decision timeout - stickyTimeout := false -WaitForStickyTimeoutLoop: - for i := 0; i < 10; i++ { - events := s.getHistory(s.domainName, workflowExecution) - for _, event := range events { - if event.GetEventType() == types.EventTypeDecisionTaskTimedOut { - s.Equal(types.TimeoutTypeScheduleToStart, event.DecisionTaskTimedOutEventAttributes.GetTimeoutType()) - stickyTimeout = true - break WaitForStickyTimeoutLoop - } - } - time.Sleep(time.Second) - } - s.True(stickyTimeout, "Decision not timed out.") - - for i := 0; i < 3; i++ { - _, err = poller.PollAndProcessDecisionTaskWithAttempt(true, false, false, true, int64(i)) - s.Logger.Info("PollAndProcessDecisionTask: %v", tag.Error(err)) - s.Nil(err) - } - - err = s.engine.SignalWorkflowExecution(createContext(), &types.SignalWorkflowExecutionRequest{ - Domain: s.domainName, - WorkflowExecution: workflowExecution, - SignalName: "signalB", - Input: []byte("signal input"), - Identity: identity, - RequestID: uuid.New(), - }) - s.Nil(err) - - for i := 0; i < 2; i++ { - _, err = poller.PollAndProcessDecisionTaskWithAttempt(true, false, false, true, int64(i)) - s.Logger.Info("PollAndProcessDecisionTask: %v", tag.Error(err)) - s.Nil(err) - } - - decisionTaskFailed := false - events := s.getHistory(s.domainName, workflowExecution) - for _, event := range events { - if event.GetEventType() == types.EventTypeDecisionTaskFailed { - decisionTaskFailed = true - break - } - } - s.True(decisionTaskFailed) - - // Complete workflow execution - _, err = poller.PollAndProcessDecisionTaskWithAttempt(true, false, false, true, int64(2)) - - // Assert for single decision task failed and workflow completion - failedDecisions := 0 - workflowComplete := false - events = s.getHistory(s.domainName, workflowExecution) - for _, event := range events { - switch event.GetEventType() { - case types.EventTypeDecisionTaskFailed: - failedDecisions++ - case types.EventTypeWorkflowExecutionCompleted: - workflowComplete = true - } - } - s.True(workflowComplete, "Workflow not complete") - s.Equal(2, failedDecisions, "Mismatched failed decision count") -} - -func (s *IntegrationSuite) TestBufferedEventsOutOfOrder() { - id := "integration-buffered-events-out-of-order-test" - wt := "integration-buffered-events-out-of-order-test-type" - tl := "integration-buffered-events-out-of-order-test-tasklist" - identity := "worker1" - - workflowType := &types.WorkflowType{Name: wt} - taskList := &types.TaskList{Name: tl} - - // Start workflow execution - request := &types.StartWorkflowExecutionRequest{ - RequestID: uuid.New(), - Domain: s.domainName, - WorkflowID: id, - WorkflowType: workflowType, - TaskList: taskList, - Input: nil, - ExecutionStartToCloseTimeoutSeconds: common.Int32Ptr(100), - TaskStartToCloseTimeoutSeconds: common.Int32Ptr(20), - Identity: identity, - } - - we, err0 := s.engine.StartWorkflowExecution(createContext(), request) - s.Nil(err0) - - s.Logger.Info("StartWorkflowExecution", tag.WorkflowRunID(we.RunID)) - workflowExecution := &types.WorkflowExecution{ - WorkflowID: id, - RunID: we.RunID, - } - - // decider logic - workflowComplete := false - firstDecision := false - secondDecision := false - dtHandler := func(execution *types.WorkflowExecution, wt *types.WorkflowType, - previousStartedEventID, startedEventID int64, history *types.History) ([]byte, []*types.Decision, error) { - - s.Logger.Info(fmt.Sprintf("Decider called: first: %v, second: %v, complete: %v\n", firstDecision, secondDecision, workflowComplete)) - - if !firstDecision { - firstDecision = true - return nil, []*types.Decision{{ - DecisionType: types.DecisionTypeRecordMarker.Ptr(), - RecordMarkerDecisionAttributes: &types.RecordMarkerDecisionAttributes{ - MarkerName: "some random marker name", - Details: []byte("some random marker details"), - }, - }, { - DecisionType: types.DecisionTypeScheduleActivityTask.Ptr(), - ScheduleActivityTaskDecisionAttributes: &types.ScheduleActivityTaskDecisionAttributes{ - ActivityID: "Activity-1", - ActivityType: &types.ActivityType{Name: "ActivityType"}, - Domain: s.domainName, - TaskList: taskList, - Input: []byte("some random activity input"), - ScheduleToCloseTimeoutSeconds: common.Int32Ptr(100), - ScheduleToStartTimeoutSeconds: common.Int32Ptr(100), - StartToCloseTimeoutSeconds: common.Int32Ptr(100), - HeartbeatTimeoutSeconds: common.Int32Ptr(100), - }, - }}, nil - } - - if !secondDecision { - secondDecision = true - return nil, []*types.Decision{{ - DecisionType: types.DecisionTypeRecordMarker.Ptr(), - RecordMarkerDecisionAttributes: &types.RecordMarkerDecisionAttributes{ - MarkerName: "some random marker name", - Details: []byte("some random marker details"), - }, - }}, nil - } - - workflowComplete = true - return nil, []*types.Decision{{ - DecisionType: types.DecisionTypeCompleteWorkflowExecution.Ptr(), - CompleteWorkflowExecutionDecisionAttributes: &types.CompleteWorkflowExecutionDecisionAttributes{ - Result: []byte("Done."), - }, - }}, nil - } - // activity handler - atHandler := func(execution *types.WorkflowExecution, activityType *types.ActivityType, - ActivityID string, input []byte, taskToken []byte) ([]byte, bool, error) { - return []byte("Activity Result."), false, nil - } - - poller := &TaskPoller{ - Engine: s.engine, - Domain: s.domainName, - TaskList: taskList, - Identity: identity, - DecisionHandler: dtHandler, - ActivityHandler: atHandler, - Logger: s.Logger, - T: s.T(), - } - - // first decision, which will schedule an activity and add marker - _, task, err := poller.PollAndProcessDecisionTaskWithAttemptAndRetryAndForceNewDecision( - true, - false, - false, - false, - int64(0), - 1, - true, - nil) - s.Logger.Info("pollAndProcessDecisionTask", tag.Error(err)) - s.Nil(err) - - // This will cause activity start and complete to be buffered - err = poller.PollAndProcessActivityTask(false) - s.Logger.Info("pollAndProcessActivityTask", tag.Error(err)) - s.Nil(err) - - // second decision, completes another local activity and forces flush of buffered activity events - newDecisionTask := task.GetDecisionTask() - s.NotNil(newDecisionTask) - task, err = poller.HandlePartialDecision(newDecisionTask) - s.Logger.Info("pollAndProcessDecisionTask", tag.Error(err)) - s.Nil(err) - s.NotNil(task) - - // third decision, which will close workflow - newDecisionTask = task.GetDecisionTask() - s.NotNil(newDecisionTask) - task, err = poller.HandlePartialDecision(newDecisionTask) - s.Logger.Info("pollAndProcessDecisionTask", tag.Error(err)) - s.Nil(err) - s.Nil(task.DecisionTask) - - events := s.getHistory(s.domainName, workflowExecution) - var scheduleEvent, startedEvent, completedEvent *types.HistoryEvent - for _, event := range events { - switch event.GetEventType() { - case types.EventTypeActivityTaskScheduled: - scheduleEvent = event - case types.EventTypeActivityTaskStarted: - startedEvent = event - case types.EventTypeActivityTaskCompleted: - completedEvent = event - } - } - - s.NotNil(scheduleEvent) - s.NotNil(startedEvent) - s.NotNil(completedEvent) - s.True(startedEvent.GetEventID() < completedEvent.GetEventID()) - s.Equal(scheduleEvent.GetEventID(), startedEvent.ActivityTaskStartedEventAttributes.GetScheduledEventID()) - s.Equal(scheduleEvent.GetEventID(), completedEvent.ActivityTaskCompletedEventAttributes.GetScheduledEventID()) - s.Equal(startedEvent.GetEventID(), completedEvent.ActivityTaskCompletedEventAttributes.GetStartedEventID()) - s.True(workflowComplete) -} - -type startFunc func() (*types.StartWorkflowExecutionResponse, error) - -func (s *IntegrationSuite) TestStartWithMemo() { - id := "integration-start-with-memo-test" - wt := "integration-start-with-memo-test-type" - tl := "integration-start-with-memo-test-tasklist" - identity := "worker1" - - workflowType := &types.WorkflowType{} - workflowType.Name = wt - - taskList := &types.TaskList{} - taskList.Name = tl - - memoInfo, _ := json.Marshal(id) - memo := &types.Memo{ - Fields: map[string][]byte{ - "Info": memoInfo, - }, - } - - request := &types.StartWorkflowExecutionRequest{ - RequestID: uuid.New(), - Domain: s.domainName, - WorkflowID: id, - WorkflowType: workflowType, - TaskList: taskList, - Input: nil, - ExecutionStartToCloseTimeoutSeconds: common.Int32Ptr(100), - TaskStartToCloseTimeoutSeconds: common.Int32Ptr(1), - Identity: identity, - Memo: memo, - } - - fn := func() (*types.StartWorkflowExecutionResponse, error) { - return s.engine.StartWorkflowExecution(createContext(), request) - } - s.startWithMemoHelper(fn, id, taskList, memo) -} - -func (s *IntegrationSuite) TestSignalWithStartWithMemo() { - id := "integration-signal-with-start-with-memo-test" - wt := "integration-signal-with-start-with-memo-test-type" - tl := "integration-signal-with-start-with-memo-test-tasklist" - identity := "worker1" - - workflowType := &types.WorkflowType{} - workflowType.Name = wt - - taskList := &types.TaskList{} - taskList.Name = tl - - memoInfo, _ := json.Marshal(id) - memo := &types.Memo{ - Fields: map[string][]byte{ - "Info": memoInfo, - }, - } - - signalName := "my signal" - signalInput := []byte("my signal input.") - request := &types.SignalWithStartWorkflowExecutionRequest{ - RequestID: uuid.New(), - Domain: s.domainName, - WorkflowID: id, - WorkflowType: workflowType, - TaskList: taskList, - Input: nil, - ExecutionStartToCloseTimeoutSeconds: common.Int32Ptr(100), - TaskStartToCloseTimeoutSeconds: common.Int32Ptr(1), - SignalName: signalName, - SignalInput: signalInput, - Identity: identity, - Memo: memo, - } - - fn := func() (*types.StartWorkflowExecutionResponse, error) { - return s.engine.SignalWithStartWorkflowExecution(createContext(), request) - } - s.startWithMemoHelper(fn, id, taskList, memo) -} - -func (s *IntegrationSuite) TestCancelTimer() { - id := "integration-cancel-timer-test" - wt := "integration-cancel-timer-test-type" - tl := "integration-cancel-timer-test-tasklist" - identity := "worker1" - - workflowType := &types.WorkflowType{} - workflowType.Name = wt - - taskList := &types.TaskList{} - taskList.Name = tl - - request := &types.StartWorkflowExecutionRequest{ - RequestID: uuid.New(), - Domain: s.domainName, - WorkflowID: id, - WorkflowType: workflowType, - TaskList: taskList, - Input: nil, - ExecutionStartToCloseTimeoutSeconds: common.Int32Ptr(100), - TaskStartToCloseTimeoutSeconds: common.Int32Ptr(1000), - Identity: identity, - } - - creatResp, err0 := s.engine.StartWorkflowExecution(createContext(), request) - s.Nil(err0) - workflowExecution := &types.WorkflowExecution{ - WorkflowID: id, - RunID: creatResp.GetRunID(), - } - - TimerID := "1" - timerScheduled := false - signalDelivered := false - timerCancelled := false - workflowComplete := false - timer := int64(2000) - dtHandler := func(execution *types.WorkflowExecution, wt *types.WorkflowType, - previousStartedEventID, startedEventID int64, history *types.History) ([]byte, []*types.Decision, error) { - - if !timerScheduled { - timerScheduled = true - return nil, []*types.Decision{{ - DecisionType: types.DecisionTypeStartTimer.Ptr(), - StartTimerDecisionAttributes: &types.StartTimerDecisionAttributes{ - TimerID: TimerID, - StartToFireTimeoutSeconds: common.Int64Ptr(timer), - }, - }}, nil - } - - resp, err := s.engine.GetWorkflowExecutionHistory(createContext(), &types.GetWorkflowExecutionHistoryRequest{ - Domain: s.domainName, - Execution: workflowExecution, - MaximumPageSize: 200, - }) - s.Nil(err) - for _, event := range resp.History.Events { - switch event.GetEventType() { - case types.EventTypeWorkflowExecutionSignaled: - signalDelivered = true - case types.EventTypeTimerCanceled: - timerCancelled = true - } - } - - if !signalDelivered { - s.Fail("should receive a signal") - } - - if !timerCancelled { - return nil, []*types.Decision{{ - DecisionType: types.DecisionTypeCancelTimer.Ptr(), - CancelTimerDecisionAttributes: &types.CancelTimerDecisionAttributes{ - TimerID: TimerID, - }, - }}, nil - } - - workflowComplete = true - return nil, []*types.Decision{{ - DecisionType: types.DecisionTypeCompleteWorkflowExecution.Ptr(), - CompleteWorkflowExecutionDecisionAttributes: &types.CompleteWorkflowExecutionDecisionAttributes{ - Result: []byte("Done."), - }, - }}, nil - } - - poller := &TaskPoller{ - Engine: s.engine, - Domain: s.domainName, - TaskList: taskList, - Identity: identity, - DecisionHandler: dtHandler, - ActivityHandler: nil, - Logger: s.Logger, - T: s.T(), - } - - // schedule the timer - _, err := poller.PollAndProcessDecisionTask(false, false) - s.Logger.Info("PollAndProcessDecisionTask: completed") - s.Nil(err) - - s.Nil(s.sendSignal(s.domainName, workflowExecution, "random signal name", []byte("random signal payload"), identity)) - - // receive the signal & cancel the timer - _, err = poller.PollAndProcessDecisionTask(false, false) - s.Logger.Info("PollAndProcessDecisionTask: completed") - s.Nil(err) - - s.Nil(s.sendSignal(s.domainName, workflowExecution, "random signal name", []byte("random signal payload"), identity)) - // complete the workflow - _, err = poller.PollAndProcessDecisionTask(false, false) - s.Logger.Info("PollAndProcessDecisionTask: completed") - s.Nil(err) - - s.True(workflowComplete) - - resp, err := s.engine.GetWorkflowExecutionHistory(createContext(), &types.GetWorkflowExecutionHistoryRequest{ - Domain: s.domainName, - Execution: workflowExecution, - MaximumPageSize: 200, - }) - s.Nil(err) - for _, event := range resp.History.Events { - switch event.GetEventType() { - case types.EventTypeWorkflowExecutionSignaled: - signalDelivered = true - case types.EventTypeTimerCanceled: - timerCancelled = true - case types.EventTypeTimerFired: - s.Fail("timer got fired") - } - } -} - -func (s *IntegrationSuite) TestCancelTimer_CancelFiredAndBuffered() { - id := "integration-cancel-timer-fired-and-buffered-test" - wt := "integration-cancel-timer-fired-and-buffered-test-type" - tl := "integration-cancel-timer-fired-and-buffered-test-tasklist" - identity := "worker1" - - workflowType := &types.WorkflowType{} - workflowType.Name = wt - - taskList := &types.TaskList{} - taskList.Name = tl - - request := &types.StartWorkflowExecutionRequest{ - RequestID: uuid.New(), - Domain: s.domainName, - WorkflowID: id, - WorkflowType: workflowType, - TaskList: taskList, - Input: nil, - ExecutionStartToCloseTimeoutSeconds: common.Int32Ptr(100), - TaskStartToCloseTimeoutSeconds: common.Int32Ptr(1000), - Identity: identity, - } - - creatResp, err0 := s.engine.StartWorkflowExecution(createContext(), request) - s.Nil(err0) - workflowExecution := &types.WorkflowExecution{ - WorkflowID: id, - RunID: creatResp.GetRunID(), - } - - TimerID := 1 - timerScheduled := false - signalDelivered := false - timerCancelled := false - workflowComplete := false - timer := int64(4) - dtHandler := func(execution *types.WorkflowExecution, wt *types.WorkflowType, - previousStartedEventID, startedEventID int64, history *types.History) ([]byte, []*types.Decision, error) { - - if !timerScheduled { - timerScheduled = true - return nil, []*types.Decision{{ - DecisionType: types.DecisionTypeStartTimer.Ptr(), - StartTimerDecisionAttributes: &types.StartTimerDecisionAttributes{ - TimerID: fmt.Sprintf("%v", TimerID), - StartToFireTimeoutSeconds: common.Int64Ptr(timer), - }, - }}, nil - } - - resp, err := s.engine.GetWorkflowExecutionHistory(createContext(), &types.GetWorkflowExecutionHistoryRequest{ - Domain: s.domainName, - Execution: workflowExecution, - MaximumPageSize: 200, - }) - s.Nil(err) - for _, event := range resp.History.Events { - switch event.GetEventType() { - case types.EventTypeWorkflowExecutionSignaled: - signalDelivered = true - case types.EventTypeTimerCanceled: - timerCancelled = true - } - } - - if !signalDelivered { - s.Fail("should receive a signal") - } - - if !timerCancelled { - time.Sleep(time.Duration(2*timer) * time.Second) - return nil, []*types.Decision{{ - DecisionType: types.DecisionTypeCancelTimer.Ptr(), - CancelTimerDecisionAttributes: &types.CancelTimerDecisionAttributes{ - TimerID: fmt.Sprintf("%v", TimerID), - }, - }}, nil - } - - workflowComplete = true - return nil, []*types.Decision{{ - DecisionType: types.DecisionTypeCompleteWorkflowExecution.Ptr(), - CompleteWorkflowExecutionDecisionAttributes: &types.CompleteWorkflowExecutionDecisionAttributes{ - Result: []byte("Done."), - }, - }}, nil - } - - poller := &TaskPoller{ - Engine: s.engine, - Domain: s.domainName, - TaskList: taskList, - Identity: identity, - DecisionHandler: dtHandler, - ActivityHandler: nil, - Logger: s.Logger, - T: s.T(), - } - - // schedule the timer - _, err := poller.PollAndProcessDecisionTask(false, false) - s.Logger.Info("PollAndProcessDecisionTask: completed") - s.Nil(err) - - s.Nil(s.sendSignal(s.domainName, workflowExecution, "random signal name", []byte("random signal payload"), identity)) - - // receive the signal & cancel the timer - _, err = poller.PollAndProcessDecisionTask(false, false) - s.Logger.Info("PollAndProcessDecisionTask: completed") - s.Nil(err) - - s.Nil(s.sendSignal(s.domainName, workflowExecution, "random signal name", []byte("random signal payload"), identity)) - // complete the workflow - _, err = poller.PollAndProcessDecisionTask(false, false) - s.Logger.Info("PollAndProcessDecisionTask: completed") - s.Nil(err) - - s.True(workflowComplete) - - resp, err := s.engine.GetWorkflowExecutionHistory(createContext(), &types.GetWorkflowExecutionHistoryRequest{ - Domain: s.domainName, - Execution: workflowExecution, - MaximumPageSize: 200, - }) - s.Nil(err) - for _, event := range resp.History.Events { - switch event.GetEventType() { - case types.EventTypeWorkflowExecutionSignaled: - signalDelivered = true - case types.EventTypeTimerCanceled: - timerCancelled = true - case types.EventTypeTimerFired: - s.Fail("timer got fired") - } - } -} - -// helper function for TestStartWithMemo and TestSignalWithStartWithMemo to reduce duplicate code -func (s *IntegrationSuite) startWithMemoHelper(startFn startFunc, id string, taskList *types.TaskList, memo *types.Memo) { - identity := "worker1" - - we, err0 := startFn() - s.Nil(err0) - - s.Logger.Info("StartWorkflowExecution: response", tag.WorkflowRunID(we.RunID)) - - dtHandler := func(execution *types.WorkflowExecution, wt *types.WorkflowType, - previousStartedEventID, startedEventID int64, history *types.History) ([]byte, []*types.Decision, error) { - return []byte(strconv.Itoa(1)), []*types.Decision{{ - DecisionType: types.DecisionTypeCompleteWorkflowExecution.Ptr(), - CompleteWorkflowExecutionDecisionAttributes: &types.CompleteWorkflowExecutionDecisionAttributes{ - Result: []byte("Done."), - }, - }}, nil - } - - poller := &TaskPoller{ - Engine: s.engine, - Domain: s.domainName, - TaskList: taskList, - Identity: identity, - DecisionHandler: dtHandler, - Logger: s.Logger, - T: s.T(), - } - - // verify open visibility - var openExecutionInfo *types.WorkflowExecutionInfo - for i := 0; i < 10; i++ { - resp, err1 := s.engine.ListOpenWorkflowExecutions(createContext(), &types.ListOpenWorkflowExecutionsRequest{ - Domain: s.domainName, - MaximumPageSize: 100, - StartTimeFilter: &types.StartTimeFilter{ - EarliestTime: common.Int64Ptr(0), - LatestTime: common.Int64Ptr(time.Now().UnixNano()), - }, - ExecutionFilter: &types.WorkflowExecutionFilter{ - WorkflowID: id, - }, - }) - s.Nil(err1) - if len(resp.Executions) == 1 { - openExecutionInfo = resp.Executions[0] - break - } - s.Logger.Info("Open WorkflowExecution is not yet visible") - time.Sleep(100 * time.Millisecond) - } - s.NotNil(openExecutionInfo) - s.Equal(memo, openExecutionInfo.Memo) - - // make progress of workflow - _, err := poller.PollAndProcessDecisionTask(false, false) - s.Logger.Info("PollAndProcessDecisionTask: %v", tag.Error(err)) - s.Nil(err) - - // verify history - execution := &types.WorkflowExecution{ - WorkflowID: id, - RunID: we.RunID, - } - historyResponse, historyErr := s.engine.GetWorkflowExecutionHistory(createContext(), &types.GetWorkflowExecutionHistoryRequest{ - Domain: s.domainName, - Execution: execution, - }) - s.Nil(historyErr) - history := historyResponse.History - firstEvent := history.Events[0] - s.Equal(types.EventTypeWorkflowExecutionStarted, firstEvent.GetEventType()) - startdEventAttributes := firstEvent.WorkflowExecutionStartedEventAttributes - s.Equal(memo, startdEventAttributes.Memo) - - // verify DescribeWorkflowExecution result - descRequest := &types.DescribeWorkflowExecutionRequest{ - Domain: s.domainName, - Execution: execution, - } - descResp, err := s.engine.DescribeWorkflowExecution(createContext(), descRequest) - s.Nil(err) - s.Equal(memo, descResp.WorkflowExecutionInfo.Memo) - - // verify closed visibility - var closedExecutionInfo *types.WorkflowExecutionInfo - for i := 0; i < 10; i++ { - resp, err1 := s.engine.ListClosedWorkflowExecutions(createContext(), &types.ListClosedWorkflowExecutionsRequest{ - Domain: s.domainName, - MaximumPageSize: 100, - StartTimeFilter: &types.StartTimeFilter{ - EarliestTime: common.Int64Ptr(0), - LatestTime: common.Int64Ptr(time.Now().UnixNano()), - }, - ExecutionFilter: &types.WorkflowExecutionFilter{ - WorkflowID: id, - }, - }) - s.Nil(err1) - if len(resp.Executions) == 1 { - closedExecutionInfo = resp.Executions[0] - break - } - s.Logger.Info("Closed WorkflowExecution is not yet visible") - time.Sleep(100 * time.Millisecond) - } - s.NotNil(closedExecutionInfo) - s.Equal(memo, closedExecutionInfo.Memo) -} - -func (s *IntegrationSuite) sendSignal(domainName string, execution *types.WorkflowExecution, signalName string, - input []byte, identity string) error { - return s.engine.SignalWorkflowExecution(createContext(), &types.SignalWorkflowExecutionRequest{ - Domain: domainName, - WorkflowExecution: execution, - SignalName: signalName, - Input: input, - Identity: identity, - }) -} diff --git a/host/ndc/nDCIntegrationTest.go b/host/ndc/nDCIntegrationTest.go new file mode 100644 index 00000000000..8b1d121d0e9 --- /dev/null +++ b/host/ndc/nDCIntegrationTest.go @@ -0,0 +1,1802 @@ +// Copyright (c) 2019 Uber Technologies, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package ndc + +import ( + "context" + "fmt" + "sync/atomic" + "testing" + "time" + + "go.uber.org/yarpc" + + "github.com/golang/mock/gomock" + "github.com/pborman/uuid" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + "go.uber.org/zap" + + adminClient "github.com/uber/cadence/client/admin" + "github.com/uber/cadence/common" + "github.com/uber/cadence/common/cache" + "github.com/uber/cadence/common/log" + "github.com/uber/cadence/common/log/loggerimpl" + "github.com/uber/cadence/common/log/tag" + "github.com/uber/cadence/common/persistence" + pt "github.com/uber/cadence/common/persistence/persistence-tests" + test "github.com/uber/cadence/common/testing" + "github.com/uber/cadence/common/types" + "github.com/uber/cadence/host" +) + +type ( + NDCIntegrationTestSuite struct { + // override suite.Suite.Assertions with require.Assertions; this means that s.NotNil(nil) will stop the test, + // not merely log an error + *require.Assertions + suite.Suite + active *host.TestCluster + generator test.Generator + serializer persistence.PayloadSerializer + logger log.Logger + + domainName string + domainID string + version int64 + versionIncrement int64 + mockAdminClient map[string]adminClient.Client + standByReplicationTasksChan chan *types.ReplicationTask + standByTaskID int64 + + clusterConfigs []*host.TestClusterConfig + defaultTestCluster pt.PersistenceTestCluster + visibilityTestCluster pt.PersistenceTestCluster + } + + NDCIntegrationTestSuiteParams struct { + ClusterConfigs []*host.TestClusterConfig + DefaultTestCluster pt.PersistenceTestCluster + VisibilityTestCluster pt.PersistenceTestCluster + } +) + +var ( + clusterName = []string{"active", "standby", "other"} + clusterReplicationConfig = []*types.ClusterReplicationConfiguration{ + {ClusterName: clusterName[0]}, + {ClusterName: clusterName[1]}, + {ClusterName: clusterName[2]}, + } +) + +func NewNDCIntegrationTestSuite(params NDCIntegrationTestSuiteParams) *NDCIntegrationTestSuite { + return &NDCIntegrationTestSuite{ + clusterConfigs: params.ClusterConfigs, + defaultTestCluster: params.DefaultTestCluster, + visibilityTestCluster: params.VisibilityTestCluster, + } +} + +func (s *NDCIntegrationTestSuite) SetupSuite() { + zapLogger, err := zap.NewDevelopment() + // cannot use s.Nil since it is not initialized + s.Require().NoError(err) + s.serializer = persistence.NewPayloadSerializer() + s.logger = loggerimpl.NewLogger(zapLogger) + + s.standByReplicationTasksChan = make(chan *types.ReplicationTask, 100) + + s.standByTaskID = 0 + s.mockAdminClient = make(map[string]adminClient.Client) + controller := gomock.NewController(s.T()) + mockStandbyClient := adminClient.NewMockClient(controller) + mockStandbyClient.EXPECT().GetReplicationMessages(gomock.Any(), gomock.Any()).DoAndReturn(s.GetReplicationMessagesMock).AnyTimes() + mockOtherClient := adminClient.NewMockClient(controller) + mockOtherClient.EXPECT().GetReplicationMessages(gomock.Any(), gomock.Any()).Return( + &types.GetReplicationMessagesResponse{ + MessagesByShard: make(map[int32]*types.ReplicationMessages), + }, nil).AnyTimes() + s.mockAdminClient["standby"] = mockStandbyClient + s.mockAdminClient["other"] = mockOtherClient + s.clusterConfigs[0].MockAdminClient = s.mockAdminClient + + clusterMetadata := host.NewClusterMetadata(s.clusterConfigs[0], s.logger.WithTags(tag.ClusterName(clusterName[0]))) + params := pt.TestBaseParams{ + DefaultTestCluster: s.defaultTestCluster, + VisibilityTestCluster: s.visibilityTestCluster, + ClusterMetadata: clusterMetadata, + } + cluster, err := host.NewCluster(s.clusterConfigs[0], s.logger.WithTags(tag.ClusterName(clusterName[0])), params) + s.Require().NoError(err) + s.active = cluster + + s.registerDomain() + + s.version = s.clusterConfigs[1].ClusterMetadata.ClusterInformation[s.clusterConfigs[1].ClusterMetadata.CurrentClusterName].InitialFailoverVersion + s.versionIncrement = s.clusterConfigs[0].ClusterMetadata.FailoverVersionIncrement + s.generator = test.InitializeHistoryEventGenerator(s.domainName, s.version) +} + +func (s *NDCIntegrationTestSuite) GetReplicationMessagesMock( + ctx context.Context, + request *types.GetReplicationMessagesRequest, + opts ...yarpc.CallOption, +) (*types.GetReplicationMessagesResponse, error) { + select { + case task := <-s.standByReplicationTasksChan: + taskID := atomic.AddInt64(&s.standByTaskID, 1) + task.SourceTaskID = taskID + tasks := []*types.ReplicationTask{task} + for len(s.standByReplicationTasksChan) > 0 { + task = <-s.standByReplicationTasksChan + taskID := atomic.AddInt64(&s.standByTaskID, 1) + task.SourceTaskID = taskID + tasks = append(tasks, task) + } + + replicationMessage := &types.ReplicationMessages{ + ReplicationTasks: tasks, + LastRetrievedMessageID: tasks[len(tasks)-1].SourceTaskID, + HasMore: true, + } + + return &types.GetReplicationMessagesResponse{ + MessagesByShard: map[int32]*types.ReplicationMessages{0: replicationMessage}, + }, nil + default: + return &types.GetReplicationMessagesResponse{ + MessagesByShard: make(map[int32]*types.ReplicationMessages), + }, nil + } +} + +func (s *NDCIntegrationTestSuite) SetupTest() { + // Have to define our overridden assertions in the test setup. If we did it earlier, s.T() will return nil + s.Assertions = require.New(s.T()) + s.generator = test.InitializeHistoryEventGenerator(s.domainName, s.version) +} + +func (s *NDCIntegrationTestSuite) TearDownSuite() { + if s.generator != nil { + s.generator.Reset() + } + s.active.TearDownCluster() +} + +func (s *NDCIntegrationTestSuite) TestSingleBranch() { + + s.setupRemoteFrontendClients() + workflowID := "ndc-single-branch-test" + uuid.New() + + workflowType := "event-generator-workflow-type" + tasklist := "event-generator-taskList" + + // active has initial version 0 + historyClient := s.active.GetHistoryClient() + + versions := []int64{101, 1, 201, 301, 401, 601, 501, 801, 1001, 901, 701, 1101} + for _, version := range versions { + runID := uuid.New() + historyBatch := []*types.History{} + s.generator = test.InitializeHistoryEventGenerator(s.domainName, version) + + for s.generator.HasNextVertex() { + events := s.generator.GetNextVertices() + historyEvents := &types.History{} + for _, event := range events { + historyEvents.Events = append(historyEvents.Events, event.GetData().(*types.HistoryEvent)) + } + historyBatch = append(historyBatch, historyEvents) + } + + versionHistory := s.eventBatchesToVersionHistory(nil, historyBatch) + s.applyEvents( + workflowID, + runID, + workflowType, + tasklist, + versionHistory, + historyBatch, + historyClient, + ) + + err := s.verifyEventHistory(workflowID, runID, historyBatch) + s.Require().NoError(err) + } +} + +func (s *NDCIntegrationTestSuite) verifyEventHistory( + workflowID string, + runID string, + historyBatch []*types.History, +) error { + // get replicated history events from passive side + passiveClient := s.active.GetFrontendClient() + replicatedHistory, err := passiveClient.GetWorkflowExecutionHistory( + s.createContext(), + &types.GetWorkflowExecutionHistoryRequest{ + Domain: s.domainName, + Execution: &types.WorkflowExecution{ + WorkflowID: workflowID, + RunID: runID, + }, + MaximumPageSize: 1000, + NextPageToken: nil, + WaitForNewEvent: false, + HistoryEventFilterType: types.HistoryEventFilterTypeAllEvent.Ptr(), + }, + ) + + if err != nil { + return fmt.Errorf("failed to get history event from passive side: %v", err) + } + + // compare origin events with replicated events + batchIndex := 0 + batch := historyBatch[batchIndex].Events + eventIndex := 0 + for _, event := range replicatedHistory.GetHistory().GetEvents() { + if eventIndex >= len(batch) { + batchIndex++ + batch = historyBatch[batchIndex].Events + eventIndex = 0 + } + originEvent := batch[eventIndex] + eventIndex++ + if originEvent.GetEventType() != event.GetEventType() { + return fmt.Errorf("the replicated event (%v) and the origin event (%v) are not the same", + originEvent.GetEventType().String(), event.GetEventType().String()) + } + } + + return nil +} + +func (s *NDCIntegrationTestSuite) TestMultipleBranches() { + + s.setupRemoteFrontendClients() + workflowID := "ndc-multiple-branches-test" + uuid.New() + + workflowType := "event-generator-workflow-type" + tasklist := "event-generator-taskList" + + // active has initial version 0 + historyClient := s.active.GetHistoryClient() + + versions := []int64{101, 1, 201} + for _, version := range versions { + runID := uuid.New() + + baseBranch := []*types.History{} + baseGenerator := test.InitializeHistoryEventGenerator(s.domainName, version) + baseGenerator.SetVersion(version) + + for i := 0; i < 10 && baseGenerator.HasNextVertex(); i++ { + events := baseGenerator.GetNextVertices() + historyEvents := &types.History{} + for _, event := range events { + historyEvents.Events = append(historyEvents.Events, event.GetData().(*types.HistoryEvent)) + } + baseBranch = append(baseBranch, historyEvents) + } + baseVersionHistory := s.eventBatchesToVersionHistory(nil, baseBranch) + + branch1 := []*types.History{} + branchVersionHistory1 := baseVersionHistory.Duplicate() + branchGenerator1 := baseGenerator.DeepCopy() + for i := 0; i < 10 && branchGenerator1.HasNextVertex(); i++ { + events := branchGenerator1.GetNextVertices() + historyEvents := &types.History{} + for _, event := range events { + historyEvents.Events = append(historyEvents.Events, event.GetData().(*types.HistoryEvent)) + } + branch1 = append(branch1, historyEvents) + } + branchVersionHistory1 = s.eventBatchesToVersionHistory(branchVersionHistory1, branch1) + + branch2 := []*types.History{} + branchVersionHistory2 := baseVersionHistory.Duplicate() + branchGenerator2 := baseGenerator.DeepCopy() + branchGenerator2.SetVersion(branchGenerator2.GetVersion() + 1) + for i := 0; i < 10 && branchGenerator2.HasNextVertex(); i++ { + events := branchGenerator2.GetNextVertices() + historyEvents := &types.History{} + for _, event := range events { + historyEvents.Events = append(historyEvents.Events, event.GetData().(*types.HistoryEvent)) + } + branch2 = append(branch2, historyEvents) + } + branchVersionHistory2 = s.eventBatchesToVersionHistory(branchVersionHistory2, branch2) + + s.applyEvents( + workflowID, + runID, + workflowType, + tasklist, + baseVersionHistory, + baseBranch, + historyClient, + ) + s.applyEvents( + workflowID, + runID, + workflowType, + tasklist, + branchVersionHistory1, + branch1, + historyClient, + ) + s.applyEvents( + workflowID, + runID, + workflowType, + tasklist, + branchVersionHistory2, + branch2, + historyClient, + ) + } +} + +func (s *NDCIntegrationTestSuite) TestHandcraftedMultipleBranches() { + + s.setupRemoteFrontendClients() + workflowID := "ndc-handcrafted-multiple-branches-test" + uuid.New() + runID := uuid.New() + + workflowType := "event-generator-workflow-type" + tasklist := "event-generator-taskList" + identity := "worker-identity" + + // active has initial version 0 + historyClient := s.active.GetHistoryClient() + + eventsBatch1 := []*types.History{ + {Events: []*types.HistoryEvent{ + { + EventID: 1, + Version: 21, + EventType: types.EventTypeWorkflowExecutionStarted.Ptr(), + WorkflowExecutionStartedEventAttributes: &types.WorkflowExecutionStartedEventAttributes{ + WorkflowType: &types.WorkflowType{Name: workflowType}, + TaskList: &types.TaskList{Name: tasklist}, + Input: nil, + ExecutionStartToCloseTimeoutSeconds: common.Int32Ptr(1000), + TaskStartToCloseTimeoutSeconds: common.Int32Ptr(1000), + FirstDecisionTaskBackoffSeconds: common.Int32Ptr(100), + }, + }, + { + EventID: 2, + Version: 21, + EventType: types.EventTypeDecisionTaskScheduled.Ptr(), + DecisionTaskScheduledEventAttributes: &types.DecisionTaskScheduledEventAttributes{ + TaskList: &types.TaskList{Name: tasklist}, + StartToCloseTimeoutSeconds: common.Int32Ptr(1000), + Attempt: 0, + }, + }, + }}, + {Events: []*types.HistoryEvent{ + { + EventID: 3, + Version: 21, + EventType: types.EventTypeDecisionTaskStarted.Ptr(), + DecisionTaskStartedEventAttributes: &types.DecisionTaskStartedEventAttributes{ + ScheduledEventID: 2, + Identity: identity, + RequestID: uuid.New(), + }, + }, + }}, + {Events: []*types.HistoryEvent{ + { + EventID: 4, + Version: 21, + EventType: types.EventTypeDecisionTaskCompleted.Ptr(), + DecisionTaskCompletedEventAttributes: &types.DecisionTaskCompletedEventAttributes{ + ScheduledEventID: 2, + StartedEventID: 3, + Identity: identity, + }, + }, + { + EventID: 5, + Version: 21, + EventType: types.EventTypeMarkerRecorded.Ptr(), + MarkerRecordedEventAttributes: &types.MarkerRecordedEventAttributes{ + MarkerName: "some marker name", + Details: []byte("some marker details"), + DecisionTaskCompletedEventID: 4, + }, + }, + { + EventID: 6, + Version: 21, + EventType: types.EventTypeActivityTaskScheduled.Ptr(), + ActivityTaskScheduledEventAttributes: &types.ActivityTaskScheduledEventAttributes{ + DecisionTaskCompletedEventID: 4, + ActivityID: "0", + ActivityType: &types.ActivityType{Name: "activity-type"}, + TaskList: &types.TaskList{Name: tasklist}, + Input: nil, + ScheduleToCloseTimeoutSeconds: common.Int32Ptr(20), + ScheduleToStartTimeoutSeconds: common.Int32Ptr(20), + StartToCloseTimeoutSeconds: common.Int32Ptr(20), + HeartbeatTimeoutSeconds: common.Int32Ptr(20), + }, + }, + }}, + {Events: []*types.HistoryEvent{ + { + EventID: 7, + Version: 21, + EventType: types.EventTypeActivityTaskStarted.Ptr(), + ActivityTaskStartedEventAttributes: &types.ActivityTaskStartedEventAttributes{ + ScheduledEventID: 6, + Identity: identity, + RequestID: uuid.New(), + Attempt: 0, + }, + }, + }}, + {Events: []*types.HistoryEvent{ + { + EventID: 8, + Version: 21, + EventType: types.EventTypeWorkflowExecutionSignaled.Ptr(), + WorkflowExecutionSignaledEventAttributes: &types.WorkflowExecutionSignaledEventAttributes{ + SignalName: "some signal name 1", + Input: []byte("some signal details 1"), + Identity: identity, + }, + }, + { + EventID: 9, + Version: 21, + EventType: types.EventTypeDecisionTaskScheduled.Ptr(), + DecisionTaskScheduledEventAttributes: &types.DecisionTaskScheduledEventAttributes{ + TaskList: &types.TaskList{Name: tasklist}, + StartToCloseTimeoutSeconds: common.Int32Ptr(1000), + Attempt: 0, + }, + }, + }}, + {Events: []*types.HistoryEvent{ + { + EventID: 10, + Version: 21, + EventType: types.EventTypeDecisionTaskStarted.Ptr(), + DecisionTaskStartedEventAttributes: &types.DecisionTaskStartedEventAttributes{ + ScheduledEventID: 9, + Identity: identity, + RequestID: uuid.New(), + }, + }, + }}, + {Events: []*types.HistoryEvent{ + { + EventID: 11, + Version: 21, + EventType: types.EventTypeDecisionTaskCompleted.Ptr(), + DecisionTaskCompletedEventAttributes: &types.DecisionTaskCompletedEventAttributes{ + ScheduledEventID: 9, + StartedEventID: 10, + Identity: identity, + }, + }, + { + EventID: 12, + Version: 21, + EventType: types.EventTypeWorkflowExecutionSignaled.Ptr(), + WorkflowExecutionSignaledEventAttributes: &types.WorkflowExecutionSignaledEventAttributes{ + SignalName: "some signal name 2", + Input: []byte("some signal details 2"), + Identity: identity, + }, + }, + { + EventID: 13, + Version: 21, + EventType: types.EventTypeDecisionTaskScheduled.Ptr(), + DecisionTaskScheduledEventAttributes: &types.DecisionTaskScheduledEventAttributes{ + TaskList: &types.TaskList{Name: tasklist}, + StartToCloseTimeoutSeconds: common.Int32Ptr(1000), + Attempt: 0, + }, + }, + { + EventID: 14, + Version: 21, + EventType: types.EventTypeDecisionTaskStarted.Ptr(), + DecisionTaskStartedEventAttributes: &types.DecisionTaskStartedEventAttributes{ + ScheduledEventID: 13, + Identity: identity, + RequestID: uuid.New(), + }, + }, + }}, + } + + eventsBatch2 := []*types.History{ + {Events: []*types.HistoryEvent{ + { + EventID: 15, + Version: 31, + EventType: types.EventTypeWorkflowExecutionTimedOut.Ptr(), + WorkflowExecutionTimedOutEventAttributes: &types.WorkflowExecutionTimedOutEventAttributes{ + TimeoutType: types.TimeoutTypeStartToClose.Ptr(), + }, + }, + }}, + } + + eventsBatch3 := []*types.History{ + {Events: []*types.HistoryEvent{ + { + EventID: 15, + Version: 30, + EventType: types.EventTypeDecisionTaskTimedOut.Ptr(), + DecisionTaskTimedOutEventAttributes: &types.DecisionTaskTimedOutEventAttributes{ + ScheduledEventID: 13, + StartedEventID: 14, + TimeoutType: types.TimeoutTypeStartToClose.Ptr(), + }, + }, + { + EventID: 16, + Version: 30, + EventType: types.EventTypeActivityTaskTimedOut.Ptr(), + ActivityTaskTimedOutEventAttributes: &types.ActivityTaskTimedOutEventAttributes{ + ScheduledEventID: 6, + StartedEventID: 7, + TimeoutType: types.TimeoutTypeStartToClose.Ptr(), + }, + }, + { + EventID: 17, + Version: 30, + EventType: types.EventTypeDecisionTaskScheduled.Ptr(), + DecisionTaskScheduledEventAttributes: &types.DecisionTaskScheduledEventAttributes{ + TaskList: &types.TaskList{Name: tasklist}, + StartToCloseTimeoutSeconds: common.Int32Ptr(1000), + Attempt: 0, + }, + }, + }}, + {Events: []*types.HistoryEvent{ + { + EventID: 18, + Version: 30, + EventType: types.EventTypeDecisionTaskStarted.Ptr(), + DecisionTaskStartedEventAttributes: &types.DecisionTaskStartedEventAttributes{ + ScheduledEventID: 17, + Identity: identity, + RequestID: uuid.New(), + }, + }, + }}, + {Events: []*types.HistoryEvent{ + { + EventID: 19, + Version: 30, + EventType: types.EventTypeDecisionTaskCompleted.Ptr(), + DecisionTaskCompletedEventAttributes: &types.DecisionTaskCompletedEventAttributes{ + ScheduledEventID: 8, + StartedEventID: 9, + Identity: identity, + }, + }, + { + EventID: 20, + Version: 30, + EventType: types.EventTypeWorkflowExecutionFailed.Ptr(), + WorkflowExecutionFailedEventAttributes: &types.WorkflowExecutionFailedEventAttributes{ + DecisionTaskCompletedEventID: 19, + Reason: common.StringPtr("some random reason"), + Details: nil, + }, + }, + }}, + } + + versionHistory1 := s.eventBatchesToVersionHistory(nil, eventsBatch1) + + versionHistory2, err := versionHistory1.DuplicateUntilLCAItem( + persistence.NewVersionHistoryItem(14, 21), + ) + s.NoError(err) + versionHistory2 = s.eventBatchesToVersionHistory(versionHistory2, eventsBatch2) + + versionHistory3, err := versionHistory1.DuplicateUntilLCAItem( + persistence.NewVersionHistoryItem(14, 21), + ) + s.NoError(err) + versionHistory3 = s.eventBatchesToVersionHistory(versionHistory3, eventsBatch3) + + s.applyEvents( + workflowID, + runID, + workflowType, + tasklist, + versionHistory1, + eventsBatch1, + historyClient, + ) + s.applyEvents( + workflowID, + runID, + workflowType, + tasklist, + versionHistory3, + eventsBatch3, + historyClient, + ) + s.applyEvents( + workflowID, + runID, + workflowType, + tasklist, + versionHistory2, + eventsBatch2, + historyClient, + ) +} + +func (s *NDCIntegrationTestSuite) TestHandcraftedMultipleBranchesWithZombieContinueAsNew() { + + s.setupRemoteFrontendClients() + workflowID := "ndc-handcrafted-multiple-branches-with-continue-as-new-test" + uuid.New() + runID := uuid.New() + + workflowType := "event-generator-workflow-type" + tasklist := "event-generator-taskList" + identity := "worker-identity" + + // active has initial version 0 + historyClient := s.active.GetHistoryClient() + + eventsBatch1 := []*types.History{ + {Events: []*types.HistoryEvent{ + { + EventID: 1, + Version: 21, + EventType: types.EventTypeWorkflowExecutionStarted.Ptr(), + WorkflowExecutionStartedEventAttributes: &types.WorkflowExecutionStartedEventAttributes{ + WorkflowType: &types.WorkflowType{Name: workflowType}, + TaskList: &types.TaskList{Name: tasklist}, + Input: nil, + ExecutionStartToCloseTimeoutSeconds: common.Int32Ptr(1000), + TaskStartToCloseTimeoutSeconds: common.Int32Ptr(1000), + FirstDecisionTaskBackoffSeconds: common.Int32Ptr(100), + }, + }, + { + EventID: 2, + Version: 21, + EventType: types.EventTypeDecisionTaskScheduled.Ptr(), + DecisionTaskScheduledEventAttributes: &types.DecisionTaskScheduledEventAttributes{ + TaskList: &types.TaskList{Name: tasklist}, + StartToCloseTimeoutSeconds: common.Int32Ptr(1000), + Attempt: 0, + }, + }, + }}, + {Events: []*types.HistoryEvent{ + { + EventID: 3, + Version: 21, + EventType: types.EventTypeDecisionTaskStarted.Ptr(), + DecisionTaskStartedEventAttributes: &types.DecisionTaskStartedEventAttributes{ + ScheduledEventID: 2, + Identity: identity, + RequestID: uuid.New(), + }, + }, + }}, + {Events: []*types.HistoryEvent{ + { + EventID: 4, + Version: 21, + EventType: types.EventTypeDecisionTaskCompleted.Ptr(), + DecisionTaskCompletedEventAttributes: &types.DecisionTaskCompletedEventAttributes{ + ScheduledEventID: 2, + StartedEventID: 3, + Identity: identity, + }, + }, + { + EventID: 5, + Version: 21, + EventType: types.EventTypeMarkerRecorded.Ptr(), + MarkerRecordedEventAttributes: &types.MarkerRecordedEventAttributes{ + MarkerName: "some marker name", + Details: []byte("some marker details"), + DecisionTaskCompletedEventID: 4, + }, + }, + { + EventID: 6, + Version: 21, + EventType: types.EventTypeActivityTaskScheduled.Ptr(), + ActivityTaskScheduledEventAttributes: &types.ActivityTaskScheduledEventAttributes{ + DecisionTaskCompletedEventID: 4, + ActivityID: "0", + ActivityType: &types.ActivityType{Name: "activity-type"}, + TaskList: &types.TaskList{Name: tasklist}, + Input: nil, + ScheduleToCloseTimeoutSeconds: common.Int32Ptr(20), + ScheduleToStartTimeoutSeconds: common.Int32Ptr(20), + StartToCloseTimeoutSeconds: common.Int32Ptr(20), + HeartbeatTimeoutSeconds: common.Int32Ptr(20), + }, + }, + }}, + {Events: []*types.HistoryEvent{ + { + EventID: 7, + Version: 21, + EventType: types.EventTypeActivityTaskStarted.Ptr(), + ActivityTaskStartedEventAttributes: &types.ActivityTaskStartedEventAttributes{ + ScheduledEventID: 6, + Identity: identity, + RequestID: uuid.New(), + Attempt: 0, + }, + }, + }}, + {Events: []*types.HistoryEvent{ + { + EventID: 8, + Version: 21, + EventType: types.EventTypeWorkflowExecutionSignaled.Ptr(), + WorkflowExecutionSignaledEventAttributes: &types.WorkflowExecutionSignaledEventAttributes{ + SignalName: "some signal name 1", + Input: []byte("some signal details 1"), + Identity: identity, + }, + }, + { + EventID: 9, + Version: 21, + EventType: types.EventTypeDecisionTaskScheduled.Ptr(), + DecisionTaskScheduledEventAttributes: &types.DecisionTaskScheduledEventAttributes{ + TaskList: &types.TaskList{Name: tasklist}, + StartToCloseTimeoutSeconds: common.Int32Ptr(1000), + Attempt: 0, + }, + }, + }}, + {Events: []*types.HistoryEvent{ + { + EventID: 10, + Version: 21, + EventType: types.EventTypeDecisionTaskStarted.Ptr(), + DecisionTaskStartedEventAttributes: &types.DecisionTaskStartedEventAttributes{ + ScheduledEventID: 9, + Identity: identity, + RequestID: uuid.New(), + }, + }, + }}, + {Events: []*types.HistoryEvent{ + { + EventID: 11, + Version: 21, + EventType: types.EventTypeDecisionTaskCompleted.Ptr(), + DecisionTaskCompletedEventAttributes: &types.DecisionTaskCompletedEventAttributes{ + ScheduledEventID: 9, + StartedEventID: 10, + Identity: identity, + }, + }, + { + EventID: 12, + Version: 21, + EventType: types.EventTypeWorkflowExecutionSignaled.Ptr(), + WorkflowExecutionSignaledEventAttributes: &types.WorkflowExecutionSignaledEventAttributes{ + SignalName: "some signal name 2", + Input: []byte("some signal details 2"), + Identity: identity, + }, + }, + { + EventID: 13, + Version: 21, + EventType: types.EventTypeDecisionTaskScheduled.Ptr(), + DecisionTaskScheduledEventAttributes: &types.DecisionTaskScheduledEventAttributes{ + TaskList: &types.TaskList{Name: tasklist}, + StartToCloseTimeoutSeconds: common.Int32Ptr(1000), + Attempt: 0, + }, + }, + { + EventID: 14, + Version: 21, + EventType: types.EventTypeDecisionTaskStarted.Ptr(), + DecisionTaskStartedEventAttributes: &types.DecisionTaskStartedEventAttributes{ + ScheduledEventID: 13, + Identity: identity, + RequestID: uuid.New(), + }, + }, + }}, + } + + eventsBatch2 := []*types.History{ + {Events: []*types.HistoryEvent{ + { + EventID: 15, + Version: 32, + EventType: types.EventTypeDecisionTaskCompleted.Ptr(), + DecisionTaskCompletedEventAttributes: &types.DecisionTaskCompletedEventAttributes{ + ScheduledEventID: 8, + StartedEventID: 9, + Identity: identity, + }, + }, + }}, + // need to keep the workflow open for testing + } + + eventsBatch3 := []*types.History{ + {Events: []*types.HistoryEvent{ + { + EventID: 15, + Version: 21, + EventType: types.EventTypeDecisionTaskCompleted.Ptr(), + DecisionTaskCompletedEventAttributes: &types.DecisionTaskCompletedEventAttributes{ + ScheduledEventID: 8, + StartedEventID: 9, + Identity: identity, + }, + }, + { + EventID: 16, + Version: 21, + EventType: types.EventTypeWorkflowExecutionContinuedAsNew.Ptr(), + WorkflowExecutionContinuedAsNewEventAttributes: &types.WorkflowExecutionContinuedAsNewEventAttributes{ + NewExecutionRunID: uuid.New(), + WorkflowType: &types.WorkflowType{Name: workflowType}, + TaskList: &types.TaskList{Name: tasklist}, + Input: nil, + ExecutionStartToCloseTimeoutSeconds: common.Int32Ptr(1000), + TaskStartToCloseTimeoutSeconds: common.Int32Ptr(1000), + DecisionTaskCompletedEventID: 19, + Initiator: types.ContinueAsNewInitiatorDecider.Ptr(), + }, + }, + }}, + } + + versionHistory1 := s.eventBatchesToVersionHistory(nil, eventsBatch1) + + versionHistory2, err := versionHistory1.DuplicateUntilLCAItem( + persistence.NewVersionHistoryItem(14, 21), + ) + s.NoError(err) + versionHistory2 = s.eventBatchesToVersionHistory(versionHistory2, eventsBatch2) + + versionHistory3, err := versionHistory1.DuplicateUntilLCAItem( + persistence.NewVersionHistoryItem(14, 21), + ) + s.NoError(err) + versionHistory3 = s.eventBatchesToVersionHistory(versionHistory3, eventsBatch3) + + s.applyEvents( + workflowID, + runID, + workflowType, + tasklist, + versionHistory1, + eventsBatch1, + historyClient, + ) + s.applyEvents( + workflowID, + runID, + workflowType, + tasklist, + versionHistory2, + eventsBatch2, + historyClient, + ) + s.applyEvents( + workflowID, + runID, + workflowType, + tasklist, + versionHistory3, + eventsBatch3, + historyClient, + ) +} + +func (s *NDCIntegrationTestSuite) TestEventsReapply_ZombieWorkflow() { + + workflowID := "ndc-single-branch-test" + uuid.New() + + workflowType := "event-generator-workflow-type" + tasklist := "event-generator-taskList" + + // active has initial version 0 + historyClient := s.active.GetHistoryClient() + + version := int64(101) + runID := uuid.New() + historyBatch := []*types.History{} + s.generator = test.InitializeHistoryEventGenerator(s.domainName, version) + + for s.generator.HasNextVertex() { + events := s.generator.GetNextVertices() + historyEvents := &types.History{} + for _, event := range events { + historyEvents.Events = append(historyEvents.Events, event.GetData().(*types.HistoryEvent)) + } + historyBatch = append(historyBatch, historyEvents) + } + + versionHistory := s.eventBatchesToVersionHistory(nil, historyBatch) + s.applyEvents( + workflowID, + runID, + workflowType, + tasklist, + versionHistory, + historyBatch, + historyClient, + ) + + version = int64(1) + runID = uuid.New() + historyBatch = []*types.History{} + s.generator = test.InitializeHistoryEventGenerator(s.domainName, version) + + // verify two batches of zombie workflow are call reapply API + s.mockAdminClient["standby"].(*adminClient.MockClient).EXPECT().ReapplyEvents(gomock.Any(), gomock.Any()).Return(nil).Times(2) + for i := 0; i < 2 && s.generator.HasNextVertex(); i++ { + events := s.generator.GetNextVertices() + historyEvents := &types.History{} + for _, event := range events { + historyEvents.Events = append(historyEvents.Events, event.GetData().(*types.HistoryEvent)) + } + historyBatch = append(historyBatch, historyEvents) + } + + versionHistory = s.eventBatchesToVersionHistory(nil, historyBatch) + s.applyEvents( + workflowID, + runID, + workflowType, + tasklist, + versionHistory, + historyBatch, + historyClient, + ) +} + +func (s *NDCIntegrationTestSuite) TestEventsReapply_UpdateNonCurrentBranch() { + + workflowID := "ndc-single-branch-test" + uuid.New() + runID := uuid.New() + workflowType := "event-generator-workflow-type" + tasklist := "event-generator-taskList" + version := int64(101) + isWorkflowFinished := false + + historyClient := s.active.GetHistoryClient() + + s.generator = test.InitializeHistoryEventGenerator(s.domainName, version) + baseBranch := []*types.History{} + var taskID int64 + for i := 0; i < 4 && s.generator.HasNextVertex(); i++ { + events := s.generator.GetNextVertices() + historyEvents := &types.History{} + for _, event := range events { + historyEvent := event.GetData().(*types.HistoryEvent) + taskID = historyEvent.GetTaskID() + historyEvents.Events = append(historyEvents.Events, historyEvent) + switch historyEvent.GetEventType() { + case types.EventTypeWorkflowExecutionCompleted, + types.EventTypeWorkflowExecutionFailed, + types.EventTypeWorkflowExecutionTimedOut, + types.EventTypeWorkflowExecutionTerminated, + types.EventTypeWorkflowExecutionContinuedAsNew, + types.EventTypeWorkflowExecutionCanceled: + isWorkflowFinished = true + } + } + baseBranch = append(baseBranch, historyEvents) + } + if isWorkflowFinished { + // cannot proceed since the test below requires workflow not finished + // this is ok since build kite will run this test several times + s.logger.Info("Encounter finish workflow history event during randomization test, skip") + return + } + + versionHistory := s.eventBatchesToVersionHistory(nil, baseBranch) + s.applyEvents( + workflowID, + runID, + workflowType, + tasklist, + versionHistory, + baseBranch, + historyClient, + ) + + newGenerator := s.generator.DeepCopy() + newBranch := []*types.History{} + newVersionHistory := versionHistory.Duplicate() + newGenerator.SetVersion(newGenerator.GetVersion() + 1) // simulate events from other cluster + for i := 0; i < 4 && newGenerator.HasNextVertex(); i++ { + events := newGenerator.GetNextVertices() + historyEvents := &types.History{} + for _, event := range events { + history := event.GetData().(*types.HistoryEvent) + taskID = history.GetTaskID() + historyEvents.Events = append(historyEvents.Events, history) + } + newBranch = append(newBranch, historyEvents) + } + newVersionHistory = s.eventBatchesToVersionHistory(newVersionHistory, newBranch) + s.applyEvents( + workflowID, + runID, + workflowType, + tasklist, + newVersionHistory, + newBranch, + historyClient, + ) + + s.mockAdminClient["standby"].(*adminClient.MockClient).EXPECT().ReapplyEvents(gomock.Any(), gomock.Any()).Return(nil).Times(1) + // Handcraft a stale signal event + baseBranchLastEventBatch := baseBranch[len(baseBranch)-1].GetEvents() + baseBranchLastEvent := baseBranchLastEventBatch[len(baseBranchLastEventBatch)-1] + staleBranch := []*types.History{ + { + Events: []*types.HistoryEvent{ + { + EventID: baseBranchLastEvent.GetEventID() + 1, + EventType: types.EventTypeWorkflowExecutionSignaled.Ptr(), + Timestamp: common.Int64Ptr(time.Now().UnixNano()), + Version: baseBranchLastEvent.GetVersion(), // dummy event from other cluster + TaskID: taskID, + WorkflowExecutionSignaledEventAttributes: &types.WorkflowExecutionSignaledEventAttributes{ + SignalName: "signal", + Input: []byte{}, + Identity: "ndc_integration_test", + }, + }, + }, + }, + } + staleVersionHistory := s.eventBatchesToVersionHistory(versionHistory.Duplicate(), staleBranch) + s.applyEvents( + workflowID, + runID, + workflowType, + tasklist, + staleVersionHistory, + staleBranch, + historyClient, + ) +} + +func (s *NDCIntegrationTestSuite) TestAdminGetWorkflowExecutionRawHistoryV2() { + + workflowID := "ndc-re-send-test" + uuid.New() + runID := uuid.New() + workflowType := "ndc-re-send-workflow-type" + tasklist := "event-generator-taskList" + identity := "ndc-re-send-test" + + historyClient := s.active.GetHistoryClient() + adminClient := s.active.GetAdminClient() + getHistory := func( + domain string, + workflowID string, + runID string, + startEventID *int64, + startEventVersion *int64, + endEventID *int64, + endEventVersion *int64, + pageSize int, + token []byte, + ) (*types.GetWorkflowExecutionRawHistoryV2Response, error) { + + execution := &types.WorkflowExecution{ + WorkflowID: workflowID, + RunID: runID, + } + return adminClient.GetWorkflowExecutionRawHistoryV2(s.createContext(), &types.GetWorkflowExecutionRawHistoryV2Request{ + Domain: domain, + Execution: execution, + StartEventID: startEventID, + StartEventVersion: startEventVersion, + EndEventID: endEventID, + EndEventVersion: endEventVersion, + MaximumPageSize: int32(pageSize), + NextPageToken: token, + }) + } + + eventsBatch1 := []*types.History{ + {Events: []*types.HistoryEvent{ + { + EventID: 1, + Version: 21, + EventType: types.EventTypeWorkflowExecutionStarted.Ptr(), + WorkflowExecutionStartedEventAttributes: &types.WorkflowExecutionStartedEventAttributes{ + WorkflowType: &types.WorkflowType{Name: workflowType}, + TaskList: &types.TaskList{Name: tasklist}, + Input: nil, + ExecutionStartToCloseTimeoutSeconds: common.Int32Ptr(1000), + TaskStartToCloseTimeoutSeconds: common.Int32Ptr(1000), + FirstDecisionTaskBackoffSeconds: common.Int32Ptr(100), + }, + }, + { + EventID: 2, + Version: 21, + EventType: types.EventTypeDecisionTaskScheduled.Ptr(), + DecisionTaskScheduledEventAttributes: &types.DecisionTaskScheduledEventAttributes{ + TaskList: &types.TaskList{Name: tasklist}, + StartToCloseTimeoutSeconds: common.Int32Ptr(1000), + Attempt: 0, + }, + }, + }}, + {Events: []*types.HistoryEvent{ + { + EventID: 3, + Version: 21, + EventType: types.EventTypeDecisionTaskStarted.Ptr(), + DecisionTaskStartedEventAttributes: &types.DecisionTaskStartedEventAttributes{ + ScheduledEventID: 2, + Identity: identity, + RequestID: uuid.New(), + }, + }, + }}, + {Events: []*types.HistoryEvent{ + { + EventID: 4, + Version: 21, + EventType: types.EventTypeDecisionTaskCompleted.Ptr(), + DecisionTaskCompletedEventAttributes: &types.DecisionTaskCompletedEventAttributes{ + ScheduledEventID: 2, + StartedEventID: 3, + Identity: identity, + }, + }, + { + EventID: 5, + Version: 21, + EventType: types.EventTypeMarkerRecorded.Ptr(), + MarkerRecordedEventAttributes: &types.MarkerRecordedEventAttributes{ + MarkerName: "some marker name", + Details: []byte("some marker details"), + DecisionTaskCompletedEventID: 4, + }, + }, + { + EventID: 6, + Version: 21, + EventType: types.EventTypeActivityTaskScheduled.Ptr(), + ActivityTaskScheduledEventAttributes: &types.ActivityTaskScheduledEventAttributes{ + DecisionTaskCompletedEventID: 4, + ActivityID: "0", + ActivityType: &types.ActivityType{Name: "activity-type"}, + TaskList: &types.TaskList{Name: tasklist}, + Input: nil, + ScheduleToCloseTimeoutSeconds: common.Int32Ptr(20), + ScheduleToStartTimeoutSeconds: common.Int32Ptr(20), + StartToCloseTimeoutSeconds: common.Int32Ptr(20), + HeartbeatTimeoutSeconds: common.Int32Ptr(20), + }, + }, + }}, + {Events: []*types.HistoryEvent{ + { + EventID: 7, + Version: 21, + EventType: types.EventTypeActivityTaskStarted.Ptr(), + ActivityTaskStartedEventAttributes: &types.ActivityTaskStartedEventAttributes{ + ScheduledEventID: 6, + Identity: identity, + RequestID: uuid.New(), + Attempt: 0, + }, + }, + }}, + {Events: []*types.HistoryEvent{ + { + EventID: 8, + Version: 21, + EventType: types.EventTypeWorkflowExecutionSignaled.Ptr(), + WorkflowExecutionSignaledEventAttributes: &types.WorkflowExecutionSignaledEventAttributes{ + SignalName: "some signal name 1", + Input: []byte("some signal details 1"), + Identity: identity, + }, + }, + { + EventID: 9, + Version: 21, + EventType: types.EventTypeDecisionTaskScheduled.Ptr(), + DecisionTaskScheduledEventAttributes: &types.DecisionTaskScheduledEventAttributes{ + TaskList: &types.TaskList{Name: tasklist}, + StartToCloseTimeoutSeconds: common.Int32Ptr(1000), + Attempt: 0, + }, + }, + }}, + {Events: []*types.HistoryEvent{ + { + EventID: 10, + Version: 21, + EventType: types.EventTypeDecisionTaskStarted.Ptr(), + DecisionTaskStartedEventAttributes: &types.DecisionTaskStartedEventAttributes{ + ScheduledEventID: 9, + Identity: identity, + RequestID: uuid.New(), + }, + }, + }}, + {Events: []*types.HistoryEvent{ + { + EventID: 11, + Version: 21, + EventType: types.EventTypeDecisionTaskCompleted.Ptr(), + DecisionTaskCompletedEventAttributes: &types.DecisionTaskCompletedEventAttributes{ + ScheduledEventID: 9, + StartedEventID: 10, + Identity: identity, + }, + }, + { + EventID: 12, + Version: 21, + EventType: types.EventTypeWorkflowExecutionSignaled.Ptr(), + WorkflowExecutionSignaledEventAttributes: &types.WorkflowExecutionSignaledEventAttributes{ + SignalName: "some signal name 2", + Input: []byte("some signal details 2"), + Identity: identity, + }, + }, + { + EventID: 13, + Version: 21, + EventType: types.EventTypeDecisionTaskScheduled.Ptr(), + DecisionTaskScheduledEventAttributes: &types.DecisionTaskScheduledEventAttributes{ + TaskList: &types.TaskList{Name: tasklist}, + StartToCloseTimeoutSeconds: common.Int32Ptr(1000), + Attempt: 0, + }, + }, + { + EventID: 14, + Version: 21, + EventType: types.EventTypeDecisionTaskStarted.Ptr(), + DecisionTaskStartedEventAttributes: &types.DecisionTaskStartedEventAttributes{ + ScheduledEventID: 13, + Identity: identity, + RequestID: uuid.New(), + }, + }, + }}, + } + + eventsBatch2 := []*types.History{ + {Events: []*types.HistoryEvent{ + { + EventID: 15, + Version: 31, + EventType: types.EventTypeDecisionTaskCompleted.Ptr(), + DecisionTaskCompletedEventAttributes: &types.DecisionTaskCompletedEventAttributes{ + ScheduledEventID: 9, + StartedEventID: 10, + Identity: identity, + }, + }, + { + EventID: 16, + Version: 31, + EventType: types.EventTypeActivityTaskScheduled.Ptr(), + ActivityTaskScheduledEventAttributes: &types.ActivityTaskScheduledEventAttributes{ + DecisionTaskCompletedEventID: 4, + ActivityID: "0", + ActivityType: &types.ActivityType{Name: "activity-type"}, + TaskList: &types.TaskList{Name: tasklist}, + Input: nil, + ScheduleToCloseTimeoutSeconds: common.Int32Ptr(20), + ScheduleToStartTimeoutSeconds: common.Int32Ptr(20), + StartToCloseTimeoutSeconds: common.Int32Ptr(20), + HeartbeatTimeoutSeconds: common.Int32Ptr(20), + }, + }, + }}, + } + + eventsBatch3 := []*types.History{ + {Events: []*types.HistoryEvent{ + { + EventID: 15, + Version: 30, + EventType: types.EventTypeDecisionTaskTimedOut.Ptr(), + DecisionTaskTimedOutEventAttributes: &types.DecisionTaskTimedOutEventAttributes{ + ScheduledEventID: 13, + StartedEventID: 14, + TimeoutType: types.TimeoutTypeStartToClose.Ptr(), + }, + }, + { + EventID: 16, + Version: 30, + EventType: types.EventTypeActivityTaskTimedOut.Ptr(), + ActivityTaskTimedOutEventAttributes: &types.ActivityTaskTimedOutEventAttributes{ + ScheduledEventID: 6, + StartedEventID: 7, + TimeoutType: types.TimeoutTypeStartToClose.Ptr(), + }, + }, + { + EventID: 17, + Version: 30, + EventType: types.EventTypeDecisionTaskScheduled.Ptr(), + DecisionTaskScheduledEventAttributes: &types.DecisionTaskScheduledEventAttributes{ + TaskList: &types.TaskList{Name: tasklist}, + StartToCloseTimeoutSeconds: common.Int32Ptr(1000), + Attempt: 0, + }, + }, + }}, + {Events: []*types.HistoryEvent{ + { + EventID: 18, + Version: 30, + EventType: types.EventTypeDecisionTaskStarted.Ptr(), + DecisionTaskStartedEventAttributes: &types.DecisionTaskStartedEventAttributes{ + ScheduledEventID: 17, + Identity: identity, + RequestID: uuid.New(), + }, + }, + }}, + {Events: []*types.HistoryEvent{ + { + EventID: 19, + Version: 30, + EventType: types.EventTypeDecisionTaskCompleted.Ptr(), + DecisionTaskCompletedEventAttributes: &types.DecisionTaskCompletedEventAttributes{ + ScheduledEventID: 8, + StartedEventID: 9, + Identity: identity, + }, + }, + { + EventID: 20, + Version: 30, + EventType: types.EventTypeWorkflowExecutionFailed.Ptr(), + WorkflowExecutionFailedEventAttributes: &types.WorkflowExecutionFailedEventAttributes{ + DecisionTaskCompletedEventID: 19, + Reason: common.StringPtr("some random reason"), + Details: nil, + }, + }, + }}, + } + + eventsBatch4 := []*types.History{ + {Events: []*types.HistoryEvent{ + { + EventID: 17, + Version: 32, + EventType: types.EventTypeWorkflowExecutionTimedOut.Ptr(), + WorkflowExecutionTimedOutEventAttributes: &types.WorkflowExecutionTimedOutEventAttributes{ + TimeoutType: types.TimeoutTypeStartToClose.Ptr(), + }, + }, + }}, + } + + versionHistory1 := s.eventBatchesToVersionHistory(nil, eventsBatch1) + + versionHistory2, err := versionHistory1.DuplicateUntilLCAItem( + persistence.NewVersionHistoryItem(14, 21), + ) + s.NoError(err) + versionHistory2 = s.eventBatchesToVersionHistory(versionHistory2, eventsBatch2) + + versionHistory3, err := versionHistory1.DuplicateUntilLCAItem( + persistence.NewVersionHistoryItem(14, 21), + ) + s.NoError(err) + versionHistory3 = s.eventBatchesToVersionHistory(versionHistory3, eventsBatch3) + + versionHistory4, err := versionHistory2.DuplicateUntilLCAItem( + persistence.NewVersionHistoryItem(16, 31), + ) + s.NoError(err) + versionHistory4 = s.eventBatchesToVersionHistory(versionHistory4, eventsBatch4) + + s.applyEvents( + workflowID, + runID, + workflowType, + tasklist, + versionHistory1, + eventsBatch1, + historyClient, + ) + s.applyEvents( + workflowID, + runID, + workflowType, + tasklist, + versionHistory3, + eventsBatch3, + historyClient, + ) + s.applyEvents( + workflowID, + runID, + workflowType, + tasklist, + versionHistory2, + eventsBatch2, + historyClient, + ) + s.applyEvents( + workflowID, + runID, + workflowType, + tasklist, + versionHistory4, + eventsBatch4, + historyClient, + ) + + // GetWorkflowExecutionRawHistoryV2 start and end + var token []byte + batchCount := 0 + for continuePaging := true; continuePaging; continuePaging = len(token) != 0 { + resp, err := getHistory( + s.domainName, + workflowID, + runID, + common.Int64Ptr(14), + common.Int64Ptr(21), + common.Int64Ptr(20), + common.Int64Ptr(30), + 1, + token, + ) + s.Nil(err) + s.True(len(resp.HistoryBatches) <= 1) + batchCount++ + token = resp.NextPageToken + } + s.Equal(batchCount, 4) + + // GetWorkflowExecutionRawHistoryV2 start and end not on the same branch + token = nil + batchCount = 0 + for continuePaging := true; continuePaging; continuePaging = len(token) != 0 { + resp, err := getHistory( + s.domainName, + workflowID, + runID, + common.Int64Ptr(17), + common.Int64Ptr(30), + common.Int64Ptr(17), + common.Int64Ptr(32), + 1, + token, + ) + s.Nil(err) + s.True(len(resp.HistoryBatches) <= 1) + batchCount++ + token = resp.NextPageToken + } + s.Equal(batchCount, 2) + + // GetWorkflowExecutionRawHistoryV2 start boundary + token = nil + batchCount = 0 + for continuePaging := true; continuePaging; continuePaging = len(token) != 0 { + resp, err := getHistory( + s.domainName, + workflowID, + runID, + common.Int64Ptr(14), + common.Int64Ptr(21), + nil, + nil, + 1, + token, + ) + s.Nil(err) + s.True(len(resp.HistoryBatches) <= 1) + batchCount++ + token = resp.NextPageToken + } + s.Equal(batchCount, 3) + + // GetWorkflowExecutionRawHistoryV2 end boundary + token = nil + batchCount = 0 + for continuePaging := true; continuePaging; continuePaging = len(token) != 0 { + resp, err := getHistory( + s.domainName, + workflowID, + runID, + nil, + nil, + common.Int64Ptr(17), + common.Int64Ptr(32), + 1, + token, + ) + s.Nil(err) + s.True(len(resp.HistoryBatches) <= 1) + batchCount++ + token = resp.NextPageToken + } + s.Equal(batchCount, 10) +} + +func (s *NDCIntegrationTestSuite) registerDomain() { + s.domainName = "test-simple-workflow-ndc-" + common.GenerateRandomString(5) + client1 := s.active.GetFrontendClient() // active + err := client1.RegisterDomain(s.createContext(), &types.RegisterDomainRequest{ + Name: s.domainName, + IsGlobalDomain: true, + Clusters: clusterReplicationConfig, + // make the active cluster `standby` and replicate to `active` cluster + ActiveClusterName: clusterName[1], + WorkflowExecutionRetentionPeriodInDays: 1, + }) + s.Require().NoError(err) + + descReq := &types.DescribeDomainRequest{ + Name: common.StringPtr(s.domainName), + } + resp, err := client1.DescribeDomain(s.createContext(), descReq) + s.Require().NoError(err) + s.Require().NotNil(resp) + s.domainID = resp.GetDomainInfo().GetUUID() + // Wait for domain cache to pick the change + time.Sleep(2 * cache.DomainCacheRefreshInterval) + + s.logger.Info(fmt.Sprintf("Domain name: %v - ID: %v", s.domainName, s.domainID)) +} + +func (s *NDCIntegrationTestSuite) generateNewRunHistory( + event *types.HistoryEvent, + domain string, + workflowID string, + runID string, + version int64, + workflowType string, + taskList string, +) *persistence.DataBlob { + + // TODO temporary code to generate first event & version history + // we should generate these as part of modeled based testing + + if event.GetWorkflowExecutionContinuedAsNewEventAttributes() == nil { + return nil + } + + event.WorkflowExecutionContinuedAsNewEventAttributes.NewExecutionRunID = uuid.New() + + newRunFirstEvent := &types.HistoryEvent{ + EventID: common.FirstEventID, + Timestamp: common.Int64Ptr(time.Now().UnixNano()), + EventType: types.EventTypeWorkflowExecutionStarted.Ptr(), + Version: version, + TaskID: 1, + WorkflowExecutionStartedEventAttributes: &types.WorkflowExecutionStartedEventAttributes{ + WorkflowType: &types.WorkflowType{Name: workflowType}, + ParentWorkflowDomain: common.StringPtr(domain), + ParentWorkflowExecution: &types.WorkflowExecution{ + WorkflowID: uuid.New(), + RunID: uuid.New(), + }, + ParentInitiatedEventID: common.Int64Ptr(event.GetEventID()), + TaskList: &types.TaskList{ + Name: taskList, + Kind: types.TaskListKindNormal.Ptr(), + }, + ExecutionStartToCloseTimeoutSeconds: common.Int32Ptr(10), + TaskStartToCloseTimeoutSeconds: common.Int32Ptr(10), + ContinuedExecutionRunID: runID, + Initiator: types.ContinueAsNewInitiatorCronSchedule.Ptr(), + OriginalExecutionRunID: runID, + Identity: "NDC-test", + FirstExecutionRunID: runID, + Attempt: 0, + ExpirationTimestamp: common.Int64Ptr(time.Now().Add(time.Minute).UnixNano()), + }, + } + + eventBlob, err := s.serializer.SerializeBatchEvents([]*types.HistoryEvent{newRunFirstEvent}, common.EncodingTypeThriftRW) + s.NoError(err) + + return eventBlob +} + +func (s *NDCIntegrationTestSuite) toInternalDataBlob( + blob *persistence.DataBlob, +) *types.DataBlob { + + if blob == nil { + return nil + } + + var encodingType types.EncodingType + switch blob.GetEncoding() { + case common.EncodingTypeThriftRW: + encodingType = types.EncodingTypeThriftRW + case common.EncodingTypeJSON, + common.EncodingTypeGob, + common.EncodingTypeUnknown, + common.EncodingTypeEmpty: + panic(fmt.Sprintf("unsupported encoding type: %v", blob.GetEncoding())) + default: + panic(fmt.Sprintf("unknown encoding type: %v", blob.GetEncoding())) + } + + return &types.DataBlob{ + EncodingType: encodingType.Ptr(), + Data: blob.Data, + } +} + +func (s *NDCIntegrationTestSuite) generateEventBlobs( + workflowID string, + runID string, + workflowType string, + tasklist string, + batch *types.History, +) (*persistence.DataBlob, *persistence.DataBlob) { + // TODO temporary code to generate next run first event + // we should generate these as part of modeled based testing + lastEvent := batch.Events[len(batch.Events)-1] + newRunEventBlob := s.generateNewRunHistory( + lastEvent, s.domainName, workflowID, runID, lastEvent.GetVersion(), workflowType, tasklist, + ) + // must serialize events batch after attempt on continue as new as generateNewRunHistory will + // modify the NewExecutionRunID attr + eventBlob, err := s.serializer.SerializeBatchEvents(batch.Events, common.EncodingTypeThriftRW) + s.NoError(err) + return eventBlob, newRunEventBlob +} + +func (s *NDCIntegrationTestSuite) applyEvents( + workflowID string, + runID string, + workflowType string, + tasklist string, + versionHistory *persistence.VersionHistory, + eventBatches []*types.History, + historyClient host.HistoryClient, +) { + for _, batch := range eventBatches { + eventBlob, newRunEventBlob := s.generateEventBlobs(workflowID, runID, workflowType, tasklist, batch) + req := &types.ReplicateEventsV2Request{ + DomainUUID: s.domainID, + WorkflowExecution: &types.WorkflowExecution{ + WorkflowID: workflowID, + RunID: runID, + }, + VersionHistoryItems: s.toInternalVersionHistoryItems(versionHistory), + Events: s.toInternalDataBlob(eventBlob), + NewRunEvents: s.toInternalDataBlob(newRunEventBlob), + } + + err := historyClient.ReplicateEventsV2(s.createContext(), req) + s.Nil(err, "Failed to replicate history event") + err = historyClient.ReplicateEventsV2(s.createContext(), req) + s.Nil(err, "Failed to dedup replicate history event") + } +} + +func (s *NDCIntegrationTestSuite) applyEventsThroughFetcher( + workflowID string, + runID string, + workflowType string, + tasklist string, + versionHistory *persistence.VersionHistory, + eventBatches []*types.History, +) { + for _, batch := range eventBatches { + eventBlob, newRunEventBlob := s.generateEventBlobs(workflowID, runID, workflowType, tasklist, batch) + + taskType := types.ReplicationTaskTypeHistoryV2 + replicationTask := &types.ReplicationTask{ + TaskType: &taskType, + SourceTaskID: 1, + HistoryTaskV2Attributes: &types.HistoryTaskV2Attributes{ + TaskID: 1, + DomainID: s.domainID, + WorkflowID: workflowID, + RunID: runID, + VersionHistoryItems: s.toInternalVersionHistoryItems(versionHistory), + Events: s.toInternalDataBlob(eventBlob), + NewRunEvents: s.toInternalDataBlob(newRunEventBlob), + }, + } + + s.standByReplicationTasksChan <- replicationTask + // this is to test whether dedup works + s.standByReplicationTasksChan <- replicationTask + } +} + +func (s *NDCIntegrationTestSuite) eventBatchesToVersionHistory( + versionHistory *persistence.VersionHistory, + eventBatches []*types.History, +) *persistence.VersionHistory { + + // TODO temporary code to generate version history + // we should generate version as part of modeled based testing + if versionHistory == nil { + versionHistory = persistence.NewVersionHistory(nil, nil) + } + for _, batch := range eventBatches { + for _, event := range batch.Events { + err := versionHistory.AddOrUpdateItem( + persistence.NewVersionHistoryItem( + event.GetEventID(), + event.GetVersion(), + )) + s.NoError(err) + } + } + + return versionHistory +} + +func (s *NDCIntegrationTestSuite) toInternalVersionHistoryItems( + versionHistory *persistence.VersionHistory, +) []*types.VersionHistoryItem { + if versionHistory == nil { + return nil + } + + return versionHistory.ToInternalType().Items +} + +func (s *NDCIntegrationTestSuite) createContext() context.Context { + ctx, _ := context.WithTimeout(context.Background(), 90*time.Second) + return ctx +} + +func (s *NDCIntegrationTestSuite) setupRemoteFrontendClients() { + s.mockAdminClient["standby"].(*adminClient.MockClient).EXPECT().ReapplyEvents(gomock.Any(), gomock.Any()).Return(nil).AnyTimes() + s.mockAdminClient["other"].(*adminClient.MockClient).EXPECT().ReapplyEvents(gomock.Any(), gomock.Any()).Return(nil).AnyTimes() +} diff --git a/host/ndc/nDC_integration_test.go b/host/ndc/nDC_integration_test.go index 77961a63155..227d5e4d7d5 100644 --- a/host/ndc/nDC_integration_test.go +++ b/host/ndc/nDC_integration_test.go @@ -1,4 +1,4 @@ -// Copyright (c) 2019 Uber Technologies, Inc. +// Copyright (c) 2016 Uber Technologies, Inc. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -21,74 +21,14 @@ package ndc import ( - "context" "flag" - "fmt" - "sync/atomic" "testing" - "time" - "go.uber.org/yarpc" - "github.com/golang/mock/gomock" - "github.com/pborman/uuid" - "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" - "go.uber.org/zap" - - adminClient "github.com/uber/cadence/client/admin" - "github.com/uber/cadence/common" - "github.com/uber/cadence/common/cache" - "github.com/uber/cadence/common/log" - "github.com/uber/cadence/common/log/loggerimpl" - "github.com/uber/cadence/common/log/tag" - "github.com/uber/cadence/common/persistence" - pt "github.com/uber/cadence/common/persistence/persistence-tests" - test "github.com/uber/cadence/common/testing" - "github.com/uber/cadence/common/types" "github.com/uber/cadence/host" ) -type ( - NDCIntegrationTestSuite struct { - // override suite.Suite.Assertions with require.Assertions; this means that s.NotNil(nil) will stop the test, - // not merely log an error - *require.Assertions - suite.Suite - active *host.TestCluster - generator test.Generator - serializer persistence.PayloadSerializer - logger log.Logger - - domainName string - domainID string - version int64 - versionIncrement int64 - mockAdminClient map[string]adminClient.Client - standByReplicationTasksChan chan *types.ReplicationTask - standByTaskID int64 - - clusterConfigs []*host.TestClusterConfig - defaultTestCluster pt.PersistenceTestCluster - visibilityTestCluster pt.PersistenceTestCluster - } - - NDCIntegrationTestSuiteParams struct { - ClusterConfigs []*host.TestClusterConfig - DefaultTestCluster pt.PersistenceTestCluster - VisibilityTestCluster pt.PersistenceTestCluster - } -) - -var ( - clusterName = []string{"active", "standby", "other"} - clusterReplicationConfig = []*types.ClusterReplicationConfiguration{ - {ClusterName: clusterName[0]}, - {ClusterName: clusterName[1]}, - {ClusterName: clusterName[2]}, - } -) - func TestNDCIntegrationTestSuite(t *testing.T) { flag.Parse() @@ -107,1716 +47,3 @@ func TestNDCIntegrationTestSuite(t *testing.T) { s := NewNDCIntegrationTestSuite(params) suite.Run(t, s) } - -func NewNDCIntegrationTestSuite(params NDCIntegrationTestSuiteParams) *NDCIntegrationTestSuite { - return &NDCIntegrationTestSuite{ - clusterConfigs: params.ClusterConfigs, - defaultTestCluster: params.DefaultTestCluster, - visibilityTestCluster: params.VisibilityTestCluster, - } -} - -func (s *NDCIntegrationTestSuite) SetupSuite() { - zapLogger, err := zap.NewDevelopment() - // cannot use s.Nil since it is not initialized - s.Require().NoError(err) - s.serializer = persistence.NewPayloadSerializer() - s.logger = loggerimpl.NewLogger(zapLogger) - - s.standByReplicationTasksChan = make(chan *types.ReplicationTask, 100) - - s.standByTaskID = 0 - s.mockAdminClient = make(map[string]adminClient.Client) - controller := gomock.NewController(s.T()) - mockStandbyClient := adminClient.NewMockClient(controller) - mockStandbyClient.EXPECT().GetReplicationMessages(gomock.Any(), gomock.Any()).DoAndReturn(s.GetReplicationMessagesMock).AnyTimes() - mockOtherClient := adminClient.NewMockClient(controller) - mockOtherClient.EXPECT().GetReplicationMessages(gomock.Any(), gomock.Any()).Return( - &types.GetReplicationMessagesResponse{ - MessagesByShard: make(map[int32]*types.ReplicationMessages), - }, nil).AnyTimes() - s.mockAdminClient["standby"] = mockStandbyClient - s.mockAdminClient["other"] = mockOtherClient - s.clusterConfigs[0].MockAdminClient = s.mockAdminClient - - clusterMetadata := host.NewClusterMetadata(s.clusterConfigs[0], s.logger.WithTags(tag.ClusterName(clusterName[0]))) - params := pt.TestBaseParams{ - DefaultTestCluster: s.defaultTestCluster, - VisibilityTestCluster: s.visibilityTestCluster, - ClusterMetadata: clusterMetadata, - } - cluster, err := host.NewCluster(s.clusterConfigs[0], s.logger.WithTags(tag.ClusterName(clusterName[0])), params) - s.Require().NoError(err) - s.active = cluster - - s.registerDomain() - - s.version = s.clusterConfigs[1].ClusterMetadata.ClusterInformation[s.clusterConfigs[1].ClusterMetadata.CurrentClusterName].InitialFailoverVersion - s.versionIncrement = s.clusterConfigs[0].ClusterMetadata.FailoverVersionIncrement - s.generator = test.InitializeHistoryEventGenerator(s.domainName, s.version) -} - -func (s *NDCIntegrationTestSuite) GetReplicationMessagesMock( - ctx context.Context, - request *types.GetReplicationMessagesRequest, - opts ...yarpc.CallOption, -) (*types.GetReplicationMessagesResponse, error) { - select { - case task := <-s.standByReplicationTasksChan: - taskID := atomic.AddInt64(&s.standByTaskID, 1) - task.SourceTaskID = taskID - tasks := []*types.ReplicationTask{task} - for len(s.standByReplicationTasksChan) > 0 { - task = <-s.standByReplicationTasksChan - taskID := atomic.AddInt64(&s.standByTaskID, 1) - task.SourceTaskID = taskID - tasks = append(tasks, task) - } - - replicationMessage := &types.ReplicationMessages{ - ReplicationTasks: tasks, - LastRetrievedMessageID: tasks[len(tasks)-1].SourceTaskID, - HasMore: true, - } - - return &types.GetReplicationMessagesResponse{ - MessagesByShard: map[int32]*types.ReplicationMessages{0: replicationMessage}, - }, nil - default: - return &types.GetReplicationMessagesResponse{ - MessagesByShard: make(map[int32]*types.ReplicationMessages), - }, nil - } -} - -func (s *NDCIntegrationTestSuite) SetupTest() { - // Have to define our overridden assertions in the test setup. If we did it earlier, s.T() will return nil - s.Assertions = require.New(s.T()) - s.generator = test.InitializeHistoryEventGenerator(s.domainName, s.version) -} - -func (s *NDCIntegrationTestSuite) TearDownSuite() { - if s.generator != nil { - s.generator.Reset() - } - s.active.TearDownCluster() -} - -func (s *NDCIntegrationTestSuite) TestSingleBranch() { - - s.setupRemoteFrontendClients() - workflowID := "ndc-single-branch-test" + uuid.New() - - workflowType := "event-generator-workflow-type" - tasklist := "event-generator-taskList" - - // active has initial version 0 - historyClient := s.active.GetHistoryClient() - - versions := []int64{101, 1, 201, 301, 401, 601, 501, 801, 1001, 901, 701, 1101} - for _, version := range versions { - runID := uuid.New() - historyBatch := []*types.History{} - s.generator = test.InitializeHistoryEventGenerator(s.domainName, version) - - for s.generator.HasNextVertex() { - events := s.generator.GetNextVertices() - historyEvents := &types.History{} - for _, event := range events { - historyEvents.Events = append(historyEvents.Events, event.GetData().(*types.HistoryEvent)) - } - historyBatch = append(historyBatch, historyEvents) - } - - versionHistory := s.eventBatchesToVersionHistory(nil, historyBatch) - s.applyEvents( - workflowID, - runID, - workflowType, - tasklist, - versionHistory, - historyBatch, - historyClient, - ) - - err := s.verifyEventHistory(workflowID, runID, historyBatch) - s.Require().NoError(err) - } -} - -func (s *NDCIntegrationTestSuite) verifyEventHistory( - workflowID string, - runID string, - historyBatch []*types.History, -) error { - // get replicated history events from passive side - passiveClient := s.active.GetFrontendClient() - replicatedHistory, err := passiveClient.GetWorkflowExecutionHistory( - s.createContext(), - &types.GetWorkflowExecutionHistoryRequest{ - Domain: s.domainName, - Execution: &types.WorkflowExecution{ - WorkflowID: workflowID, - RunID: runID, - }, - MaximumPageSize: 1000, - NextPageToken: nil, - WaitForNewEvent: false, - HistoryEventFilterType: types.HistoryEventFilterTypeAllEvent.Ptr(), - }, - ) - - if err != nil { - return fmt.Errorf("failed to get history event from passive side: %v", err) - } - - // compare origin events with replicated events - batchIndex := 0 - batch := historyBatch[batchIndex].Events - eventIndex := 0 - for _, event := range replicatedHistory.GetHistory().GetEvents() { - if eventIndex >= len(batch) { - batchIndex++ - batch = historyBatch[batchIndex].Events - eventIndex = 0 - } - originEvent := batch[eventIndex] - eventIndex++ - if originEvent.GetEventType() != event.GetEventType() { - return fmt.Errorf("the replicated event (%v) and the origin event (%v) are not the same", - originEvent.GetEventType().String(), event.GetEventType().String()) - } - } - - return nil -} - -func (s *NDCIntegrationTestSuite) TestMultipleBranches() { - - s.setupRemoteFrontendClients() - workflowID := "ndc-multiple-branches-test" + uuid.New() - - workflowType := "event-generator-workflow-type" - tasklist := "event-generator-taskList" - - // active has initial version 0 - historyClient := s.active.GetHistoryClient() - - versions := []int64{101, 1, 201} - for _, version := range versions { - runID := uuid.New() - - baseBranch := []*types.History{} - baseGenerator := test.InitializeHistoryEventGenerator(s.domainName, version) - baseGenerator.SetVersion(version) - - for i := 0; i < 10 && baseGenerator.HasNextVertex(); i++ { - events := baseGenerator.GetNextVertices() - historyEvents := &types.History{} - for _, event := range events { - historyEvents.Events = append(historyEvents.Events, event.GetData().(*types.HistoryEvent)) - } - baseBranch = append(baseBranch, historyEvents) - } - baseVersionHistory := s.eventBatchesToVersionHistory(nil, baseBranch) - - branch1 := []*types.History{} - branchVersionHistory1 := baseVersionHistory.Duplicate() - branchGenerator1 := baseGenerator.DeepCopy() - for i := 0; i < 10 && branchGenerator1.HasNextVertex(); i++ { - events := branchGenerator1.GetNextVertices() - historyEvents := &types.History{} - for _, event := range events { - historyEvents.Events = append(historyEvents.Events, event.GetData().(*types.HistoryEvent)) - } - branch1 = append(branch1, historyEvents) - } - branchVersionHistory1 = s.eventBatchesToVersionHistory(branchVersionHistory1, branch1) - - branch2 := []*types.History{} - branchVersionHistory2 := baseVersionHistory.Duplicate() - branchGenerator2 := baseGenerator.DeepCopy() - branchGenerator2.SetVersion(branchGenerator2.GetVersion() + 1) - for i := 0; i < 10 && branchGenerator2.HasNextVertex(); i++ { - events := branchGenerator2.GetNextVertices() - historyEvents := &types.History{} - for _, event := range events { - historyEvents.Events = append(historyEvents.Events, event.GetData().(*types.HistoryEvent)) - } - branch2 = append(branch2, historyEvents) - } - branchVersionHistory2 = s.eventBatchesToVersionHistory(branchVersionHistory2, branch2) - - s.applyEvents( - workflowID, - runID, - workflowType, - tasklist, - baseVersionHistory, - baseBranch, - historyClient, - ) - s.applyEvents( - workflowID, - runID, - workflowType, - tasklist, - branchVersionHistory1, - branch1, - historyClient, - ) - s.applyEvents( - workflowID, - runID, - workflowType, - tasklist, - branchVersionHistory2, - branch2, - historyClient, - ) - } -} - -func (s *NDCIntegrationTestSuite) TestHandcraftedMultipleBranches() { - - s.setupRemoteFrontendClients() - workflowID := "ndc-handcrafted-multiple-branches-test" + uuid.New() - runID := uuid.New() - - workflowType := "event-generator-workflow-type" - tasklist := "event-generator-taskList" - identity := "worker-identity" - - // active has initial version 0 - historyClient := s.active.GetHistoryClient() - - eventsBatch1 := []*types.History{ - {Events: []*types.HistoryEvent{ - { - EventID: 1, - Version: 21, - EventType: types.EventTypeWorkflowExecutionStarted.Ptr(), - WorkflowExecutionStartedEventAttributes: &types.WorkflowExecutionStartedEventAttributes{ - WorkflowType: &types.WorkflowType{Name: workflowType}, - TaskList: &types.TaskList{Name: tasklist}, - Input: nil, - ExecutionStartToCloseTimeoutSeconds: common.Int32Ptr(1000), - TaskStartToCloseTimeoutSeconds: common.Int32Ptr(1000), - FirstDecisionTaskBackoffSeconds: common.Int32Ptr(100), - }, - }, - { - EventID: 2, - Version: 21, - EventType: types.EventTypeDecisionTaskScheduled.Ptr(), - DecisionTaskScheduledEventAttributes: &types.DecisionTaskScheduledEventAttributes{ - TaskList: &types.TaskList{Name: tasklist}, - StartToCloseTimeoutSeconds: common.Int32Ptr(1000), - Attempt: 0, - }, - }, - }}, - {Events: []*types.HistoryEvent{ - { - EventID: 3, - Version: 21, - EventType: types.EventTypeDecisionTaskStarted.Ptr(), - DecisionTaskStartedEventAttributes: &types.DecisionTaskStartedEventAttributes{ - ScheduledEventID: 2, - Identity: identity, - RequestID: uuid.New(), - }, - }, - }}, - {Events: []*types.HistoryEvent{ - { - EventID: 4, - Version: 21, - EventType: types.EventTypeDecisionTaskCompleted.Ptr(), - DecisionTaskCompletedEventAttributes: &types.DecisionTaskCompletedEventAttributes{ - ScheduledEventID: 2, - StartedEventID: 3, - Identity: identity, - }, - }, - { - EventID: 5, - Version: 21, - EventType: types.EventTypeMarkerRecorded.Ptr(), - MarkerRecordedEventAttributes: &types.MarkerRecordedEventAttributes{ - MarkerName: "some marker name", - Details: []byte("some marker details"), - DecisionTaskCompletedEventID: 4, - }, - }, - { - EventID: 6, - Version: 21, - EventType: types.EventTypeActivityTaskScheduled.Ptr(), - ActivityTaskScheduledEventAttributes: &types.ActivityTaskScheduledEventAttributes{ - DecisionTaskCompletedEventID: 4, - ActivityID: "0", - ActivityType: &types.ActivityType{Name: "activity-type"}, - TaskList: &types.TaskList{Name: tasklist}, - Input: nil, - ScheduleToCloseTimeoutSeconds: common.Int32Ptr(20), - ScheduleToStartTimeoutSeconds: common.Int32Ptr(20), - StartToCloseTimeoutSeconds: common.Int32Ptr(20), - HeartbeatTimeoutSeconds: common.Int32Ptr(20), - }, - }, - }}, - {Events: []*types.HistoryEvent{ - { - EventID: 7, - Version: 21, - EventType: types.EventTypeActivityTaskStarted.Ptr(), - ActivityTaskStartedEventAttributes: &types.ActivityTaskStartedEventAttributes{ - ScheduledEventID: 6, - Identity: identity, - RequestID: uuid.New(), - Attempt: 0, - }, - }, - }}, - {Events: []*types.HistoryEvent{ - { - EventID: 8, - Version: 21, - EventType: types.EventTypeWorkflowExecutionSignaled.Ptr(), - WorkflowExecutionSignaledEventAttributes: &types.WorkflowExecutionSignaledEventAttributes{ - SignalName: "some signal name 1", - Input: []byte("some signal details 1"), - Identity: identity, - }, - }, - { - EventID: 9, - Version: 21, - EventType: types.EventTypeDecisionTaskScheduled.Ptr(), - DecisionTaskScheduledEventAttributes: &types.DecisionTaskScheduledEventAttributes{ - TaskList: &types.TaskList{Name: tasklist}, - StartToCloseTimeoutSeconds: common.Int32Ptr(1000), - Attempt: 0, - }, - }, - }}, - {Events: []*types.HistoryEvent{ - { - EventID: 10, - Version: 21, - EventType: types.EventTypeDecisionTaskStarted.Ptr(), - DecisionTaskStartedEventAttributes: &types.DecisionTaskStartedEventAttributes{ - ScheduledEventID: 9, - Identity: identity, - RequestID: uuid.New(), - }, - }, - }}, - {Events: []*types.HistoryEvent{ - { - EventID: 11, - Version: 21, - EventType: types.EventTypeDecisionTaskCompleted.Ptr(), - DecisionTaskCompletedEventAttributes: &types.DecisionTaskCompletedEventAttributes{ - ScheduledEventID: 9, - StartedEventID: 10, - Identity: identity, - }, - }, - { - EventID: 12, - Version: 21, - EventType: types.EventTypeWorkflowExecutionSignaled.Ptr(), - WorkflowExecutionSignaledEventAttributes: &types.WorkflowExecutionSignaledEventAttributes{ - SignalName: "some signal name 2", - Input: []byte("some signal details 2"), - Identity: identity, - }, - }, - { - EventID: 13, - Version: 21, - EventType: types.EventTypeDecisionTaskScheduled.Ptr(), - DecisionTaskScheduledEventAttributes: &types.DecisionTaskScheduledEventAttributes{ - TaskList: &types.TaskList{Name: tasklist}, - StartToCloseTimeoutSeconds: common.Int32Ptr(1000), - Attempt: 0, - }, - }, - { - EventID: 14, - Version: 21, - EventType: types.EventTypeDecisionTaskStarted.Ptr(), - DecisionTaskStartedEventAttributes: &types.DecisionTaskStartedEventAttributes{ - ScheduledEventID: 13, - Identity: identity, - RequestID: uuid.New(), - }, - }, - }}, - } - - eventsBatch2 := []*types.History{ - {Events: []*types.HistoryEvent{ - { - EventID: 15, - Version: 31, - EventType: types.EventTypeWorkflowExecutionTimedOut.Ptr(), - WorkflowExecutionTimedOutEventAttributes: &types.WorkflowExecutionTimedOutEventAttributes{ - TimeoutType: types.TimeoutTypeStartToClose.Ptr(), - }, - }, - }}, - } - - eventsBatch3 := []*types.History{ - {Events: []*types.HistoryEvent{ - { - EventID: 15, - Version: 30, - EventType: types.EventTypeDecisionTaskTimedOut.Ptr(), - DecisionTaskTimedOutEventAttributes: &types.DecisionTaskTimedOutEventAttributes{ - ScheduledEventID: 13, - StartedEventID: 14, - TimeoutType: types.TimeoutTypeStartToClose.Ptr(), - }, - }, - { - EventID: 16, - Version: 30, - EventType: types.EventTypeActivityTaskTimedOut.Ptr(), - ActivityTaskTimedOutEventAttributes: &types.ActivityTaskTimedOutEventAttributes{ - ScheduledEventID: 6, - StartedEventID: 7, - TimeoutType: types.TimeoutTypeStartToClose.Ptr(), - }, - }, - { - EventID: 17, - Version: 30, - EventType: types.EventTypeDecisionTaskScheduled.Ptr(), - DecisionTaskScheduledEventAttributes: &types.DecisionTaskScheduledEventAttributes{ - TaskList: &types.TaskList{Name: tasklist}, - StartToCloseTimeoutSeconds: common.Int32Ptr(1000), - Attempt: 0, - }, - }, - }}, - {Events: []*types.HistoryEvent{ - { - EventID: 18, - Version: 30, - EventType: types.EventTypeDecisionTaskStarted.Ptr(), - DecisionTaskStartedEventAttributes: &types.DecisionTaskStartedEventAttributes{ - ScheduledEventID: 17, - Identity: identity, - RequestID: uuid.New(), - }, - }, - }}, - {Events: []*types.HistoryEvent{ - { - EventID: 19, - Version: 30, - EventType: types.EventTypeDecisionTaskCompleted.Ptr(), - DecisionTaskCompletedEventAttributes: &types.DecisionTaskCompletedEventAttributes{ - ScheduledEventID: 8, - StartedEventID: 9, - Identity: identity, - }, - }, - { - EventID: 20, - Version: 30, - EventType: types.EventTypeWorkflowExecutionFailed.Ptr(), - WorkflowExecutionFailedEventAttributes: &types.WorkflowExecutionFailedEventAttributes{ - DecisionTaskCompletedEventID: 19, - Reason: common.StringPtr("some random reason"), - Details: nil, - }, - }, - }}, - } - - versionHistory1 := s.eventBatchesToVersionHistory(nil, eventsBatch1) - - versionHistory2, err := versionHistory1.DuplicateUntilLCAItem( - persistence.NewVersionHistoryItem(14, 21), - ) - s.NoError(err) - versionHistory2 = s.eventBatchesToVersionHistory(versionHistory2, eventsBatch2) - - versionHistory3, err := versionHistory1.DuplicateUntilLCAItem( - persistence.NewVersionHistoryItem(14, 21), - ) - s.NoError(err) - versionHistory3 = s.eventBatchesToVersionHistory(versionHistory3, eventsBatch3) - - s.applyEvents( - workflowID, - runID, - workflowType, - tasklist, - versionHistory1, - eventsBatch1, - historyClient, - ) - s.applyEvents( - workflowID, - runID, - workflowType, - tasklist, - versionHistory3, - eventsBatch3, - historyClient, - ) - s.applyEvents( - workflowID, - runID, - workflowType, - tasklist, - versionHistory2, - eventsBatch2, - historyClient, - ) -} - -func (s *NDCIntegrationTestSuite) TestHandcraftedMultipleBranchesWithZombieContinueAsNew() { - - s.setupRemoteFrontendClients() - workflowID := "ndc-handcrafted-multiple-branches-with-continue-as-new-test" + uuid.New() - runID := uuid.New() - - workflowType := "event-generator-workflow-type" - tasklist := "event-generator-taskList" - identity := "worker-identity" - - // active has initial version 0 - historyClient := s.active.GetHistoryClient() - - eventsBatch1 := []*types.History{ - {Events: []*types.HistoryEvent{ - { - EventID: 1, - Version: 21, - EventType: types.EventTypeWorkflowExecutionStarted.Ptr(), - WorkflowExecutionStartedEventAttributes: &types.WorkflowExecutionStartedEventAttributes{ - WorkflowType: &types.WorkflowType{Name: workflowType}, - TaskList: &types.TaskList{Name: tasklist}, - Input: nil, - ExecutionStartToCloseTimeoutSeconds: common.Int32Ptr(1000), - TaskStartToCloseTimeoutSeconds: common.Int32Ptr(1000), - FirstDecisionTaskBackoffSeconds: common.Int32Ptr(100), - }, - }, - { - EventID: 2, - Version: 21, - EventType: types.EventTypeDecisionTaskScheduled.Ptr(), - DecisionTaskScheduledEventAttributes: &types.DecisionTaskScheduledEventAttributes{ - TaskList: &types.TaskList{Name: tasklist}, - StartToCloseTimeoutSeconds: common.Int32Ptr(1000), - Attempt: 0, - }, - }, - }}, - {Events: []*types.HistoryEvent{ - { - EventID: 3, - Version: 21, - EventType: types.EventTypeDecisionTaskStarted.Ptr(), - DecisionTaskStartedEventAttributes: &types.DecisionTaskStartedEventAttributes{ - ScheduledEventID: 2, - Identity: identity, - RequestID: uuid.New(), - }, - }, - }}, - {Events: []*types.HistoryEvent{ - { - EventID: 4, - Version: 21, - EventType: types.EventTypeDecisionTaskCompleted.Ptr(), - DecisionTaskCompletedEventAttributes: &types.DecisionTaskCompletedEventAttributes{ - ScheduledEventID: 2, - StartedEventID: 3, - Identity: identity, - }, - }, - { - EventID: 5, - Version: 21, - EventType: types.EventTypeMarkerRecorded.Ptr(), - MarkerRecordedEventAttributes: &types.MarkerRecordedEventAttributes{ - MarkerName: "some marker name", - Details: []byte("some marker details"), - DecisionTaskCompletedEventID: 4, - }, - }, - { - EventID: 6, - Version: 21, - EventType: types.EventTypeActivityTaskScheduled.Ptr(), - ActivityTaskScheduledEventAttributes: &types.ActivityTaskScheduledEventAttributes{ - DecisionTaskCompletedEventID: 4, - ActivityID: "0", - ActivityType: &types.ActivityType{Name: "activity-type"}, - TaskList: &types.TaskList{Name: tasklist}, - Input: nil, - ScheduleToCloseTimeoutSeconds: common.Int32Ptr(20), - ScheduleToStartTimeoutSeconds: common.Int32Ptr(20), - StartToCloseTimeoutSeconds: common.Int32Ptr(20), - HeartbeatTimeoutSeconds: common.Int32Ptr(20), - }, - }, - }}, - {Events: []*types.HistoryEvent{ - { - EventID: 7, - Version: 21, - EventType: types.EventTypeActivityTaskStarted.Ptr(), - ActivityTaskStartedEventAttributes: &types.ActivityTaskStartedEventAttributes{ - ScheduledEventID: 6, - Identity: identity, - RequestID: uuid.New(), - Attempt: 0, - }, - }, - }}, - {Events: []*types.HistoryEvent{ - { - EventID: 8, - Version: 21, - EventType: types.EventTypeWorkflowExecutionSignaled.Ptr(), - WorkflowExecutionSignaledEventAttributes: &types.WorkflowExecutionSignaledEventAttributes{ - SignalName: "some signal name 1", - Input: []byte("some signal details 1"), - Identity: identity, - }, - }, - { - EventID: 9, - Version: 21, - EventType: types.EventTypeDecisionTaskScheduled.Ptr(), - DecisionTaskScheduledEventAttributes: &types.DecisionTaskScheduledEventAttributes{ - TaskList: &types.TaskList{Name: tasklist}, - StartToCloseTimeoutSeconds: common.Int32Ptr(1000), - Attempt: 0, - }, - }, - }}, - {Events: []*types.HistoryEvent{ - { - EventID: 10, - Version: 21, - EventType: types.EventTypeDecisionTaskStarted.Ptr(), - DecisionTaskStartedEventAttributes: &types.DecisionTaskStartedEventAttributes{ - ScheduledEventID: 9, - Identity: identity, - RequestID: uuid.New(), - }, - }, - }}, - {Events: []*types.HistoryEvent{ - { - EventID: 11, - Version: 21, - EventType: types.EventTypeDecisionTaskCompleted.Ptr(), - DecisionTaskCompletedEventAttributes: &types.DecisionTaskCompletedEventAttributes{ - ScheduledEventID: 9, - StartedEventID: 10, - Identity: identity, - }, - }, - { - EventID: 12, - Version: 21, - EventType: types.EventTypeWorkflowExecutionSignaled.Ptr(), - WorkflowExecutionSignaledEventAttributes: &types.WorkflowExecutionSignaledEventAttributes{ - SignalName: "some signal name 2", - Input: []byte("some signal details 2"), - Identity: identity, - }, - }, - { - EventID: 13, - Version: 21, - EventType: types.EventTypeDecisionTaskScheduled.Ptr(), - DecisionTaskScheduledEventAttributes: &types.DecisionTaskScheduledEventAttributes{ - TaskList: &types.TaskList{Name: tasklist}, - StartToCloseTimeoutSeconds: common.Int32Ptr(1000), - Attempt: 0, - }, - }, - { - EventID: 14, - Version: 21, - EventType: types.EventTypeDecisionTaskStarted.Ptr(), - DecisionTaskStartedEventAttributes: &types.DecisionTaskStartedEventAttributes{ - ScheduledEventID: 13, - Identity: identity, - RequestID: uuid.New(), - }, - }, - }}, - } - - eventsBatch2 := []*types.History{ - {Events: []*types.HistoryEvent{ - { - EventID: 15, - Version: 32, - EventType: types.EventTypeDecisionTaskCompleted.Ptr(), - DecisionTaskCompletedEventAttributes: &types.DecisionTaskCompletedEventAttributes{ - ScheduledEventID: 8, - StartedEventID: 9, - Identity: identity, - }, - }, - }}, - // need to keep the workflow open for testing - } - - eventsBatch3 := []*types.History{ - {Events: []*types.HistoryEvent{ - { - EventID: 15, - Version: 21, - EventType: types.EventTypeDecisionTaskCompleted.Ptr(), - DecisionTaskCompletedEventAttributes: &types.DecisionTaskCompletedEventAttributes{ - ScheduledEventID: 8, - StartedEventID: 9, - Identity: identity, - }, - }, - { - EventID: 16, - Version: 21, - EventType: types.EventTypeWorkflowExecutionContinuedAsNew.Ptr(), - WorkflowExecutionContinuedAsNewEventAttributes: &types.WorkflowExecutionContinuedAsNewEventAttributes{ - NewExecutionRunID: uuid.New(), - WorkflowType: &types.WorkflowType{Name: workflowType}, - TaskList: &types.TaskList{Name: tasklist}, - Input: nil, - ExecutionStartToCloseTimeoutSeconds: common.Int32Ptr(1000), - TaskStartToCloseTimeoutSeconds: common.Int32Ptr(1000), - DecisionTaskCompletedEventID: 19, - Initiator: types.ContinueAsNewInitiatorDecider.Ptr(), - }, - }, - }}, - } - - versionHistory1 := s.eventBatchesToVersionHistory(nil, eventsBatch1) - - versionHistory2, err := versionHistory1.DuplicateUntilLCAItem( - persistence.NewVersionHistoryItem(14, 21), - ) - s.NoError(err) - versionHistory2 = s.eventBatchesToVersionHistory(versionHistory2, eventsBatch2) - - versionHistory3, err := versionHistory1.DuplicateUntilLCAItem( - persistence.NewVersionHistoryItem(14, 21), - ) - s.NoError(err) - versionHistory3 = s.eventBatchesToVersionHistory(versionHistory3, eventsBatch3) - - s.applyEvents( - workflowID, - runID, - workflowType, - tasklist, - versionHistory1, - eventsBatch1, - historyClient, - ) - s.applyEvents( - workflowID, - runID, - workflowType, - tasklist, - versionHistory2, - eventsBatch2, - historyClient, - ) - s.applyEvents( - workflowID, - runID, - workflowType, - tasklist, - versionHistory3, - eventsBatch3, - historyClient, - ) -} - -func (s *NDCIntegrationTestSuite) TestEventsReapply_ZombieWorkflow() { - - workflowID := "ndc-single-branch-test" + uuid.New() - - workflowType := "event-generator-workflow-type" - tasklist := "event-generator-taskList" - - // active has initial version 0 - historyClient := s.active.GetHistoryClient() - - version := int64(101) - runID := uuid.New() - historyBatch := []*types.History{} - s.generator = test.InitializeHistoryEventGenerator(s.domainName, version) - - for s.generator.HasNextVertex() { - events := s.generator.GetNextVertices() - historyEvents := &types.History{} - for _, event := range events { - historyEvents.Events = append(historyEvents.Events, event.GetData().(*types.HistoryEvent)) - } - historyBatch = append(historyBatch, historyEvents) - } - - versionHistory := s.eventBatchesToVersionHistory(nil, historyBatch) - s.applyEvents( - workflowID, - runID, - workflowType, - tasklist, - versionHistory, - historyBatch, - historyClient, - ) - - version = int64(1) - runID = uuid.New() - historyBatch = []*types.History{} - s.generator = test.InitializeHistoryEventGenerator(s.domainName, version) - - // verify two batches of zombie workflow are call reapply API - s.mockAdminClient["standby"].(*adminClient.MockClient).EXPECT().ReapplyEvents(gomock.Any(), gomock.Any()).Return(nil).Times(2) - for i := 0; i < 2 && s.generator.HasNextVertex(); i++ { - events := s.generator.GetNextVertices() - historyEvents := &types.History{} - for _, event := range events { - historyEvents.Events = append(historyEvents.Events, event.GetData().(*types.HistoryEvent)) - } - historyBatch = append(historyBatch, historyEvents) - } - - versionHistory = s.eventBatchesToVersionHistory(nil, historyBatch) - s.applyEvents( - workflowID, - runID, - workflowType, - tasklist, - versionHistory, - historyBatch, - historyClient, - ) -} - -func (s *NDCIntegrationTestSuite) TestEventsReapply_UpdateNonCurrentBranch() { - - workflowID := "ndc-single-branch-test" + uuid.New() - runID := uuid.New() - workflowType := "event-generator-workflow-type" - tasklist := "event-generator-taskList" - version := int64(101) - isWorkflowFinished := false - - historyClient := s.active.GetHistoryClient() - - s.generator = test.InitializeHistoryEventGenerator(s.domainName, version) - baseBranch := []*types.History{} - var taskID int64 - for i := 0; i < 4 && s.generator.HasNextVertex(); i++ { - events := s.generator.GetNextVertices() - historyEvents := &types.History{} - for _, event := range events { - historyEvent := event.GetData().(*types.HistoryEvent) - taskID = historyEvent.GetTaskID() - historyEvents.Events = append(historyEvents.Events, historyEvent) - switch historyEvent.GetEventType() { - case types.EventTypeWorkflowExecutionCompleted, - types.EventTypeWorkflowExecutionFailed, - types.EventTypeWorkflowExecutionTimedOut, - types.EventTypeWorkflowExecutionTerminated, - types.EventTypeWorkflowExecutionContinuedAsNew, - types.EventTypeWorkflowExecutionCanceled: - isWorkflowFinished = true - } - } - baseBranch = append(baseBranch, historyEvents) - } - if isWorkflowFinished { - // cannot proceed since the test below requires workflow not finished - // this is ok since build kite will run this test several times - s.logger.Info("Encounter finish workflow history event during randomization test, skip") - return - } - - versionHistory := s.eventBatchesToVersionHistory(nil, baseBranch) - s.applyEvents( - workflowID, - runID, - workflowType, - tasklist, - versionHistory, - baseBranch, - historyClient, - ) - - newGenerator := s.generator.DeepCopy() - newBranch := []*types.History{} - newVersionHistory := versionHistory.Duplicate() - newGenerator.SetVersion(newGenerator.GetVersion() + 1) // simulate events from other cluster - for i := 0; i < 4 && newGenerator.HasNextVertex(); i++ { - events := newGenerator.GetNextVertices() - historyEvents := &types.History{} - for _, event := range events { - history := event.GetData().(*types.HistoryEvent) - taskID = history.GetTaskID() - historyEvents.Events = append(historyEvents.Events, history) - } - newBranch = append(newBranch, historyEvents) - } - newVersionHistory = s.eventBatchesToVersionHistory(newVersionHistory, newBranch) - s.applyEvents( - workflowID, - runID, - workflowType, - tasklist, - newVersionHistory, - newBranch, - historyClient, - ) - - s.mockAdminClient["standby"].(*adminClient.MockClient).EXPECT().ReapplyEvents(gomock.Any(), gomock.Any()).Return(nil).Times(1) - // Handcraft a stale signal event - baseBranchLastEventBatch := baseBranch[len(baseBranch)-1].GetEvents() - baseBranchLastEvent := baseBranchLastEventBatch[len(baseBranchLastEventBatch)-1] - staleBranch := []*types.History{ - { - Events: []*types.HistoryEvent{ - { - EventID: baseBranchLastEvent.GetEventID() + 1, - EventType: types.EventTypeWorkflowExecutionSignaled.Ptr(), - Timestamp: common.Int64Ptr(time.Now().UnixNano()), - Version: baseBranchLastEvent.GetVersion(), // dummy event from other cluster - TaskID: taskID, - WorkflowExecutionSignaledEventAttributes: &types.WorkflowExecutionSignaledEventAttributes{ - SignalName: "signal", - Input: []byte{}, - Identity: "ndc_integration_test", - }, - }, - }, - }, - } - staleVersionHistory := s.eventBatchesToVersionHistory(versionHistory.Duplicate(), staleBranch) - s.applyEvents( - workflowID, - runID, - workflowType, - tasklist, - staleVersionHistory, - staleBranch, - historyClient, - ) -} - -func (s *NDCIntegrationTestSuite) TestAdminGetWorkflowExecutionRawHistoryV2() { - - workflowID := "ndc-re-send-test" + uuid.New() - runID := uuid.New() - workflowType := "ndc-re-send-workflow-type" - tasklist := "event-generator-taskList" - identity := "ndc-re-send-test" - - historyClient := s.active.GetHistoryClient() - adminClient := s.active.GetAdminClient() - getHistory := func( - domain string, - workflowID string, - runID string, - startEventID *int64, - startEventVersion *int64, - endEventID *int64, - endEventVersion *int64, - pageSize int, - token []byte, - ) (*types.GetWorkflowExecutionRawHistoryV2Response, error) { - - execution := &types.WorkflowExecution{ - WorkflowID: workflowID, - RunID: runID, - } - return adminClient.GetWorkflowExecutionRawHistoryV2(s.createContext(), &types.GetWorkflowExecutionRawHistoryV2Request{ - Domain: domain, - Execution: execution, - StartEventID: startEventID, - StartEventVersion: startEventVersion, - EndEventID: endEventID, - EndEventVersion: endEventVersion, - MaximumPageSize: int32(pageSize), - NextPageToken: token, - }) - } - - eventsBatch1 := []*types.History{ - {Events: []*types.HistoryEvent{ - { - EventID: 1, - Version: 21, - EventType: types.EventTypeWorkflowExecutionStarted.Ptr(), - WorkflowExecutionStartedEventAttributes: &types.WorkflowExecutionStartedEventAttributes{ - WorkflowType: &types.WorkflowType{Name: workflowType}, - TaskList: &types.TaskList{Name: tasklist}, - Input: nil, - ExecutionStartToCloseTimeoutSeconds: common.Int32Ptr(1000), - TaskStartToCloseTimeoutSeconds: common.Int32Ptr(1000), - FirstDecisionTaskBackoffSeconds: common.Int32Ptr(100), - }, - }, - { - EventID: 2, - Version: 21, - EventType: types.EventTypeDecisionTaskScheduled.Ptr(), - DecisionTaskScheduledEventAttributes: &types.DecisionTaskScheduledEventAttributes{ - TaskList: &types.TaskList{Name: tasklist}, - StartToCloseTimeoutSeconds: common.Int32Ptr(1000), - Attempt: 0, - }, - }, - }}, - {Events: []*types.HistoryEvent{ - { - EventID: 3, - Version: 21, - EventType: types.EventTypeDecisionTaskStarted.Ptr(), - DecisionTaskStartedEventAttributes: &types.DecisionTaskStartedEventAttributes{ - ScheduledEventID: 2, - Identity: identity, - RequestID: uuid.New(), - }, - }, - }}, - {Events: []*types.HistoryEvent{ - { - EventID: 4, - Version: 21, - EventType: types.EventTypeDecisionTaskCompleted.Ptr(), - DecisionTaskCompletedEventAttributes: &types.DecisionTaskCompletedEventAttributes{ - ScheduledEventID: 2, - StartedEventID: 3, - Identity: identity, - }, - }, - { - EventID: 5, - Version: 21, - EventType: types.EventTypeMarkerRecorded.Ptr(), - MarkerRecordedEventAttributes: &types.MarkerRecordedEventAttributes{ - MarkerName: "some marker name", - Details: []byte("some marker details"), - DecisionTaskCompletedEventID: 4, - }, - }, - { - EventID: 6, - Version: 21, - EventType: types.EventTypeActivityTaskScheduled.Ptr(), - ActivityTaskScheduledEventAttributes: &types.ActivityTaskScheduledEventAttributes{ - DecisionTaskCompletedEventID: 4, - ActivityID: "0", - ActivityType: &types.ActivityType{Name: "activity-type"}, - TaskList: &types.TaskList{Name: tasklist}, - Input: nil, - ScheduleToCloseTimeoutSeconds: common.Int32Ptr(20), - ScheduleToStartTimeoutSeconds: common.Int32Ptr(20), - StartToCloseTimeoutSeconds: common.Int32Ptr(20), - HeartbeatTimeoutSeconds: common.Int32Ptr(20), - }, - }, - }}, - {Events: []*types.HistoryEvent{ - { - EventID: 7, - Version: 21, - EventType: types.EventTypeActivityTaskStarted.Ptr(), - ActivityTaskStartedEventAttributes: &types.ActivityTaskStartedEventAttributes{ - ScheduledEventID: 6, - Identity: identity, - RequestID: uuid.New(), - Attempt: 0, - }, - }, - }}, - {Events: []*types.HistoryEvent{ - { - EventID: 8, - Version: 21, - EventType: types.EventTypeWorkflowExecutionSignaled.Ptr(), - WorkflowExecutionSignaledEventAttributes: &types.WorkflowExecutionSignaledEventAttributes{ - SignalName: "some signal name 1", - Input: []byte("some signal details 1"), - Identity: identity, - }, - }, - { - EventID: 9, - Version: 21, - EventType: types.EventTypeDecisionTaskScheduled.Ptr(), - DecisionTaskScheduledEventAttributes: &types.DecisionTaskScheduledEventAttributes{ - TaskList: &types.TaskList{Name: tasklist}, - StartToCloseTimeoutSeconds: common.Int32Ptr(1000), - Attempt: 0, - }, - }, - }}, - {Events: []*types.HistoryEvent{ - { - EventID: 10, - Version: 21, - EventType: types.EventTypeDecisionTaskStarted.Ptr(), - DecisionTaskStartedEventAttributes: &types.DecisionTaskStartedEventAttributes{ - ScheduledEventID: 9, - Identity: identity, - RequestID: uuid.New(), - }, - }, - }}, - {Events: []*types.HistoryEvent{ - { - EventID: 11, - Version: 21, - EventType: types.EventTypeDecisionTaskCompleted.Ptr(), - DecisionTaskCompletedEventAttributes: &types.DecisionTaskCompletedEventAttributes{ - ScheduledEventID: 9, - StartedEventID: 10, - Identity: identity, - }, - }, - { - EventID: 12, - Version: 21, - EventType: types.EventTypeWorkflowExecutionSignaled.Ptr(), - WorkflowExecutionSignaledEventAttributes: &types.WorkflowExecutionSignaledEventAttributes{ - SignalName: "some signal name 2", - Input: []byte("some signal details 2"), - Identity: identity, - }, - }, - { - EventID: 13, - Version: 21, - EventType: types.EventTypeDecisionTaskScheduled.Ptr(), - DecisionTaskScheduledEventAttributes: &types.DecisionTaskScheduledEventAttributes{ - TaskList: &types.TaskList{Name: tasklist}, - StartToCloseTimeoutSeconds: common.Int32Ptr(1000), - Attempt: 0, - }, - }, - { - EventID: 14, - Version: 21, - EventType: types.EventTypeDecisionTaskStarted.Ptr(), - DecisionTaskStartedEventAttributes: &types.DecisionTaskStartedEventAttributes{ - ScheduledEventID: 13, - Identity: identity, - RequestID: uuid.New(), - }, - }, - }}, - } - - eventsBatch2 := []*types.History{ - {Events: []*types.HistoryEvent{ - { - EventID: 15, - Version: 31, - EventType: types.EventTypeDecisionTaskCompleted.Ptr(), - DecisionTaskCompletedEventAttributes: &types.DecisionTaskCompletedEventAttributes{ - ScheduledEventID: 9, - StartedEventID: 10, - Identity: identity, - }, - }, - { - EventID: 16, - Version: 31, - EventType: types.EventTypeActivityTaskScheduled.Ptr(), - ActivityTaskScheduledEventAttributes: &types.ActivityTaskScheduledEventAttributes{ - DecisionTaskCompletedEventID: 4, - ActivityID: "0", - ActivityType: &types.ActivityType{Name: "activity-type"}, - TaskList: &types.TaskList{Name: tasklist}, - Input: nil, - ScheduleToCloseTimeoutSeconds: common.Int32Ptr(20), - ScheduleToStartTimeoutSeconds: common.Int32Ptr(20), - StartToCloseTimeoutSeconds: common.Int32Ptr(20), - HeartbeatTimeoutSeconds: common.Int32Ptr(20), - }, - }, - }}, - } - - eventsBatch3 := []*types.History{ - {Events: []*types.HistoryEvent{ - { - EventID: 15, - Version: 30, - EventType: types.EventTypeDecisionTaskTimedOut.Ptr(), - DecisionTaskTimedOutEventAttributes: &types.DecisionTaskTimedOutEventAttributes{ - ScheduledEventID: 13, - StartedEventID: 14, - TimeoutType: types.TimeoutTypeStartToClose.Ptr(), - }, - }, - { - EventID: 16, - Version: 30, - EventType: types.EventTypeActivityTaskTimedOut.Ptr(), - ActivityTaskTimedOutEventAttributes: &types.ActivityTaskTimedOutEventAttributes{ - ScheduledEventID: 6, - StartedEventID: 7, - TimeoutType: types.TimeoutTypeStartToClose.Ptr(), - }, - }, - { - EventID: 17, - Version: 30, - EventType: types.EventTypeDecisionTaskScheduled.Ptr(), - DecisionTaskScheduledEventAttributes: &types.DecisionTaskScheduledEventAttributes{ - TaskList: &types.TaskList{Name: tasklist}, - StartToCloseTimeoutSeconds: common.Int32Ptr(1000), - Attempt: 0, - }, - }, - }}, - {Events: []*types.HistoryEvent{ - { - EventID: 18, - Version: 30, - EventType: types.EventTypeDecisionTaskStarted.Ptr(), - DecisionTaskStartedEventAttributes: &types.DecisionTaskStartedEventAttributes{ - ScheduledEventID: 17, - Identity: identity, - RequestID: uuid.New(), - }, - }, - }}, - {Events: []*types.HistoryEvent{ - { - EventID: 19, - Version: 30, - EventType: types.EventTypeDecisionTaskCompleted.Ptr(), - DecisionTaskCompletedEventAttributes: &types.DecisionTaskCompletedEventAttributes{ - ScheduledEventID: 8, - StartedEventID: 9, - Identity: identity, - }, - }, - { - EventID: 20, - Version: 30, - EventType: types.EventTypeWorkflowExecutionFailed.Ptr(), - WorkflowExecutionFailedEventAttributes: &types.WorkflowExecutionFailedEventAttributes{ - DecisionTaskCompletedEventID: 19, - Reason: common.StringPtr("some random reason"), - Details: nil, - }, - }, - }}, - } - - eventsBatch4 := []*types.History{ - {Events: []*types.HistoryEvent{ - { - EventID: 17, - Version: 32, - EventType: types.EventTypeWorkflowExecutionTimedOut.Ptr(), - WorkflowExecutionTimedOutEventAttributes: &types.WorkflowExecutionTimedOutEventAttributes{ - TimeoutType: types.TimeoutTypeStartToClose.Ptr(), - }, - }, - }}, - } - - versionHistory1 := s.eventBatchesToVersionHistory(nil, eventsBatch1) - - versionHistory2, err := versionHistory1.DuplicateUntilLCAItem( - persistence.NewVersionHistoryItem(14, 21), - ) - s.NoError(err) - versionHistory2 = s.eventBatchesToVersionHistory(versionHistory2, eventsBatch2) - - versionHistory3, err := versionHistory1.DuplicateUntilLCAItem( - persistence.NewVersionHistoryItem(14, 21), - ) - s.NoError(err) - versionHistory3 = s.eventBatchesToVersionHistory(versionHistory3, eventsBatch3) - - versionHistory4, err := versionHistory2.DuplicateUntilLCAItem( - persistence.NewVersionHistoryItem(16, 31), - ) - s.NoError(err) - versionHistory4 = s.eventBatchesToVersionHistory(versionHistory4, eventsBatch4) - - s.applyEvents( - workflowID, - runID, - workflowType, - tasklist, - versionHistory1, - eventsBatch1, - historyClient, - ) - s.applyEvents( - workflowID, - runID, - workflowType, - tasklist, - versionHistory3, - eventsBatch3, - historyClient, - ) - s.applyEvents( - workflowID, - runID, - workflowType, - tasklist, - versionHistory2, - eventsBatch2, - historyClient, - ) - s.applyEvents( - workflowID, - runID, - workflowType, - tasklist, - versionHistory4, - eventsBatch4, - historyClient, - ) - - // GetWorkflowExecutionRawHistoryV2 start and end - var token []byte - batchCount := 0 - for continuePaging := true; continuePaging; continuePaging = len(token) != 0 { - resp, err := getHistory( - s.domainName, - workflowID, - runID, - common.Int64Ptr(14), - common.Int64Ptr(21), - common.Int64Ptr(20), - common.Int64Ptr(30), - 1, - token, - ) - s.Nil(err) - s.True(len(resp.HistoryBatches) <= 1) - batchCount++ - token = resp.NextPageToken - } - s.Equal(batchCount, 4) - - // GetWorkflowExecutionRawHistoryV2 start and end not on the same branch - token = nil - batchCount = 0 - for continuePaging := true; continuePaging; continuePaging = len(token) != 0 { - resp, err := getHistory( - s.domainName, - workflowID, - runID, - common.Int64Ptr(17), - common.Int64Ptr(30), - common.Int64Ptr(17), - common.Int64Ptr(32), - 1, - token, - ) - s.Nil(err) - s.True(len(resp.HistoryBatches) <= 1) - batchCount++ - token = resp.NextPageToken - } - s.Equal(batchCount, 2) - - // GetWorkflowExecutionRawHistoryV2 start boundary - token = nil - batchCount = 0 - for continuePaging := true; continuePaging; continuePaging = len(token) != 0 { - resp, err := getHistory( - s.domainName, - workflowID, - runID, - common.Int64Ptr(14), - common.Int64Ptr(21), - nil, - nil, - 1, - token, - ) - s.Nil(err) - s.True(len(resp.HistoryBatches) <= 1) - batchCount++ - token = resp.NextPageToken - } - s.Equal(batchCount, 3) - - // GetWorkflowExecutionRawHistoryV2 end boundary - token = nil - batchCount = 0 - for continuePaging := true; continuePaging; continuePaging = len(token) != 0 { - resp, err := getHistory( - s.domainName, - workflowID, - runID, - nil, - nil, - common.Int64Ptr(17), - common.Int64Ptr(32), - 1, - token, - ) - s.Nil(err) - s.True(len(resp.HistoryBatches) <= 1) - batchCount++ - token = resp.NextPageToken - } - s.Equal(batchCount, 10) -} - -func (s *NDCIntegrationTestSuite) registerDomain() { - s.domainName = "test-simple-workflow-ndc-" + common.GenerateRandomString(5) - client1 := s.active.GetFrontendClient() // active - err := client1.RegisterDomain(s.createContext(), &types.RegisterDomainRequest{ - Name: s.domainName, - IsGlobalDomain: true, - Clusters: clusterReplicationConfig, - // make the active cluster `standby` and replicate to `active` cluster - ActiveClusterName: clusterName[1], - WorkflowExecutionRetentionPeriodInDays: 1, - }) - s.Require().NoError(err) - - descReq := &types.DescribeDomainRequest{ - Name: common.StringPtr(s.domainName), - } - resp, err := client1.DescribeDomain(s.createContext(), descReq) - s.Require().NoError(err) - s.Require().NotNil(resp) - s.domainID = resp.GetDomainInfo().GetUUID() - // Wait for domain cache to pick the change - time.Sleep(2 * cache.DomainCacheRefreshInterval) - - s.logger.Info(fmt.Sprintf("Domain name: %v - ID: %v", s.domainName, s.domainID)) -} - -func (s *NDCIntegrationTestSuite) generateNewRunHistory( - event *types.HistoryEvent, - domain string, - workflowID string, - runID string, - version int64, - workflowType string, - taskList string, -) *persistence.DataBlob { - - // TODO temporary code to generate first event & version history - // we should generate these as part of modeled based testing - - if event.GetWorkflowExecutionContinuedAsNewEventAttributes() == nil { - return nil - } - - event.WorkflowExecutionContinuedAsNewEventAttributes.NewExecutionRunID = uuid.New() - - newRunFirstEvent := &types.HistoryEvent{ - EventID: common.FirstEventID, - Timestamp: common.Int64Ptr(time.Now().UnixNano()), - EventType: types.EventTypeWorkflowExecutionStarted.Ptr(), - Version: version, - TaskID: 1, - WorkflowExecutionStartedEventAttributes: &types.WorkflowExecutionStartedEventAttributes{ - WorkflowType: &types.WorkflowType{Name: workflowType}, - ParentWorkflowDomain: common.StringPtr(domain), - ParentWorkflowExecution: &types.WorkflowExecution{ - WorkflowID: uuid.New(), - RunID: uuid.New(), - }, - ParentInitiatedEventID: common.Int64Ptr(event.GetEventID()), - TaskList: &types.TaskList{ - Name: taskList, - Kind: types.TaskListKindNormal.Ptr(), - }, - ExecutionStartToCloseTimeoutSeconds: common.Int32Ptr(10), - TaskStartToCloseTimeoutSeconds: common.Int32Ptr(10), - ContinuedExecutionRunID: runID, - Initiator: types.ContinueAsNewInitiatorCronSchedule.Ptr(), - OriginalExecutionRunID: runID, - Identity: "NDC-test", - FirstExecutionRunID: runID, - Attempt: 0, - ExpirationTimestamp: common.Int64Ptr(time.Now().Add(time.Minute).UnixNano()), - }, - } - - eventBlob, err := s.serializer.SerializeBatchEvents([]*types.HistoryEvent{newRunFirstEvent}, common.EncodingTypeThriftRW) - s.NoError(err) - - return eventBlob -} - -func (s *NDCIntegrationTestSuite) toInternalDataBlob( - blob *persistence.DataBlob, -) *types.DataBlob { - - if blob == nil { - return nil - } - - var encodingType types.EncodingType - switch blob.GetEncoding() { - case common.EncodingTypeThriftRW: - encodingType = types.EncodingTypeThriftRW - case common.EncodingTypeJSON, - common.EncodingTypeGob, - common.EncodingTypeUnknown, - common.EncodingTypeEmpty: - panic(fmt.Sprintf("unsupported encoding type: %v", blob.GetEncoding())) - default: - panic(fmt.Sprintf("unknown encoding type: %v", blob.GetEncoding())) - } - - return &types.DataBlob{ - EncodingType: encodingType.Ptr(), - Data: blob.Data, - } -} - -func (s *NDCIntegrationTestSuite) generateEventBlobs( - workflowID string, - runID string, - workflowType string, - tasklist string, - batch *types.History, -) (*persistence.DataBlob, *persistence.DataBlob) { - // TODO temporary code to generate next run first event - // we should generate these as part of modeled based testing - lastEvent := batch.Events[len(batch.Events)-1] - newRunEventBlob := s.generateNewRunHistory( - lastEvent, s.domainName, workflowID, runID, lastEvent.GetVersion(), workflowType, tasklist, - ) - // must serialize events batch after attempt on continue as new as generateNewRunHistory will - // modify the NewExecutionRunID attr - eventBlob, err := s.serializer.SerializeBatchEvents(batch.Events, common.EncodingTypeThriftRW) - s.NoError(err) - return eventBlob, newRunEventBlob -} - -func (s *NDCIntegrationTestSuite) applyEvents( - workflowID string, - runID string, - workflowType string, - tasklist string, - versionHistory *persistence.VersionHistory, - eventBatches []*types.History, - historyClient host.HistoryClient, -) { - for _, batch := range eventBatches { - eventBlob, newRunEventBlob := s.generateEventBlobs(workflowID, runID, workflowType, tasklist, batch) - req := &types.ReplicateEventsV2Request{ - DomainUUID: s.domainID, - WorkflowExecution: &types.WorkflowExecution{ - WorkflowID: workflowID, - RunID: runID, - }, - VersionHistoryItems: s.toInternalVersionHistoryItems(versionHistory), - Events: s.toInternalDataBlob(eventBlob), - NewRunEvents: s.toInternalDataBlob(newRunEventBlob), - } - - err := historyClient.ReplicateEventsV2(s.createContext(), req) - s.Nil(err, "Failed to replicate history event") - err = historyClient.ReplicateEventsV2(s.createContext(), req) - s.Nil(err, "Failed to dedup replicate history event") - } -} - -func (s *NDCIntegrationTestSuite) applyEventsThroughFetcher( - workflowID string, - runID string, - workflowType string, - tasklist string, - versionHistory *persistence.VersionHistory, - eventBatches []*types.History, -) { - for _, batch := range eventBatches { - eventBlob, newRunEventBlob := s.generateEventBlobs(workflowID, runID, workflowType, tasklist, batch) - - taskType := types.ReplicationTaskTypeHistoryV2 - replicationTask := &types.ReplicationTask{ - TaskType: &taskType, - SourceTaskID: 1, - HistoryTaskV2Attributes: &types.HistoryTaskV2Attributes{ - TaskID: 1, - DomainID: s.domainID, - WorkflowID: workflowID, - RunID: runID, - VersionHistoryItems: s.toInternalVersionHistoryItems(versionHistory), - Events: s.toInternalDataBlob(eventBlob), - NewRunEvents: s.toInternalDataBlob(newRunEventBlob), - }, - } - - s.standByReplicationTasksChan <- replicationTask - // this is to test whether dedup works - s.standByReplicationTasksChan <- replicationTask - } -} - -func (s *NDCIntegrationTestSuite) eventBatchesToVersionHistory( - versionHistory *persistence.VersionHistory, - eventBatches []*types.History, -) *persistence.VersionHistory { - - // TODO temporary code to generate version history - // we should generate version as part of modeled based testing - if versionHistory == nil { - versionHistory = persistence.NewVersionHistory(nil, nil) - } - for _, batch := range eventBatches { - for _, event := range batch.Events { - err := versionHistory.AddOrUpdateItem( - persistence.NewVersionHistoryItem( - event.GetEventID(), - event.GetVersion(), - )) - s.NoError(err) - } - } - - return versionHistory -} - -func (s *NDCIntegrationTestSuite) toInternalVersionHistoryItems( - versionHistory *persistence.VersionHistory, -) []*types.VersionHistoryItem { - if versionHistory == nil { - return nil - } - - return versionHistory.ToInternalType().Items -} - -func (s *NDCIntegrationTestSuite) createContext() context.Context { - ctx, _ := context.WithTimeout(context.Background(), 90*time.Second) - return ctx -} - -func (s *NDCIntegrationTestSuite) setupRemoteFrontendClients() { - s.mockAdminClient["standby"].(*adminClient.MockClient).EXPECT().ReapplyEvents(gomock.Any(), gomock.Any()).Return(nil).AnyTimes() - s.mockAdminClient["other"].(*adminClient.MockClient).EXPECT().ReapplyEvents(gomock.Any(), gomock.Any()).Return(nil).AnyTimes() -} diff --git a/host/ndc/replication_integration_test.go b/host/ndc/replicationIntegrationTest.go similarity index 100% rename from host/ndc/replication_integration_test.go rename to host/ndc/replicationIntegrationTest.go diff --git a/host/queryworkflow_test.go b/host/queryworkflowTest.go similarity index 100% rename from host/queryworkflow_test.go rename to host/queryworkflowTest.go diff --git a/host/resetworkflow_test.go b/host/resetworkflowTest.go similarity index 100% rename from host/resetworkflow_test.go rename to host/resetworkflowTest.go diff --git a/host/signalworkflow_test.go b/host/signalworkflowTest.go similarity index 100% rename from host/signalworkflow_test.go rename to host/signalworkflowTest.go diff --git a/host/sizelimitTest.go b/host/sizelimitTest.go new file mode 100644 index 00000000000..7a46a1dfa8d --- /dev/null +++ b/host/sizelimitTest.go @@ -0,0 +1,191 @@ +// Copyright (c) 2016 Uber Technologies, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package host + +import ( + "bytes" + "encoding/binary" + "strconv" + "testing" + "time" + + "github.com/pborman/uuid" + "github.com/stretchr/testify/require" + + "github.com/uber/cadence/common" + "github.com/uber/cadence/common/log/tag" + "github.com/uber/cadence/common/types" +) + +type SizeLimitIntegrationSuite struct { + // override suite.Suite.Assertions with require.Assertions; this means that s.NotNil(nil) will stop the test, + // not merely log an error + *require.Assertions + IntegrationBase +} + +// This cluster use customized threshold for history config +func (s *SizeLimitIntegrationSuite) SetupSuite() { + s.setupSuite() +} + +func (s *SizeLimitIntegrationSuite) TearDownSuite() { + s.tearDownSuite() +} + +func (s *SizeLimitIntegrationSuite) SetupTest() { + // Have to define our overridden assertions in the test setup. If we did it earlier, s.T() will return nil + s.Assertions = require.New(s.T()) +} + +func (s *SizeLimitIntegrationSuite) TestTerminateWorkflowCausedBySizeLimit() { + id := "integration-terminate-workflow-by-size-limit-test" + wt := "integration-terminate-workflow-by-size-limit-test-type" + tl := "integration-terminate-workflow-by-size-limit-test-tasklist" + identity := "worker1" + activityName := "activity_type1" + + workflowType := &types.WorkflowType{} + workflowType.Name = wt + + taskList := &types.TaskList{} + taskList.Name = tl + + request := &types.StartWorkflowExecutionRequest{ + RequestID: uuid.New(), + Domain: s.domainName, + WorkflowID: id, + WorkflowType: workflowType, + TaskList: taskList, + Input: nil, + ExecutionStartToCloseTimeoutSeconds: common.Int32Ptr(100), + TaskStartToCloseTimeoutSeconds: common.Int32Ptr(1), + Identity: identity, + } + + we, err0 := s.engine.StartWorkflowExecution(createContext(), request) + s.Nil(err0) + + s.Logger.Info("StartWorkflowExecution", tag.WorkflowRunID(we.RunID)) + + activityCount := int32(4) + activityCounter := int32(0) + dtHandler := func(execution *types.WorkflowExecution, wt *types.WorkflowType, + previousStartedEventID, startedEventID int64, history *types.History) ([]byte, []*types.Decision, error) { + if activityCounter < activityCount { + activityCounter++ + buf := new(bytes.Buffer) + s.Nil(binary.Write(buf, binary.LittleEndian, activityCounter)) + + return []byte(strconv.Itoa(int(activityCounter))), []*types.Decision{{ + DecisionType: types.DecisionTypeScheduleActivityTask.Ptr(), + ScheduleActivityTaskDecisionAttributes: &types.ScheduleActivityTaskDecisionAttributes{ + ActivityID: strconv.Itoa(int(activityCounter)), + ActivityType: &types.ActivityType{Name: activityName}, + TaskList: &types.TaskList{Name: tl}, + Input: buf.Bytes(), + ScheduleToCloseTimeoutSeconds: common.Int32Ptr(100), + ScheduleToStartTimeoutSeconds: common.Int32Ptr(10), + StartToCloseTimeoutSeconds: common.Int32Ptr(50), + HeartbeatTimeoutSeconds: common.Int32Ptr(5), + }, + }}, nil + } + + return []byte(strconv.Itoa(int(activityCounter))), []*types.Decision{{ + DecisionType: types.DecisionTypeCompleteWorkflowExecution.Ptr(), + CompleteWorkflowExecutionDecisionAttributes: &types.CompleteWorkflowExecutionDecisionAttributes{ + Result: []byte("Done."), + }, + }}, nil + } + + atHandler := func(execution *types.WorkflowExecution, activityType *types.ActivityType, + activityID string, input []byte, taskToken []byte) ([]byte, bool, error) { + + return []byte("Activity Result."), false, nil + } + + poller := &TaskPoller{ + Engine: s.engine, + Domain: s.domainName, + TaskList: taskList, + Identity: identity, + DecisionHandler: dtHandler, + ActivityHandler: atHandler, + Logger: s.Logger, + T: s.T(), + } + + for i := int32(0); i < activityCount-1; i++ { + _, err := poller.PollAndProcessDecisionTask(false, false) + s.Logger.Info("PollAndProcessDecisionTask", tag.Error(err)) + s.Nil(err) + + err = poller.PollAndProcessActivityTask(false) + s.Logger.Info("PollAndProcessActivityTask", tag.Error(err)) + s.Nil(err) + } + + // process this decision will trigger history exceed limit error + _, err := poller.PollAndProcessDecisionTask(false, false) + s.Logger.Info("PollAndProcessDecisionTask", tag.Error(err)) + s.Nil(err) + + // verify last event is terminated event + historyResponse, err := s.engine.GetWorkflowExecutionHistory(createContext(), &types.GetWorkflowExecutionHistoryRequest{ + Domain: s.domainName, + Execution: &types.WorkflowExecution{ + WorkflowID: id, + RunID: we.GetRunID(), + }, + }) + s.Nil(err) + history := historyResponse.History + lastEvent := history.Events[len(history.Events)-1] + s.Equal(types.EventTypeWorkflowExecutionFailed, lastEvent.GetEventType()) + failedEventAttributes := lastEvent.WorkflowExecutionFailedEventAttributes + s.Equal(common.FailureReasonSizeExceedsLimit, failedEventAttributes.GetReason()) + + // verify visibility is correctly processed from open to close + isCloseCorrect := false + for i := 0; i < 10; i++ { + resp, err1 := s.engine.ListClosedWorkflowExecutions(createContext(), &types.ListClosedWorkflowExecutionsRequest{ + Domain: s.domainName, + MaximumPageSize: 100, + StartTimeFilter: &types.StartTimeFilter{ + EarliestTime: common.Int64Ptr(0), + LatestTime: common.Int64Ptr(time.Now().UnixNano()), + }, + ExecutionFilter: &types.WorkflowExecutionFilter{ + WorkflowID: id, + }, + }) + s.Nil(err1) + if len(resp.Executions) == 1 { + isCloseCorrect = true + break + } + s.Logger.Info("Closed WorkflowExecution is not yet visible") + time.Sleep(100 * time.Millisecond) + } + s.True(isCloseCorrect) +} diff --git a/host/sizelimit_test.go b/host/sizelimit_test.go index 3a3d2472a85..71e2b13c7c9 100644 --- a/host/sizelimit_test.go +++ b/host/sizelimit_test.go @@ -21,43 +21,12 @@ package host import ( - "bytes" - "encoding/binary" "flag" - "strconv" "testing" - "time" - "github.com/pborman/uuid" - "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" - - "github.com/uber/cadence/common" - "github.com/uber/cadence/common/log/tag" - "github.com/uber/cadence/common/types" ) -type SizeLimitIntegrationSuite struct { - // override suite.Suite.Assertions with require.Assertions; this means that s.NotNil(nil) will stop the test, - // not merely log an error - *require.Assertions - IntegrationBase -} - -// This cluster use customized threshold for history config -func (s *SizeLimitIntegrationSuite) SetupSuite() { - s.setupSuite() -} - -func (s *SizeLimitIntegrationSuite) TearDownSuite() { - s.tearDownSuite() -} - -func (s *SizeLimitIntegrationSuite) SetupTest() { - // Have to define our overridden assertions in the test setup. If we did it earlier, s.T() will return nil - s.Assertions = require.New(s.T()) -} - func TestSizeLimitIntegrationSuite(t *testing.T) { flag.Parse() @@ -76,137 +45,3 @@ func TestSizeLimitIntegrationSuite(t *testing.T) { s.IntegrationBase = NewIntegrationBase(params) suite.Run(t, s) } - -func (s *SizeLimitIntegrationSuite) TestTerminateWorkflowCausedBySizeLimit() { - id := "integration-terminate-workflow-by-size-limit-test" - wt := "integration-terminate-workflow-by-size-limit-test-type" - tl := "integration-terminate-workflow-by-size-limit-test-tasklist" - identity := "worker1" - activityName := "activity_type1" - - workflowType := &types.WorkflowType{} - workflowType.Name = wt - - taskList := &types.TaskList{} - taskList.Name = tl - - request := &types.StartWorkflowExecutionRequest{ - RequestID: uuid.New(), - Domain: s.domainName, - WorkflowID: id, - WorkflowType: workflowType, - TaskList: taskList, - Input: nil, - ExecutionStartToCloseTimeoutSeconds: common.Int32Ptr(100), - TaskStartToCloseTimeoutSeconds: common.Int32Ptr(1), - Identity: identity, - } - - we, err0 := s.engine.StartWorkflowExecution(createContext(), request) - s.Nil(err0) - - s.Logger.Info("StartWorkflowExecution", tag.WorkflowRunID(we.RunID)) - - activityCount := int32(4) - activityCounter := int32(0) - dtHandler := func(execution *types.WorkflowExecution, wt *types.WorkflowType, - previousStartedEventID, startedEventID int64, history *types.History) ([]byte, []*types.Decision, error) { - if activityCounter < activityCount { - activityCounter++ - buf := new(bytes.Buffer) - s.Nil(binary.Write(buf, binary.LittleEndian, activityCounter)) - - return []byte(strconv.Itoa(int(activityCounter))), []*types.Decision{{ - DecisionType: types.DecisionTypeScheduleActivityTask.Ptr(), - ScheduleActivityTaskDecisionAttributes: &types.ScheduleActivityTaskDecisionAttributes{ - ActivityID: strconv.Itoa(int(activityCounter)), - ActivityType: &types.ActivityType{Name: activityName}, - TaskList: &types.TaskList{Name: tl}, - Input: buf.Bytes(), - ScheduleToCloseTimeoutSeconds: common.Int32Ptr(100), - ScheduleToStartTimeoutSeconds: common.Int32Ptr(10), - StartToCloseTimeoutSeconds: common.Int32Ptr(50), - HeartbeatTimeoutSeconds: common.Int32Ptr(5), - }, - }}, nil - } - - return []byte(strconv.Itoa(int(activityCounter))), []*types.Decision{{ - DecisionType: types.DecisionTypeCompleteWorkflowExecution.Ptr(), - CompleteWorkflowExecutionDecisionAttributes: &types.CompleteWorkflowExecutionDecisionAttributes{ - Result: []byte("Done."), - }, - }}, nil - } - - atHandler := func(execution *types.WorkflowExecution, activityType *types.ActivityType, - activityID string, input []byte, taskToken []byte) ([]byte, bool, error) { - - return []byte("Activity Result."), false, nil - } - - poller := &TaskPoller{ - Engine: s.engine, - Domain: s.domainName, - TaskList: taskList, - Identity: identity, - DecisionHandler: dtHandler, - ActivityHandler: atHandler, - Logger: s.Logger, - T: s.T(), - } - - for i := int32(0); i < activityCount-1; i++ { - _, err := poller.PollAndProcessDecisionTask(false, false) - s.Logger.Info("PollAndProcessDecisionTask", tag.Error(err)) - s.Nil(err) - - err = poller.PollAndProcessActivityTask(false) - s.Logger.Info("PollAndProcessActivityTask", tag.Error(err)) - s.Nil(err) - } - - // process this decision will trigger history exceed limit error - _, err := poller.PollAndProcessDecisionTask(false, false) - s.Logger.Info("PollAndProcessDecisionTask", tag.Error(err)) - s.Nil(err) - - // verify last event is terminated event - historyResponse, err := s.engine.GetWorkflowExecutionHistory(createContext(), &types.GetWorkflowExecutionHistoryRequest{ - Domain: s.domainName, - Execution: &types.WorkflowExecution{ - WorkflowID: id, - RunID: we.GetRunID(), - }, - }) - s.Nil(err) - history := historyResponse.History - lastEvent := history.Events[len(history.Events)-1] - s.Equal(types.EventTypeWorkflowExecutionFailed, lastEvent.GetEventType()) - failedEventAttributes := lastEvent.WorkflowExecutionFailedEventAttributes - s.Equal(common.FailureReasonSizeExceedsLimit, failedEventAttributes.GetReason()) - - // verify visibility is correctly processed from open to close - isCloseCorrect := false - for i := 0; i < 10; i++ { - resp, err1 := s.engine.ListClosedWorkflowExecutions(createContext(), &types.ListClosedWorkflowExecutionsRequest{ - Domain: s.domainName, - MaximumPageSize: 100, - StartTimeFilter: &types.StartTimeFilter{ - EarliestTime: common.Int64Ptr(0), - LatestTime: common.Int64Ptr(time.Now().UnixNano()), - }, - ExecutionFilter: &types.WorkflowExecutionFilter{ - WorkflowID: id, - }, - }) - s.Nil(err1) - if len(resp.Executions) == 1 { - isCloseCorrect = true - break - } - s.Logger.Info("Closed WorkflowExecution is not yet visible") - time.Sleep(100 * time.Millisecond) - } - s.True(isCloseCorrect) -}