Skip to content

Commit

Permalink
Merge pull request #6730 from planetscale/rn-ci-test2
Browse files Browse the repository at this point in the history 
VReplication e2e tests: Simplify and split tests to overcome current CI resource constraints
  • Loading branch information
@deepthi
deepthi authored Sep 17, 2020
2 parents 8afa9ae + ece08d9 commit d58a0fe
Show file tree
Hide file tree
Showing 2 changed files with 56 additions and 27 deletions.
56 changes: 29 additions & 27 deletions go/test/endtoend/vreplication/vreplication_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,8 @@ func TestBasicVreplicationWorkflow(t *testing.T) {
allCellNames = "zone1"
vc = InitCluster(t, []string{defaultCellName})
assert.NotNil(t, vc)
defaultReplicas = 0 // because of CI resource constraints we can only run this test with master tablets
defer func() { defaultReplicas = 1 }()

defer vc.TearDown()

Expand Down Expand Up @@ -113,12 +115,13 @@ func TestMultiCellVreplicationWorkflow(t *testing.T) {
insertInitialData(t)
shardCustomer(t, true, []*Cell{cell1, cell2}, cell2.Name)

insertMoreCustomers(t, 16)
reshardCustomer2to4Split(t, []*Cell{cell1, cell2}, cell2.Name)
validateCount(t, vtgateConn, "customer:-40", "customer", 5)
validateCount(t, vtgateConn, "customer:40-80", "customer", 5)
validateCount(t, vtgateConn, "customer:80-c0", "customer", 6)
validateCount(t, vtgateConn, "customer:c0-", "customer", 5)
// commenting lines below because of resource constraints
//insertMoreCustomers(t, 16)
//reshardCustomer2to4Split(t, []*Cell{cell1, cell2}, cell2.Name)
//validateCount(t, vtgateConn, "customer:-40", "customer", 5)
//validateCount(t, vtgateConn, "customer:40-80", "customer", 5)
//validateCount(t, vtgateConn, "customer:80-c0", "customer", 6)
//validateCount(t, vtgateConn, "customer:c0-", "customer", 5)
}

func TestCellAliasVreplicationWorkflow(t *testing.T) {
Expand Down Expand Up @@ -151,12 +154,13 @@ func TestCellAliasVreplicationWorkflow(t *testing.T) {
insertInitialData(t)
shardCustomer(t, true, []*Cell{cell1, cell2}, "alias")

insertMoreCustomers(t, 16)
reshardCustomer2to4Split(t, []*Cell{cell1, cell2}, "alias")
validateCount(t, vtgateConn, "customer:-40", "customer", 5)
validateCount(t, vtgateConn, "customer:40-80", "customer", 5)
validateCount(t, vtgateConn, "customer:80-c0", "customer", 6)
validateCount(t, vtgateConn, "customer:c0-", "customer", 5)
// commenting lines below because of resource constraints
//insertMoreCustomers(t, 16)
//reshardCustomer2to4Split(t, []*Cell{cell1, cell2}, "alias")
//validateCount(t, vtgateConn, "customer:-40", "customer", 5)
//validateCount(t, vtgateConn, "customer:40-80", "customer", 5)
//validateCount(t, vtgateConn, "customer:80-c0", "customer", 6)
//validateCount(t, vtgateConn, "customer:c0-", "customer", 5)
}

func insertInitialData(t *testing.T) {
Expand Down Expand Up @@ -199,7 +203,7 @@ func shardCustomer(t *testing.T, testReverse bool, cells []*Cell, sourceCellOrAl
}

if err := vc.VtctlClient.ExecuteCommand("MoveTables", "-cells="+sourceCellOrAlias, "-workflow=p2c",
"-tablet_types="+"replica,rdonly", "product", "customer", "customer"); err != nil {
"-tablet_types="+"master,replica,rdonly", "product", "customer", "customer"); err != nil {
t.Fatalf("MoveTables command failed with %+v\n", err)
}

Expand All @@ -209,16 +213,15 @@ func shardCustomer(t *testing.T, testReverse bool, cells []*Cell, sourceCellOrAl
customerTab1 := vc.Cells[defaultCell.Name].Keyspaces["customer"].Shards["-80"].Tablets["zone1-200"].Vttablet
customerTab2 := vc.Cells[defaultCell.Name].Keyspaces["customer"].Shards["80-"].Tablets["zone1-300"].Vttablet

if vc.WaitForVReplicationToCatchup(customerTab1, "p2c", "vt_customer", 1*time.Second) != nil {
if vc.WaitForVReplicationToCatchup(customerTab1, "p2c", "vt_customer", 5*time.Second) != nil {
t.Fatal("MoveTables timed out for customer.p2c -80")

}
if vc.WaitForVReplicationToCatchup(customerTab2, "p2c", "vt_customer", 1*time.Second) != nil {
if vc.WaitForVReplicationToCatchup(customerTab2, "p2c", "vt_customer", 5*time.Second) != nil {
t.Fatal("MoveTables timed out for customer.p2c 80-")
}

productTab := vc.Cells[defaultCell.Name].Keyspaces["product"].Shards["0"].Tablets["zone1-100"].Vttablet
productTabReplica := vc.Cells[defaultCell.Name].Keyspaces["product"].Shards["0"].Tablets["zone1-101"].Vttablet
query := "select * from customer"
assert.True(t, validateThatQueryExecutesOnTablet(t, vtgateConn, productTab, "product", query, query))
insertQuery1 := "insert into customer(cid, name) values(1001, 'tempCustomer1')"
Expand All @@ -240,7 +243,6 @@ func shardCustomer(t *testing.T, testReverse bool, cells []*Cell, sourceCellOrAl
t.Fatalf("SwitchReads error: %s\n", output)
}

assert.False(t, validateThatQueryExecutesOnTablet(t, vtgateConn, productTabReplica, "customer", query, query))
assert.True(t, validateThatQueryExecutesOnTablet(t, vtgateConn, productTab, "customer", query, query))
want = dryRunResultsSwitchWritesCustomerShard
if output, err = vc.VtctlClient.ExecuteCommandWithOutput("SwitchWrites", "-dry_run", "customer.p2c"); err != nil {
Expand Down Expand Up @@ -499,16 +501,16 @@ func shardOrders(t *testing.T) {
t.Fatal(err)
}
if err := vc.VtctlClient.ExecuteCommand("MoveTables", "-cells="+defaultCell.Name, "-workflow=o2c",
"-tablet_types="+"replica,rdonly", "product", "customer", "orders"); err != nil {
"-tablet_types="+"master,replica,rdonly", "product", "customer", "orders"); err != nil {
t.Fatal(err)
}
customerTab1 := vc.Cells[defaultCell.Name].Keyspaces["customer"].Shards["-80"].Tablets["zone1-200"].Vttablet
customerTab2 := vc.Cells[defaultCell.Name].Keyspaces["customer"].Shards["80-"].Tablets["zone1-300"].Vttablet
if vc.WaitForVReplicationToCatchup(customerTab1, "o2c", "vt_customer", 1*time.Second) != nil {
if vc.WaitForVReplicationToCatchup(customerTab1, "o2c", "vt_customer", 5*time.Second) != nil {
assert.Fail(t, "MoveTables timed out for customer.o2c -80")

}
if vc.WaitForVReplicationToCatchup(customerTab2, "o2c", "vt_customer", 1*time.Second) != nil {
if vc.WaitForVReplicationToCatchup(customerTab2, "o2c", "vt_customer", 5*time.Second) != nil {
assert.Fail(t, "MoveTables timed out for customer.o2c 80-")
}

Expand Down Expand Up @@ -542,17 +544,17 @@ func shardMerchant(t *testing.T) {
t.Fatal(err)
}
if err := vc.VtctlClient.ExecuteCommand("MoveTables", "-cells="+defaultCell.Name, "-workflow=p2m",
"-tablet_types="+"replica,rdonly", "product", "merchant", "merchant"); err != nil {
"-tablet_types="+"master,replica,rdonly", "product", "merchant", "merchant"); err != nil {
t.Fatal(err)
}

merchantTab1 := vc.Cells[defaultCell.Name].Keyspaces["merchant"].Shards["-80"].Tablets["zone1-400"].Vttablet
merchantTab2 := vc.Cells[defaultCell.Name].Keyspaces["merchant"].Shards["80-"].Tablets["zone1-500"].Vttablet
if vc.WaitForVReplicationToCatchup(merchantTab1, "p2m", "vt_merchant", 1*time.Second) != nil {
if vc.WaitForVReplicationToCatchup(merchantTab1, "p2m", "vt_merchant", 5*time.Second) != nil {
t.Fatal("MoveTables timed out for merchant.p2m -80")

}
if vc.WaitForVReplicationToCatchup(merchantTab2, "p2m", "vt_merchant", 1*time.Second) != nil {
if vc.WaitForVReplicationToCatchup(merchantTab2, "p2m", "vt_merchant", 5*time.Second) != nil {
t.Fatal("MoveTables timed out for merchant.p2m 80-")
}

Expand Down Expand Up @@ -606,7 +608,7 @@ func materializeProduct(t *testing.T) {
}
customerTablets := vc.getVttabletsInKeyspace(t, defaultCell, "customer", "master")
for _, tab := range customerTablets {
if vc.WaitForVReplicationToCatchup(tab, workflow, "vt_customer", 3*time.Second) != nil {
if vc.WaitForVReplicationToCatchup(tab, workflow, "vt_customer", 5*time.Second) != nil {
t.Fatal("Materialize timed out")
}
}
Expand All @@ -623,7 +625,7 @@ func materializeSales(t *testing.T) {
t.Fatal(err)
}
productTab := vc.Cells[defaultCell.Name].Keyspaces["product"].Shards["0"].Tablets["zone1-100"].Vttablet
if vc.WaitForVReplicationToCatchup(productTab, "sales", "vt_product", 3*time.Second) != nil {
if vc.WaitForVReplicationToCatchup(productTab, "sales", "vt_product", 5*time.Second) != nil {
assert.Fail(t, "Materialize timed out for product.sales")
}
assert.Empty(t, validateCount(t, vtgateConn, "product", "sales", 2))
Expand All @@ -639,7 +641,7 @@ func materializeMerchantSales(t *testing.T) {
}
merchantTablets := vc.getVttabletsInKeyspace(t, defaultCell, "merchant", "master")
for _, tab := range merchantTablets {
if vc.WaitForVReplicationToCatchup(tab, workflow, "vt_merchant", 1*time.Second) != nil {
if vc.WaitForVReplicationToCatchup(tab, workflow, "vt_merchant", 5*time.Second) != nil {
t.Fatal("Materialize timed out")
}
}
Expand All @@ -660,7 +662,7 @@ func materializeMerchantOrders(t *testing.T) {
}
merchantTablets := vc.getVttabletsInKeyspace(t, defaultCell, "merchant", "master")
for _, tab := range merchantTablets {
if vc.WaitForVReplicationToCatchup(tab, workflow, "vt_merchant", 1*time.Second) != nil {
if vc.WaitForVReplicationToCatchup(tab, workflow, "vt_merchant", 5*time.Second) != nil {
t.Fatal("Materialize timed out")
}
}
Expand Down
27 changes: 27 additions & 0 deletions test/config.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -519,6 +519,33 @@
"Tags": [
"worker_test"
]
},
"vreplication_multicell": {
"File": "unused.go",
"Args": ["vitess.io/vitess/go/test/endtoend/vreplication", "-run", "MultiCell"],
"Command": [],
"Manual": false,
"Shard": 22,
"RetryMax": 0,
"Tags": []
},
"vreplication_cellalias": {
"File": "unused.go",
"Args": ["vitess.io/vitess/go/test/endtoend/vreplication", "-run", "CellAlias"],
"Command": [],
"Manual": false,
"Shard": 23,
"RetryMax": 0,
"Tags": []
},
"vreplication_basic": {
"File": "unused.go",
"Args": ["vitess.io/vitess/go/test/endtoend/vreplication", "-run", "Basic"],
"Command": [],
"Manual": false,
"Shard": 24,
"RetryMax": 0,
"Tags": []
}
}
}

0 comments on commit d58a0fe

Please sign in to comment.