forked from tarantool/tarantool
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
test: cover cluster upgrade with vshard
This commit adds a test, that covers the integration of tarantool's config module with vshard and verifies correctness of changes in tarantool#9514 and tarantool/vshard#458. It checks, that we're able to upgrade several vshard clusters without downtime. NO_DOC=test NO_CHANGELOG=test
- Loading branch information
1 parent
557567e
commit 505224c
Showing
6 changed files
with
317 additions
and
0 deletions.
There are no files selected for viewing
Binary file added
BIN
+9.82 KB
test/box-luatest/upgrade/2.11.0/vshard/instance-001/00000000000000003049.snap
Binary file not shown.
Binary file added
BIN
+9.82 KB
test/box-luatest/upgrade/2.11.0/vshard/instance-002/00000000000000003049.snap
Binary file not shown.
Binary file added
BIN
+10.2 KB
test/box-luatest/upgrade/2.11.0/vshard/instance-003/00000000000000003049.snap
Binary file not shown.
Binary file added
BIN
+10.2 KB
test/box-luatest/upgrade/2.11.0/vshard/instance-004/00000000000000003049.snap
Binary file not shown.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,60 @@ | ||
-- Use vshard's example with the following storage script. | ||
|
||
-- Get instance name | ||
local fio = require('fio') | ||
local NAME = fio.basename(arg[0], '.lua') | ||
|
||
-- Call a configuration provider | ||
local cfg = { | ||
sharding = { | ||
['cbf06940-0790-498b-948d-042b62cf3d29'] = { -- replicaset #1 | ||
replicas = { | ||
['8a274925-a26d-47fc-9e1b-af88ce939412'] = { | ||
uri = 'storage:storage@127.0.0.1:3301', | ||
name = 'storage_1_a', | ||
master = true | ||
}, | ||
['3de2e3e1-9ebe-4d0d-abb1-26d301b84633'] = { | ||
uri = 'storage:storage@127.0.0.1:3302', | ||
name = 'storage_1_b' | ||
} | ||
}, | ||
}, -- replicaset #1 | ||
['ac522f65-aa94-4134-9f64-51ee384f1a54'] = { -- replicaset #2 | ||
replicas = { | ||
['1e02ae8a-afc0-4e91-ba34-843a356b8ed7'] = { | ||
uri = 'storage:storage@127.0.0.1:3303', | ||
name = 'storage_2_a', | ||
master = true | ||
}, | ||
['001688c3-66f8-4a31-8e19-036c17d489c2'] = { | ||
uri = 'storage:storage@127.0.0.1:3304', | ||
name = 'storage_2_b' | ||
} | ||
}, | ||
}, -- replicaset #2 | ||
}, -- sharding | ||
} | ||
|
||
-- Name to uuid map | ||
local names = { | ||
['storage_1_a'] = '8a274925-a26d-47fc-9e1b-af88ce939412', | ||
['storage_1_b'] = '3de2e3e1-9ebe-4d0d-abb1-26d301b84633', | ||
['storage_2_a'] = '1e02ae8a-afc0-4e91-ba34-843a356b8ed7', | ||
['storage_2_b'] = '001688c3-66f8-4a31-8e19-036c17d489c2', | ||
} | ||
|
||
-- Start the database with sharding | ||
local vshard = require('vshard') | ||
vshard.storage.cfg(cfg, names[NAME]) | ||
|
||
box.once("testapp", function() | ||
box.schema.user.grant('guest', 'super') | ||
box.schema.func.create('put') | ||
box.schema.func.create('get') | ||
|
||
local format = {{'id', 'unsigned'}, {'bucket_id', 'unsigned'}} | ||
local a = box.schema.space.create('a', {format = format}) | ||
a:create_index('id', {parts = {'id'}}) | ||
a:create_index('bucket_id', {parts = {'bucket_id'}, unique = false}) | ||
end) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,257 @@ | ||
local t = require('luatest') | ||
local treegen = require('test.treegen') | ||
local server = require('test.luatest_helpers.server') | ||
local yaml = require('yaml') | ||
local fun = require('fun') | ||
local fio = require('fio') | ||
|
||
local g = t.group() | ||
|
||
local has_vshard = pcall(require, 'vshard') | ||
|
||
-- | ||
-- Test, that Tarantool upgrade with vshard happens without downtime: | ||
-- | ||
-- 1. Recover cluster from 2.11 xlogs with config. | ||
-- 2. Vshard works before schema upgrade, no names are set. | ||
-- 3. Schema upgrade on one shard, check vshard, wait for names. | ||
-- 4. Vshard works, when names are set on the half of the cluster. | ||
-- 5. Schema upgrade on the other shard, check vshard, wait for names. | ||
-- 6. Vshard works after all names are applied. | ||
-- | ||
g.before_all(function(g) | ||
t.skip_if(not has_vshard, 'Module "vshard" is not available') | ||
treegen.init(g) | ||
local uuids = { | ||
['rs-001'] = 'cbf06940-0790-498b-948d-042b62cf3d29', | ||
['rs-002'] = 'ac522f65-aa94-4134-9f64-51ee384f1a54', | ||
['instance-001'] = '8a274925-a26d-47fc-9e1b-af88ce939412', | ||
['instance-002'] = '3de2e3e1-9ebe-4d0d-abb1-26d301b84633', | ||
['instance-003'] = '1e02ae8a-afc0-4e91-ba34-843a356b8ed7', | ||
['instance-004'] = '001688c3-66f8-4a31-8e19-036c17d489c2', | ||
} | ||
|
||
local prefix = 'test/box-luatest/upgrade/2.11.0/vshard/' | ||
local datadirs = { | ||
fio.abspath(fio.pathjoin(prefix, 'instance-001')), | ||
fio.abspath(fio.pathjoin(prefix, 'instance-002')), | ||
fio.abspath(fio.pathjoin(prefix, 'instance-003')), | ||
fio.abspath(fio.pathjoin(prefix, 'instance-004')), | ||
} | ||
|
||
local dir = treegen.prepare_directory(g, {}, {}) | ||
local workdirs = { | ||
fio.pathjoin(dir, 'instance-001'), | ||
fio.pathjoin(dir, 'instance-002'), | ||
fio.pathjoin(dir, 'instance-003'), | ||
fio.pathjoin(dir, 'instance-004'), | ||
} | ||
|
||
for i, workdir in ipairs(workdirs) do | ||
fio.mktree(workdir) | ||
fio.copytree(datadirs[i], workdir) | ||
end | ||
|
||
local config = { | ||
iproto = { | ||
listen = {{uri = 'unix/:./{{ instance_name }}.iproto'}}, | ||
advertise = { | ||
sharding = { | ||
login = 'storage', | ||
password = 'storage', | ||
} | ||
} | ||
}, | ||
sharding = { bucket_count = 3000, }, | ||
groups = { | ||
['group-001'] = { | ||
replicasets = { | ||
['replicaset-001'] = { | ||
database = {replicaset_uuid = uuids['rs-001']}, | ||
sharding = {roles = {'storage'}}, | ||
instances = { | ||
['instance-001'] = { | ||
snapshot = {dir = workdirs[1]}, | ||
wal = {dir = workdirs[1]}, | ||
database = { | ||
instance_uuid = uuids['instance-001'], | ||
mode = 'rw', | ||
}, | ||
}, | ||
['instance-002'] = { | ||
snapshot = {dir = workdirs[2]}, | ||
wal = {dir = workdirs[2]}, | ||
database = { | ||
instance_uuid = uuids['instance-002'] | ||
} | ||
}, | ||
}, | ||
}, | ||
['replicaset-002'] = { | ||
database = {replicaset_uuid = uuids['rs-002']}, | ||
sharding = {roles = {'storage'}}, | ||
instances = { | ||
['instance-003'] = { | ||
snapshot = {dir = workdirs[3]}, | ||
wal = {dir = workdirs[3]}, | ||
database = { | ||
instance_uuid = uuids['instance-003'], | ||
mode = 'rw', | ||
}, | ||
}, | ||
['instance-004'] = { | ||
snapshot = {dir = workdirs[4]}, | ||
wal = {dir = workdirs[4]}, | ||
database = { | ||
instance_uuid = uuids['instance-004'] | ||
} | ||
}, | ||
}, | ||
}, | ||
['replicaset-003'] = { | ||
sharding = {roles = {'router'}}, | ||
instances = { | ||
['instance-005'] = { | ||
database = {mode = 'rw'}, | ||
credentials = { | ||
users = { | ||
guest = {roles = {'super'}} | ||
} | ||
} | ||
} | ||
} | ||
} | ||
}, | ||
}, | ||
}, | ||
} | ||
|
||
local cfg = yaml.encode(config) | ||
local config_file = treegen.write_script(dir, 'cfg.yaml', cfg) | ||
local opts = { | ||
env = {LUA_PATH = os.environ()['LUA_PATH']}, | ||
config_file = config_file, | ||
chdir = dir | ||
} | ||
g.instance_1 = server:new(fun.chain(opts, {alias = 'instance-001'}):tomap()) | ||
g.instance_2 = server:new(fun.chain(opts, {alias = 'instance-002'}):tomap()) | ||
g.instance_3 = server:new(fun.chain(opts, {alias = 'instance-003'}):tomap()) | ||
g.instance_4 = server:new(fun.chain(opts, {alias = 'instance-004'}):tomap()) | ||
g.instance_5 = server:new(fun.chain(opts, {alias = 'instance-005'}):tomap()) | ||
g.instance_1:start({wait_until_ready = false}) | ||
g.instance_2:start({wait_until_ready = false}) | ||
g.instance_3:start({wait_until_ready = false}) | ||
g.instance_4:start({wait_until_ready = false}) | ||
g.instance_5:start({wait_until_ready = false}) | ||
g.instance_1:wait_until_ready() | ||
g.instance_2:wait_until_ready() | ||
g.instance_3:wait_until_ready() | ||
g.instance_4:wait_until_ready() | ||
g.instance_5:wait_until_ready() | ||
|
||
local exec = [[ | ||
function put(v) | ||
box.space.a:insert({v.id, v.bucket_id}) | ||
return true | ||
end | ||
function get(id) | ||
return box.space.a:get(id) | ||
end | ||
]] | ||
g.instance_1:eval(exec) | ||
g.instance_2:eval(exec) | ||
g.instance_3:eval(exec) | ||
g.instance_4:eval(exec) | ||
end) | ||
|
||
g.after_all(function(g) | ||
g.instance_1:drop() | ||
g.instance_2:drop() | ||
g.instance_3:drop() | ||
g.instance_4:drop() | ||
g.instance_5:drop() | ||
treegen.clean(g) | ||
end) | ||
|
||
local function check_vshard(g, id_1, id_2) | ||
local write_exec = string.format([[ | ||
local ok, err = nil, {} | ||
local opts = {timeout = 20} | ||
ok, err[1] = vshard.router.call(1, 'write', 'put', | ||
{{id = %s, bucket_id = 1}}, opts) | ||
ok, err[2] = vshard.router.call(2000, 'write', 'put', | ||
{{id = %s, bucket_id = 2000}}, opts) | ||
return err | ||
]], id_1, id_2) | ||
local err = g.instance_5:eval(write_exec) | ||
t.assert_equals(err[1], nil) | ||
t.assert_equals(err[2], nil) | ||
|
||
local read_exec = string.format([[ | ||
local ret = {} | ||
ret[1] = vshard.router.call(1, 'read', 'get', {%s}) | ||
ret[2] = vshard.router.call(2000, 'read', 'get', {%s}) | ||
return ret | ||
]], id_1, id_2) | ||
t.helpers.retrying({timeout = 20}, function() | ||
local ret = g.instance_5:eval(read_exec) | ||
t.assert_equals(ret[1], {id_1, 1}) | ||
t.assert_equals(ret[2], {id_2, 2000}) | ||
end) | ||
end | ||
|
||
local function wait_for_names(instance, replicaset_name) | ||
t.helpers.retrying({timeout = 20, delay = 0.1}, function() | ||
local name, rs_name = instance:exec(function() | ||
local info = box.info | ||
return info.name, info.replicaset.name | ||
end) | ||
|
||
local info = box.info | ||
t.assert_equals(name, instance.alias) | ||
t.assert_equals(rs_name, replicaset_name) | ||
end) | ||
end | ||
|
||
g.test_vshard_upgrade = function(g) | ||
local version_exec = [[ return box.space._schema:get{'version'} ]] | ||
local version = g.instance_1:eval(version_exec) | ||
t.assert_equals(version, {'version', 2, 11, 1}) | ||
-- Assert, that tested buckets are on different shards. | ||
local bucket_exec = [[ return vshard.storage.buckets_info()[%s] ]] | ||
local bucket = g.instance_1:eval(string.format(bucket_exec, 2000)) | ||
t.assert_not_equals(bucket, nil) | ||
bucket = g.instance_3:eval(string.format(bucket_exec, 1)) | ||
t.assert_not_equals(bucket, nil) | ||
|
||
-- Check, that vshard works before schema upgrade. | ||
check_vshard(g, 1, 2000) | ||
|
||
-- Upgrade schema on one shard, wait for names to be applied. | ||
local upgrade_exec = [[ | ||
box.ctl.wait_rw() | ||
box.schema.upgrade() | ||
]] | ||
g.instance_1:eval(upgrade_exec) | ||
-- Before all names are applied, but after schema upgrade. | ||
-- Either none, some or all of the names were applied. | ||
check_vshard(g, 2, 2001) | ||
local rs_name = 'replicaset-001' | ||
wait_for_names(g.instance_1, rs_name) | ||
wait_for_names(g.instance_2, rs_name) | ||
-- Check, that vshard works, when one shard has names, other - doesn't. | ||
check_vshard(g, 3, 2002) | ||
|
||
-- Upgrade schema on the other shard, wait for names. | ||
g.instance_3:eval(upgrade_exec) | ||
-- Before all names are applied, but after schema upgrade. | ||
check_vshard(g, 4, 2003) | ||
-- Wait for names to be applied. | ||
rs_name = 'replicaset-002' | ||
wait_for_names(g.instance_3, rs_name) | ||
wait_for_names(g.instance_4, rs_name) | ||
|
||
-- Check, that vshard works after all names are set. | ||
check_vshard(g, 5, 2004) | ||
end |