Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

merge pingcap/release-3.1 #10

Merged
merged 25 commits into from
Feb 15, 2020
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
25 commits
Select commit Hold shift + click to select a range
c1fa994
[3.1] external_storage: add S3 support (#6209) (#6536)
kennytm Feb 8, 2020
647daec
lock_manager: more metrics (#6392) (#6422) (#6444)
youjiali1995 Feb 11, 2020
1de4312
lock_manager: update default config (#6426) (#6429) (#6446)
youjiali1995 Feb 11, 2020
62b5d65
Update rocksdb and titan. Update rust-rocksdb url (#6554)
Feb 12, 2020
52a7cae
raft_client: connect to other stores using peer_address (#5569) (#6491)
disksing Feb 13, 2020
e190308
*: pick threads statistics for regions (#6605)
nolouch Feb 13, 2020
dee5c75
fix test_replica_read::test_read_after_cleanup_range_for_snap (#6493)
sre-bot Feb 13, 2020
d334f90
deadlock: more solid role change observer (#6415) (#6431) (#6445)
youjiali1995 Feb 14, 2020
2f02ee7
raftstore: read index message count metric (#6579) (#6610)
5kbpers Feb 14, 2020
5ef9d0b
raftstore: speed up conf change (#6421) (#6432)
BusyJay Feb 14, 2020
6068bf5
raftstore: fix a panic in read index queue (#6609) (#6613)
sre-bot Feb 14, 2020
2600f9a
pick #6414, #6487, #6477 and #6539 to release-3.1 (#6564)
BusyJay Feb 14, 2020
ec49dfb
raftstore: set wait_merge_state to none after resuming pending state …
gengliqi Feb 14, 2020
2a093e4
raftstore: fix a panic in read index queue (#6609) (#6613)
sre-bot Feb 14, 2020
9cb1e13
Merge remote-tracking branch 'pingcap/release-3.1' into __tiflash-pro…
solotzg Feb 14, 2020
2374a83
update cargo.lock
solotzg Feb 15, 2020
f5bd0d9
raftstore: learner load merge target & fix a merge network recovery b…
gengliqi Feb 15, 2020
ae7acda
1
solotzg Feb 15, 2020
3f6df2b
Merge remote-tracking branch 'pingcap/release-3.1' into __tiflash-pro…
solotzg Feb 15, 2020
914dd2a
fix
solotzg Feb 15, 2020
054b2e6
1
solotzg Feb 15, 2020
c9a2bc4
Merge remote-tracking branch 'origin/tiflash-proxy-master' into __tif…
solotzg Feb 15, 2020
d1f9d48
change merge flow path (#6617)
gengliqi Feb 15, 2020
e2dcc1f
wip
Feb 15, 2020
826d907
Merge remote-tracking branch 'pingcap/release-3.1' into __tiflash-pro…
solotzg Feb 15, 2020
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3,521 changes: 2,076 additions & 1,445 deletions Cargo.lock

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -105,6 +105,7 @@ engine = { path = "components/engine" }
tikv_util = { path = "components/tikv_util" }
farmhash = "1.1.5"
external_storage = { path = "components/external_storage" }
batch-system = { path = "components/batch-system" }

[dependencies.murmur3]
git = "https://github.com/pingcap/murmur3.git"
Expand All @@ -122,7 +123,6 @@ default-features = false
[replace]
"log:0.3.9" = { git = "https://github.com/busyjay/log", branch = "use-static-module" }
"log:0.4.6" = { git = "https://github.com/busyjay/log", branch = "revert-to-static" }
"https://github.com/pingcap/kvproto.git#kvproto:0.0.1" = { git = "https://github.com/solotzg/kvproto.git", branch = "learner-merge" }
"prometheus:0.4.2" = { git = "https://github.com/birdstorm/rust-prometheus", branch = "0.4.2" }

[dev-dependencies]
Expand Down
18 changes: 11 additions & 7 deletions components/backup/src/writer.rs
Original file line number Diff line number Diff line change
Expand Up @@ -69,14 +69,18 @@ impl Writer {
.observe(sst_info.file_size() as f64);
let file_name = format!("{}_{}.sst", name, cf);

let reader = Sha256Reader::new(sst_reader)
let (reader, hasher) = Sha256Reader::new(sst_reader)
.map_err(|e| Error::Other(box_err!("Sha256 error: {:?}", e)))?;
let mut reader = limiter.limit(AllowStdIo::new(reader));
storage.write(&file_name, &mut reader)?;
let sha256 = reader
.into_inner()
.into_inner()
.hash()
storage.write(
&file_name,
Box::new(limiter.limit(AllowStdIo::new(reader))),
sst_info.file_size(),
)?;
let sha256 = hasher
.lock()
.unwrap()
.finish()
.map(|digest| digest.to_vec())
.map_err(|e| Error::Other(box_err!("Sha256 error: {:?}", e)))?;

let mut file = File::new();
Expand Down
29 changes: 29 additions & 0 deletions components/batch-system/Cargo.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
[package]
name = "batch-system"
version = "0.1.0"
edition = "2018"

[dependencies]
crossbeam = "0.7"
tikv_util = { path = "../tikv_util" }
slog = { version = "2.3", features = ["max_level_trace", "release_max_level_debug"] }
slog-global = { version = "0.1", git = "https://github.com/breeswish/slog-global.git", rev = "91904ade" }
derive_more = "0.15.0"

[dev-dependencies]
criterion = "0.3"

[[test]]
name = "tests"
path = "tests/cases/mod.rs"
required-features = ["test-runner"]

[[bench]]
name = "router"
harness = false
required-features = ["test-runner"]

[[bench]]
name = "batch-system"
harness = false
required-features = ["test-runner"]
142 changes: 142 additions & 0 deletions components/batch-system/benches/batch-system.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,142 @@
// Copyright 2020 TiKV Project Authors. Licensed under Apache-2.0.

#![feature(test)]

extern crate test;

use batch_system::test_runner::*;
use batch_system::*;
use criterion::*;
use std::sync::atomic::*;
use std::sync::Arc;

fn end_hook(tx: &std::sync::mpsc::Sender<()>) -> Message {
let tx = tx.clone();
Message::Callback(Box::new(move |_| {
tx.send(()).unwrap();
}))
}

/// Benches how it performs when many messages are sent to the bench system.
///
/// A better router and lightweight batch scheduling can lead to better result.
fn bench_spawn_many(c: &mut Criterion) {
let (control_tx, control_fsm) = Runner::new(100000);
let (router, mut system) = batch_system::create_system(2, 2, control_tx, control_fsm);
system.spawn("test".to_owned(), Builder::new());
const ID_LIMIT: u64 = 32;
const MESSAGE_LIMIT: usize = 256;
for id in 0..ID_LIMIT {
let (normal_tx, normal_fsm) = Runner::new(100000);
let normal_box = BasicMailbox::new(normal_tx, normal_fsm);
router.register(id, normal_box);
}

let (tx, rx) = std::sync::mpsc::channel();
c.bench_function("spawn_many", |b| {
b.iter(|| {
for id in 0..ID_LIMIT {
for i in 0..MESSAGE_LIMIT {
router.send(id, Message::Loop(i)).unwrap();
}
router.send(id, end_hook(&tx)).unwrap();
}
for _ in 0..ID_LIMIT {
rx.recv().unwrap();
}
})
});
system.shutdown();
}

/// Bench how it performs if two hot FSMs are shown up at the same time.
///
/// A good scheduling algorithm should be able to spread the hot FSMs to
/// all available threads as soon as possible.
fn bench_imbalance(c: &mut Criterion) {
let (control_tx, control_fsm) = Runner::new(100000);
let (router, mut system) = batch_system::create_system(2, 2, control_tx, control_fsm);
system.spawn("test".to_owned(), Builder::new());
const ID_LIMIT: u64 = 10;
const MESSAGE_LIMIT: usize = 512;
for id in 0..ID_LIMIT {
let (normal_tx, normal_fsm) = Runner::new(100000);
let normal_box = BasicMailbox::new(normal_tx, normal_fsm);
router.register(id, normal_box);
}

let (tx, rx) = std::sync::mpsc::channel();
c.bench_function("imbalance", |b| {
b.iter(|| {
for i in 0..MESSAGE_LIMIT {
for id in 0..2 {
router.send(id, Message::Loop(i)).unwrap();
}
}
for id in 0..2 {
router.send(id, end_hook(&tx)).unwrap();
}
for _ in 0..2 {
rx.recv().unwrap();
}
})
});
system.shutdown();
}

/// Bench how it performs when scheduling a lot of quick tasks during an long-polling
/// tasks.
///
/// A good scheduling algorithm should not starve the quick tasks.
fn bench_fairness(c: &mut Criterion) {
let (control_tx, control_fsm) = Runner::new(100000);
let (router, mut system) = batch_system::create_system(2, 2, control_tx, control_fsm);
system.spawn("test".to_owned(), Builder::new());
for id in 0..10 {
let (normal_tx, normal_fsm) = Runner::new(100000);
let normal_box = BasicMailbox::new(normal_tx, normal_fsm);
router.register(id, normal_box);
}

let (tx, _rx) = std::sync::mpsc::channel();
let running = Arc::new(AtomicBool::new(true));
let router1 = router.clone();
let running1 = running.clone();
let handle = std::thread::spawn(move || {
while running1.load(Ordering::SeqCst) {
// Using 4 to ensure all worker threads are busy spinning.
for id in 0..4 {
let _ = router1.send(id, Message::Loop(16));
}
}
tx.send(()).unwrap();
});

let (tx2, rx2) = std::sync::mpsc::channel();
c.bench_function("fairness", |b| {
b.iter(|| {
for _ in 0..10 {
for id in 4..6 {
router.send(id, Message::Loop(10)).unwrap();
}
}
for id in 4..6 {
router.send(id, end_hook(&tx2)).unwrap();
}
for _ in 4..6 {
rx2.recv().unwrap();
}
})
});
running.store(false, Ordering::SeqCst);
system.shutdown();
let _ = handle.join();
}

criterion_group!(fair, bench_fairness);
criterion_group!(
name = load;
config = Criterion::default().sample_size(30);
targets = bench_imbalance, bench_spawn_many
);
criterion_main!(fair, load);
24 changes: 24 additions & 0 deletions components/batch-system/benches/router.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
// Copyright 2020 TiKV Project Authors. Licensed under Apache-2.0.

use batch_system::test_runner::*;
use batch_system::*;
use criterion::*;

fn bench_send(c: &mut Criterion) {
let (control_tx, control_fsm) = Runner::new(100000);
let (router, mut system) = batch_system::create_system(2, 2, control_tx, control_fsm);
system.spawn("test".to_owned(), Builder::new());
let (normal_tx, normal_fsm) = Runner::new(100000);
let normal_box = BasicMailbox::new(normal_tx, normal_fsm);
router.register(1, normal_box);

c.bench_function("router::send", |b| {
b.iter(|| {
router.send(1, Message::Loop(0)).unwrap();
})
});
system.shutdown();
}

criterion_group!(benches, bench_send);
criterion_main!(benches);
Loading