Skip to content

Commit

Permalink
Merge pull request #754 from Mark-Simulacrum/debug-crater
Browse files Browse the repository at this point in the history
Debug crater
  • Loading branch information
Mark-Simulacrum authored Dec 8, 2024
2 parents 82aff62 + d67f575 commit 61ed2a4
Show file tree
Hide file tree
Showing 3 changed files with 52 additions and 41 deletions.
5 changes: 5 additions & 0 deletions .cargo/config.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
[build]
rustflags = [
"-Cforce-frame-pointers",
"-Csymbol-mangling-version=v0"
]
3 changes: 3 additions & 0 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,9 @@ default-run = "crater"
[profile.dev]
opt-level = 0

[profile.release]
strip = false

[dependencies]
base64 = "0.21.5"
bytes = "1"
Expand Down
85 changes: 44 additions & 41 deletions src/server/routes/agent.rs
Original file line number Diff line number Diff line change
Expand Up @@ -210,48 +210,51 @@ impl RecordProgressThread {
in_flight_requests,
};
let ret = this.clone();
std::thread::spawn(move || loop {
// Panics should already be logged and otherwise there's not much we
// can/should do.
let _ = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
let result = rx.recv().unwrap();
this.block_until_idle();

let start = std::time::Instant::now();

if let Some(ex) = Experiment::get(&db, &result.experiment_name).unwrap() {
let db = DatabaseDB::new(&db);
if let Err(e) = db.store(&ex, &result.data, EncodingType::Plain) {
// Failing to record a result is basically fine -- this
// just means that we'll have to re-try this job.
log::error!("Failed to store result into database: {:?}", e);
crate::utils::report_failure(&e);
std::thread::Builder::new()
.name(String::from("record-prog-crater"))
.spawn(move || loop {
// Panics should already be logged and otherwise there's not much we
// can/should do.
let _ = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
let result = rx.recv().unwrap();
this.block_until_idle();

let start = std::time::Instant::now();

if let Some(ex) = Experiment::get(&db, &result.experiment_name).unwrap() {
let db = DatabaseDB::new(&db);
if let Err(e) = db.store(&ex, &result.data, EncodingType::Plain) {
// Failing to record a result is basically fine -- this
// just means that we'll have to re-try this job.
log::error!("Failed to store result into database: {:?}", e);
crate::utils::report_failure(&e);
}

metrics.record_completed_jobs(&ex.name, 1);

if let Err(e) = db.clear_stale_records() {
// Not a hard failure. We can continue even if we failed
// to clear records from already completed runs...
log::error!("Failed to clear stale records: {:?}", e);
crate::utils::report_failure(&e);
}

metrics
.crater_endpoint_time
.with_label_values(&["record_progress_worker"])
.observe(start.elapsed().as_secs_f64());

metrics
.crater_progress_report
.with_label_values(&[
ex.name.as_str(),
&result.data.result.result.to_string(),
])
.inc();
}

metrics.record_completed_jobs(&ex.name, 1);

if let Err(e) = db.clear_stale_records() {
// Not a hard failure. We can continue even if we failed
// to clear records from already completed runs...
log::error!("Failed to clear stale records: {:?}", e);
crate::utils::report_failure(&e);
}

metrics
.crater_endpoint_time
.with_label_values(&["record_progress_worker"])
.observe(start.elapsed().as_secs_f64());

metrics
.crater_progress_report
.with_label_values(&[
ex.name.as_str(),
&result.data.result.result.to_string(),
])
.inc();
}
}));
});
}));
})
.unwrap();

ret
}
Expand Down

0 comments on commit 61ed2a4

Please sign in to comment.