Skip to content

Make bad disk detection more robust #10317

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
moonchen merged 1 commit into from
Aug 31, 2023
Merged

Make bad disk detection more robust #10317

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 5 additions & 5 deletions iocore/aio/AIO.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,8 +58,8 @@ int thread_is_created = 0;
RecInt cache_config_threads_per_disk = 12;
RecInt api_config_threads_per_disk = 12;

RecRawStatBlock *aio_rsb = nullptr;
Continuation *aio_err_callbck = nullptr;
RecRawStatBlock *aio_rsb = nullptr;
Continuation *aio_err_callback = nullptr;
// AIO Stats
std::atomic<uint64_t> aio_num_read = 0;
std::atomic<uint64_t> aio_bytes_read = 0;
Expand Down Expand Up @@ -153,9 +153,9 @@ new_AIOCallback()
}

void
ink_aio_set_callback(Continuation *callback)
ink_aio_set_err_callback(Continuation *callback)
{
aio_err_callbck = callback;
aio_err_callback = callback;
}

void
Expand Down Expand Up @@ -455,7 +455,7 @@ cache_op(AIOCallbackInternal *op)
res += err;
}
op->aio_result = res;
ink_assert(op->aio_result == (int64_t)a->aio_nbytes);
ink_assert(op->ok());
}
return 1;
}
Expand Down
2 changes: 1 addition & 1 deletion iocore/aio/I_AIO.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,7 @@ struct AIOCallback : public Continuation {
};

void ink_aio_init(ts::ModuleVersion version, AIOBackend backend = AIO_BACKEND_AUTO);
void ink_aio_set_callback(Continuation *error_callback);
void ink_aio_set_err_callback(Continuation *error_callback);

int ink_aio_read(AIOCallback *op,
int fromAPI = 0); // fromAPI is a boolean to indicate if this is from an API call such as upload proxy feature
Expand Down
15 changes: 9 additions & 6 deletions iocore/aio/P_AIO.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,10 +45,10 @@ static constexpr ts::ModuleVersion AIO_MODULE_INTERNAL_VERSION{AIO_MODULE_PUBLIC
TS_INLINE int
AIOCallback::ok()
{
return (off_t)aiocb.aio_nbytes == (off_t)aio_result;
return (aiocb.aio_nbytes == static_cast<size_t>(aio_result)) && (aio_result >= 0);
}

extern Continuation *aio_err_callbck;
extern Continuation *aio_err_callback;

struct AIO_Reqs;

Expand Down Expand Up @@ -92,13 +92,16 @@ AIOCallbackInternal::io_complete(int event, void *data)
{
(void)event;
(void)data;
if (aio_err_callbck && !ok()) {
if (aio_err_callback && !ok()) {
AIOCallback *err_op = new AIOCallbackInternal();
err_op->aiocb.aio_fildes = this->aiocb.aio_fildes;
err_op->aiocb.aio_lio_opcode = this->aiocb.aio_lio_opcode;
err_op->mutex = aio_err_callbck->mutex;
err_op->action = aio_err_callbck;
eventProcessor.schedule_imm(err_op);
err_op->mutex = aio_err_callback->mutex;
err_op->action = aio_err_callback;

// Take this lock in-line because we want to stop other I/O operations on this disk ASAP
SCOPED_MUTEX_LOCK(lock, aio_err_callback->mutex, this_ethread());
err_op->action.continuation->handleEvent(EVENT_NONE, err_op);
}
if (!action.cancelled && action.continuation) {
action.continuation->handleEvent(AIO_EVENT_DONE, this);
Expand Down
14 changes: 7 additions & 7 deletions iocore/cache/Cache.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -569,7 +569,7 @@ CacheProcessor::start_internal(int flags)
memset(sds, 0, sizeof(Span *) * gndisks);

gndisks = 0;
ink_aio_set_callback(new AIO_Callback_handler());
ink_aio_set_err_callback(new AIO_failure_handler());

config_volumes.read_config_file();

Expand Down Expand Up @@ -1263,7 +1263,7 @@ Vol::handle_dir_clear(int event, void *data)

if (event == AIO_EVENT_DONE) {
op = static_cast<AIOCallback *>(data);
if (static_cast<size_t>(op->aio_result) != op->aiocb.aio_nbytes) {
if (!op->ok()) {
Warning("unable to clear cache directory '%s'", hash_text.get());
disk->incrErrors(op);
fd = -1;
Expand Down Expand Up @@ -1292,7 +1292,7 @@ Vol::handle_dir_read(int event, void *data)
AIOCallback *op = static_cast<AIOCallback *>(data);

if (event == AIO_EVENT_DONE) {
if (static_cast<size_t>(op->aio_result) != op->aiocb.aio_nbytes) {
if (!op->ok()) {
Note("Directory read failed: clearing cache directory %s", this->hash_text.get());
clear_dir();
return EVENT_DONE;
Expand Down Expand Up @@ -1387,7 +1387,7 @@ Vol::handle_recover_from_data(int event, void * /* data ATS_UNUSED */)
io.aiocb.aio_nbytes = (skip + len) - recover_pos;
}
} else if (event == AIO_EVENT_DONE) {
if (io.aiocb.aio_nbytes != static_cast<size_t>(io.aio_result)) {
if (!io.ok()) {
Warning("disk read error on recover '%s', clearing", hash_text.get());
disk->incrErrors(&io);
goto Lclear;
Expand Down Expand Up @@ -1647,7 +1647,7 @@ Vol::handle_header_read(int event, void *data)
for (auto &i : hf) {
ink_assert(op != nullptr);
i = static_cast<VolHeaderFooter *>(op->aiocb.aio_buf);
if (static_cast<size_t>(op->aio_result) != op->aiocb.aio_nbytes) {
if (!op->ok()) {
Note("Header read failed: clearing cache directory %s", this->hash_text.get());
clear_dir();
return EVENT_DONE;
Expand Down Expand Up @@ -1929,7 +1929,7 @@ CacheProcessor::has_online_storage() const
}

int
AIO_Callback_handler::handle_disk_failure(int /* event ATS_UNUSED */, void *data)
AIO_failure_handler::handle_disk_failure(int /* event ATS_UNUSED */, void *data)
{
/* search for the matching file descriptor */
if (!CacheProcessor::cache_ready) {
Expand Down Expand Up @@ -2351,7 +2351,7 @@ CacheVC::removeEvent(int /* event ATS_UNUSED */, Event * /* e ATS_UNUSED */)
goto Lcollision;
}
// check read completed correct FIXME: remove bad vols
if (static_cast<size_t>(io.aio_result) != io.aiocb.aio_nbytes) {
if (!io.ok()) {
goto Ldone;
}
{
Expand Down
2 changes: 1 addition & 1 deletion iocore/cache/CacheDir.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1103,7 +1103,7 @@ CacheSync::mainEvent(int event, Event *e)

if (event == AIO_EVENT_DONE) {
// AIO Thread
if (io.aio_result != static_cast<int64_t>(io.aiocb.aio_nbytes)) {
if (!io.ok()) {
Warning("vol write error during directory sync '%s'", gvol[vol_idx]->hash_text.get());
event = EVENT_NONE;
goto Ldone;
Expand Down
6 changes: 3 additions & 3 deletions iocore/cache/CacheDisk.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -147,7 +147,7 @@ CacheDisk::clearDone(int event, void * /* data ATS_UNUSED */)
{
ink_assert(event == AIO_EVENT_DONE);

if (io.aiocb.aio_nbytes != static_cast<size_t>(io.aio_result)) {
if (!io.ok()) {
Warning("Could not clear disk header for disk %s: declaring disk bad", path);
incrErrors(&io);
SET_DISK_BAD(this);
Expand All @@ -163,7 +163,7 @@ CacheDisk::openStart(int event, void * /* data ATS_UNUSED */)
{
ink_assert(event == AIO_EVENT_DONE);

if (io.aiocb.aio_nbytes != static_cast<size_t>(io.aio_result)) {
if (!io.ok()) {
Warning("could not read disk header for disk %s: declaring disk bad", path);

// the header could have random values by the AIO read error
Expand Down Expand Up @@ -237,7 +237,7 @@ CacheDisk::syncDone(int event, void * /* data ATS_UNUSED */)
{
ink_assert(event == AIO_EVENT_DONE);

if (io.aiocb.aio_nbytes != static_cast<size_t>(io.aio_result)) {
if (!io.ok()) {
Warning("Error writing disk header for disk %s:disk bad", path);
incrErrors(&io);
SET_DISK_BAD(this);
Expand Down
2 changes: 1 addition & 1 deletion iocore/cache/CacheVol.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -210,7 +210,7 @@ CacheVC::scanObject(int /* event ATS_UNUSED */, Event * /* e ATS_UNUSED */)
goto Lread;
}

if (static_cast<size_t>(io.aio_result) != io.aiocb.aio_nbytes) {
if (!io.ok()) {
result = (void *)-ECACHE_READ_FAIL;
goto Ldone;
}
Expand Down
4 changes: 2 additions & 2 deletions iocore/cache/P_CacheVol.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -256,10 +256,10 @@ struct Vol : public Continuation {
~Vol() override { ats_free(agg_buffer); }
};

struct AIO_Callback_handler : public Continuation {
struct AIO_failure_handler : public Continuation {
int handle_disk_failure(int event, void *data);

AIO_Callback_handler() : Continuation(new_ProxyMutex()) { SET_HANDLER(&AIO_Callback_handler::handle_disk_failure); }
AIO_failure_handler() : Continuation(new_ProxyMutex()) { SET_HANDLER(&AIO_failure_handler::handle_disk_failure); }
};

struct CacheVol {
Expand Down