Skip to content

Stop crash on disk failure #7218

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
shinrich merged 3 commits into from
Oct 5, 2020
Merged

Stop crash on disk failure #7218

Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
b21d458
Stop crash on disk failure
shinrich Sep 25, 2020
3d78d16
Defeat clang-analyzer
shinrich Sep 28, 2020
63a19e5
Tidy up declarations
shinrich Sep 28, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
75 changes: 48 additions & 27 deletions iocore/cache/Cache.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -589,13 +589,22 @@ CacheProcessor::start_internal(int flags)
fix = !!(flags & PROCESSOR_FIX);
check = (flags & PROCESSOR_CHECK) != 0;
start_done = 0;
Span *sd;

/* read the config file and create the data structures corresponding
to the file */
gndisks = theCacheStore.n_disks;
gdisks = static_cast<CacheDisk **>(ats_malloc(gndisks * sizeof(CacheDisk *)));

// Temporaries to carry values between loops
char **paths = static_cast<char **>(alloca(sizeof(char *) * gndisks));
memset(paths, 0, sizeof(char *) * gndisks);
int *fds = static_cast<int *>(alloca(sizeof(int) * gndisks));
memset(fds, 0, sizeof(int) * gndisks);
int *sector_sizes = static_cast<int *>(alloca(sizeof(int) * gndisks));
memset(sector_sizes, 0, sizeof(int) * gndisks);
Span **sds = static_cast<Span **>(alloca(sizeof(Span *) * gndisks));
memset(sds, 0, sizeof(Span *) * gndisks);

gndisks = 0;
ink_aio_set_callback(new AIO_Callback_handler());

Expand All @@ -605,13 +614,15 @@ CacheProcessor::start_internal(int flags)
create CacheDisk objects for each span in the configuration file and store in gdisks
*/
for (unsigned i = 0; i < theCacheStore.n_disks; i++) {
sd = theCacheStore.disk[i];
char path[PATH_NAME_MAX];
Span *sd = theCacheStore.disk[i];
int opts = DEFAULT_CACHE_OPTIONS;

ink_strlcpy(path, sd->pathname, sizeof(path));
if (!paths[gndisks]) {
paths[gndisks] = static_cast<char *>(alloca(PATH_NAME_MAX));
}
ink_strlcpy(paths[gndisks], sd->pathname, PATH_NAME_MAX);
if (!sd->file_pathname) {
ink_strlcat(path, "/cache.db", sizeof(path));
ink_strlcat(paths[gndisks], "/cache.db", PATH_NAME_MAX);
opts |= O_CREAT;
}

Expand All @@ -626,29 +637,29 @@ CacheProcessor::start_internal(int flags)
opts |= O_RDONLY;
}

int fd = open(path, opts, 0644);
int fd = open(paths[gndisks], opts, 0644);
int64_t blocks = sd->blocks;

if (fd < 0 && (opts & O_CREAT)) { // Try without O_DIRECT if this is a file on filesystem, e.g. tmpfs.
fd = open(path, DEFAULT_CACHE_OPTIONS | O_CREAT, 0644);
fd = open(paths[gndisks], DEFAULT_CACHE_OPTIONS | O_CREAT, 0644);
}

if (fd >= 0) {
bool diskok = true;
if (!sd->file_pathname) {
if (!check) {
if (ftruncate(fd, blocks * STORE_BLOCK_SIZE) < 0) {
Warning("unable to truncate cache file '%s' to %" PRId64 " blocks", path, blocks);
Warning("unable to truncate cache file '%s' to %" PRId64 " blocks", paths[gndisks], blocks);
diskok = false;
}
} else { // read-only mode checks
struct stat sbuf;
if (-1 == fstat(fd, &sbuf)) {
fprintf(stderr, "Failed to stat cache file for directory %s\n", path);
fprintf(stderr, "Failed to stat cache file for directory %s\n", paths[gndisks]);
diskok = false;
} else if (blocks != sbuf.st_size / STORE_BLOCK_SIZE) {
fprintf(stderr, "Cache file for directory %s is %" PRId64 " bytes, expected %" PRId64 "\n", path, sbuf.st_size,
blocks * static_cast<int64_t>(STORE_BLOCK_SIZE));
fprintf(stderr, "Cache file for directory %s is %" PRId64 " bytes, expected %" PRId64 "\n", paths[gndisks],
sbuf.st_size, blocks * static_cast<int64_t>(STORE_BLOCK_SIZE));
diskok = false;
}
}
Expand Down Expand Up @@ -676,31 +687,26 @@ CacheProcessor::start_internal(int flags)
Note("resetting hardware sector size from %d to %d", sector_size, STORE_BLOCK_SIZE);
sector_size = STORE_BLOCK_SIZE;
}

off_t skip = ROUND_TO_STORE_BLOCK((sd->offset < START_POS ? START_POS + sd->alignment : sd->offset));
blocks = blocks - (skip >> STORE_BLOCK_SHIFT);
#if AIO_MODE == AIO_MODE_NATIVE
eventProcessor.schedule_imm(new DiskInit(gdisks[gndisks], path, blocks, skip, sector_size, fd, clear));
#else
gdisks[gndisks]->open(path, blocks, skip, sector_size, fd, clear);
#endif

Debug("cache_hosting", "Disk: %d:%s, blocks: %" PRId64 "", gndisks, path, blocks);
fd = -1;
sector_sizes[gndisks] = sector_size;
fds[gndisks] = fd;
sds[gndisks] = sd;
fd = -1;
gndisks++;
}
} else {
if (errno == EINVAL) {
Warning("cache unable to open '%s': It must be placed on a file system that supports direct I/O.", path);
Warning("cache unable to open '%s': It must be placed on a file system that supports direct I/O.", paths[gndisks]);
} else {
Warning("cache unable to open '%s': %s", path, strerror(errno));
Warning("cache unable to open '%s': %s", paths[gndisks], strerror(errno));
}
}
if (fd >= 0) {
close(fd);
}
}

// Before we kick off asynchronous operations, make sure sufficient disks are available and we don't just shutdown
// Exiting with background threads in operation will likely cause a seg fault
start_done = 1;

if (gndisks == 0) {
Expand All @@ -711,7 +717,7 @@ CacheProcessor::start_internal(int flags)
}

if (this->waitForCache() > 1) {
Fatal("Cache initialization failed - no disks available but cache required");
Emergency("Cache initialization failed - no disks available but cache required");
} else {
Warning("unable to open cache disk(s): Cache Disabled\n");
return -1; // pointless, AFAICT this is ignored.
Expand All @@ -721,8 +727,23 @@ CacheProcessor::start_internal(int flags)
if (cb_after_init) {
cb_after_init();
}
Fatal("Cache initialization failed - only %d out of %d disks were valid and all were required.", gndisks,
theCacheStore.n_disks_in_config);
Emergency("Cache initialization failed - only %d out of %d disks were valid and all were required.", gndisks,
theCacheStore.n_disks_in_config);
}

// If we got here, we have enough disks to proceed
for (int j = 0; j < gndisks; j++) {
Span *sd = sds[j];
ink_release_assert(sds[j] != nullptr); // Defeat clang-analyzer
off_t skip = ROUND_TO_STORE_BLOCK((sd->offset < START_POS ? START_POS + sd->alignment : sd->offset));
int64_t blocks = sd->blocks - (skip >> STORE_BLOCK_SHIFT);
#if AIO_MODE == AIO_MODE_NATIVE
eventProcessor.schedule_imm(new DiskInit(gdisks[j], paths[j], blocks, skip, sector_sizes[j], fds[j], clear));
#else
gdisks[j]->open(paths[j], blocks, skip, sector_sizes[j], fds[j], clear);
#endif

Debug("cache_hosting", "Disk: %d:%s, blocks: %" PRId64 "", gndisks, paths[j], blocks);
}

return 0;
Expand Down