Skip to content

Commit

Permalink
Fix shard failure due to translog tragic close on upload failure
Browse files Browse the repository at this point in the history
Signed-off-by: Ashish Singh <ssashish@amazon.com>
  • Loading branch information
ashking94 committed Oct 4, 2023
1 parent 7159e2e commit 28cecbd
Show file tree
Hide file tree
Showing 2 changed files with 28 additions and 18 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -242,15 +242,10 @@ public static TranslogTransferManager buildTranslogTransferManager(

@Override
public boolean ensureSynced(Location location) throws IOException {
try {
assert location.generation <= current.getGeneration();
if (location.generation == current.getGeneration()) {
ensureOpen();
return prepareAndUpload(primaryTermSupplier.getAsLong(), location.generation);
}
} catch (final Exception ex) {
closeOnTragicEvent(ex);
throw ex;
assert location.generation <= current.getGeneration();
if (location.generation == current.getGeneration()) {
ensureOpen();
return prepareAndUpload(primaryTermSupplier.getAsLong(), location.generation);
}
return false;
}
Expand Down Expand Up @@ -355,14 +350,8 @@ private boolean syncToDisk() throws IOException {

@Override
public void sync() throws IOException {
try {
if (syncToDisk() || syncNeeded()) {
prepareAndUpload(primaryTermSupplier.getAsLong(), null);
}
} catch (final Exception e) {
tragedy.setTragicException(e);
closeOnTragicEvent(e);
throw e;
if (syncToDisk() || syncNeeded()) {
prepareAndUpload(primaryTermSupplier.getAsLong(), null);
}
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,7 @@
import java.nio.ByteBuffer;
import java.nio.channels.FileChannel;
import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.NoSuchFileException;
import java.nio.file.Path;
Expand Down Expand Up @@ -1049,7 +1050,7 @@ public void testSyncUpTo() throws IOException {
}
}

public void testSyncUpFailure() throws IOException {
public void testSyncUpLocationFailure() throws IOException {
int translogOperations = randomIntBetween(1, 20);
int count = 0;
fail.failAlways();
Expand Down Expand Up @@ -1101,6 +1102,26 @@ public void testSyncUpFailure() throws IOException {
assertDownloadStatsNoDownloads(statsTracker);
}

public void testSyncUpAlwaysFailure() throws IOException {
int translogOperations = randomIntBetween(1, 20);
int count = 0;
fail.failAlways();
for (int op = 0; op < translogOperations; op++) {
translog.add(
new Translog.Index(String.valueOf(op), count, primaryTerm.get(), Integer.toString(count).getBytes(StandardCharsets.UTF_8))
);
try {
translog.sync();
fail("io exception expected");
} catch (IOException e) {
assertTrue("at least one operation pending", translog.syncNeeded());
}
}
assertTrue(translog.isOpen());
fail.failNever();
translog.sync();
}

public void testSyncUpToStream() throws IOException {
int iters = randomIntBetween(5, 10);
for (int i = 0; i < iters; i++) {
Expand Down

0 comments on commit 28cecbd

Please sign in to comment.