Skip to content

Commit 6654c53

Browse files
committed
fixes from merge
1 parent de0a596 commit 6654c53

File tree

2 files changed

+4
-2
lines changed

2 files changed

+4
-2
lines changed

core/src/main/java/org/apache/spark/shuffle/unsafe/UnsafeShuffleWriter.java

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -175,6 +175,8 @@ void closeAndWriteOutput() throws IOException {
175175
final SpillInfo[] spills = sorter.closeAndGetSpills();
176176
sorter = null;
177177
final long[] partitionLengths;
178+
final File outputFile = shuffleBlockResolver.getDataFile(shuffleId, mapId);
179+
final long initialFileLength = outputFile.length();
178180
try {
179181
partitionLengths = mergeSpills(spills);
180182
} finally {
@@ -184,7 +186,7 @@ void closeAndWriteOutput() throws IOException {
184186
}
185187
}
186188
}
187-
shuffleBlockResolver.writeIndexFile(shuffleId, mapId, partitionLengths);
189+
shuffleBlockResolver.writeIndexFile(shuffleId, mapId, partitionLengths, initialFileLength);
188190
mapStatus = MapStatus$.MODULE$.apply(blockManager.shuffleServerId(), partitionLengths);
189191
}
190192

core/src/test/java/org/apache/spark/shuffle/unsafe/UnsafeShuffleWriterSuite.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -175,7 +175,7 @@ public Void answer(InvocationOnMock invocationOnMock) throws Throwable {
175175
partitionSizesInMergedFile = (long[]) invocationOnMock.getArguments()[2];
176176
return null;
177177
}
178-
}).when(shuffleBlockResolver).writeIndexFile(anyInt(), anyInt(), any(long[].class));
178+
}).when(shuffleBlockResolver).writeIndexFile(anyInt(), anyInt(), any(long[].class), 0L);
179179

180180
when(diskBlockManager.createTempShuffleBlock()).thenAnswer(
181181
new Answer<Tuple2<TempShuffleBlockId, File>>() {

0 commit comments

Comments
 (0)