From f52d851d9da9b3fd5716e46fd9a2dc2f6e5ae713 Mon Sep 17 00:00:00 2001 From: Xianjin YE Date: Fri, 11 Dec 2020 17:52:41 +0800 Subject: [PATCH] SPARK-33756: make BytesToBytesMap's MapIterator idempotent --- .../org/apache/spark/unsafe/map/BytesToBytesMap.java | 10 ++++++---- .../spark/unsafe/map/AbstractBytesToBytesMapSuite.java | 2 ++ 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/core/src/main/java/org/apache/spark/unsafe/map/BytesToBytesMap.java b/core/src/main/java/org/apache/spark/unsafe/map/BytesToBytesMap.java index d7940fc08e1a5..f474c30b8b3d8 100644 --- a/core/src/main/java/org/apache/spark/unsafe/map/BytesToBytesMap.java +++ b/core/src/main/java/org/apache/spark/unsafe/map/BytesToBytesMap.java @@ -393,10 +393,12 @@ public void remove() { } private void handleFailedDelete() { - // remove the spill file from disk - File file = spillWriters.removeFirst().getFile(); - if (file != null && file.exists() && !file.delete()) { - logger.error("Was unable to delete spill file {}", file.getAbsolutePath()); + if (spillWriters.size() > 0) { + // remove the spill file from disk + File file = spillWriters.removeFirst().getFile(); + if (file != null && file.exists() && !file.delete()) { + logger.error("Was unable to delete spill file {}", file.getAbsolutePath()); + } } } } diff --git a/core/src/test/java/org/apache/spark/unsafe/map/AbstractBytesToBytesMapSuite.java b/core/src/test/java/org/apache/spark/unsafe/map/AbstractBytesToBytesMapSuite.java index f4e952f465e54..f35176a69d94b 100644 --- a/core/src/test/java/org/apache/spark/unsafe/map/AbstractBytesToBytesMapSuite.java +++ b/core/src/test/java/org/apache/spark/unsafe/map/AbstractBytesToBytesMapSuite.java @@ -576,6 +576,8 @@ public void spillInIterator() throws IOException { iter2.next(); } assertFalse(iter2.hasNext()); + // calls hasNext twice deliberately, make sure it's idempotent + assertFalse(iter2.hasNext()); } finally { map.free(); for (File spillFile : spillFilesCreated) {