From 615aa5a4450dfa719cce0747dbce18005727d2fd Mon Sep 17 00:00:00 2001 From: proflin Date: Sun, 17 Jan 2016 17:15:57 +0800 Subject: [PATCH 1/5] PARQUET-431 --- .../java/org/apache/parquet/hadoop/ParquetOutputFormat.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetOutputFormat.java b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetOutputFormat.java index 31cc96ba10..29b13c670b 100644 --- a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetOutputFormat.java +++ b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetOutputFormat.java @@ -445,7 +445,7 @@ public OutputCommitter getOutputCommitter(TaskAttemptContext context) /** * This memory manager is for all the real writers (InternalParquetRecordWriter) in one task. */ - private static MemoryManager memoryManager; + private static volatile MemoryManager memoryManager; public static MemoryManager getMemoryManager() { return memoryManager; From fd1df4ed712185cd9b564acb59ce9c011478579c Mon Sep 17 00:00:00 2001 From: Liwei Lin Date: Tue, 2 Feb 2016 16:19:24 +0800 Subject: [PATCH 2/5] Adds synchronization around the creation of memoryManager as well as getMemoryManager() Adds synchronization around the creation of memoryManager as well as getMemoryManager() --- .../parquet/hadoop/ParquetOutputFormat.java | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetOutputFormat.java b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetOutputFormat.java index 29b13c670b..83fb235226 100644 --- a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetOutputFormat.java +++ b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetOutputFormat.java @@ -394,9 +394,12 @@ public RecordWriter getRecordWriter(Configuration conf, Path file, Comp MemoryManager.DEFAULT_MEMORY_POOL_RATIO); long minAllocation = conf.getLong(ParquetOutputFormat.MIN_MEMORY_ALLOCATION, MemoryManager.DEFAULT_MIN_MEMORY_ALLOCATION); - if (memoryManager == null) { - memoryManager = new MemoryManager(maxLoad, minAllocation); - } else if (memoryManager.getMemoryPoolRatio() != maxLoad) { + synchronized (ParquetOutputFormat.class) { + if (memoryManager == null) { + memoryManager = new MemoryManager(maxLoad, minAllocation); + } + } + if (memoryManager.getMemoryPoolRatio() != maxLoad) { LOG.warn("The configuration " + MEMORY_POOL_RATIO + " has been set. It should not " + "be reset by the new value: " + maxLoad); } @@ -447,7 +450,11 @@ public OutputCommitter getOutputCommitter(TaskAttemptContext context) */ private static volatile MemoryManager memoryManager; - public static MemoryManager getMemoryManager() { + /** + * This method is intended to ONLY be used in test codes + * */ + @Deprecated + public synchronized static MemoryManager getMemoryManager() { return memoryManager; } } From d54bb99e98694e5a2b5209687ef59ec10913aaf0 Mon Sep 17 00:00:00 2001 From: Liwei Lin Date: Tue, 2 Feb 2016 16:34:55 +0800 Subject: [PATCH 3/5] Undo the Deprecated anotation --- .../java/org/apache/parquet/hadoop/ParquetOutputFormat.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetOutputFormat.java b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetOutputFormat.java index 83fb235226..cf4061ef11 100644 --- a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetOutputFormat.java +++ b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetOutputFormat.java @@ -450,10 +450,6 @@ public OutputCommitter getOutputCommitter(TaskAttemptContext context) */ private static volatile MemoryManager memoryManager; - /** - * This method is intended to ONLY be used in test codes - * */ - @Deprecated public synchronized static MemoryManager getMemoryManager() { return memoryManager; } From 5e94fa3ff073951d1906999dd39d88ddc6dba955 Mon Sep 17 00:00:00 2001 From: Liwei Lin Date: Wed, 3 Feb 2016 09:38:10 +0800 Subject: [PATCH 4/5] Remove the volatile modifier for memoryManager --- .../java/org/apache/parquet/hadoop/ParquetOutputFormat.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetOutputFormat.java b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetOutputFormat.java index cf4061ef11..6b1feb903b 100644 --- a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetOutputFormat.java +++ b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetOutputFormat.java @@ -448,7 +448,7 @@ public OutputCommitter getOutputCommitter(TaskAttemptContext context) /** * This memory manager is for all the real writers (InternalParquetRecordWriter) in one task. */ - private static volatile MemoryManager memoryManager; + private static MemoryManager memoryManager; public synchronized static MemoryManager getMemoryManager() { return memoryManager; From 1aa4a441b58adf9abf3feee2602e7a208fef0079 Mon Sep 17 00:00:00 2001 From: Liwei Lin Date: Wed, 3 Feb 2016 14:55:36 +0800 Subject: [PATCH 5/5] empty commit to trigger CI --- .../main/java/org/apache/parquet/hadoop/ParquetOutputFormat.java | 1 - 1 file changed, 1 deletion(-) diff --git a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetOutputFormat.java b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetOutputFormat.java index 6b1feb903b..6accce17c5 100644 --- a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetOutputFormat.java +++ b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetOutputFormat.java @@ -444,7 +444,6 @@ public OutputCommitter getOutputCommitter(TaskAttemptContext context) return committer; } - /** * This memory manager is for all the real writers (InternalParquetRecordWriter) in one task. */