Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
59 commits
Select commit Hold shift + click to select a range
839b458
Merge remote-tracking branch 'refs/remotes/apache/master'
lw-lin Feb 1, 2016
bb4283a
Merge branch 'master' of https://github.com/proflin/parquet-mr
lw-lin Feb 1, 2016
0610e8f
Preview of replacing Log.java with slf4j
lw-lin Feb 2, 2016
5c77437
Remove the static import of BaseRecordReader
lw-lin Feb 2, 2016
a0d2f03
03~10
lw-lin Feb 2, 2016
dfb87aa
11~15
lw-lin Feb 2, 2016
5df4d16
16~20
lw-lin Feb 2, 2016
1430dae
21~25
lw-lin Feb 2, 2016
d14b3f8
26~30
lw-lin Feb 2, 2016
0116fc2
31~35
lw-lin Feb 2, 2016
b50426b
Revert "03~10"
lw-lin Feb 3, 2016
c98b22b
Revert "11~15"
lw-lin Feb 3, 2016
6640a7d
Revert "16~20"
lw-lin Feb 3, 2016
f40c091
Revert "21~25"
lw-lin Feb 3, 2016
0133d05
Revert "26~30"
lw-lin Feb 3, 2016
22e93a4
Revert "31~35"
lw-lin Feb 3, 2016
f22c5ef
Revert "Remove the static import of BaseRecordReader"
lw-lin Feb 3, 2016
0473687
Revert "Preview of replacing Log.java with slf4j"
lw-lin Feb 3, 2016
da95567
Updates
lw-lin Jan 13, 2016
9b43f9a
replace Log.java with slf4j for module [parquet.common]
lw-lin Feb 3, 2016
d22d1af
replace Log.java with slf4j for module [parquet.common]
lw-lin Feb 3, 2016
9b152ff
replace Log.java with slf4j for module [parquet-encoding]
lw-lin Feb 3, 2016
cd2861f
replace Log.java with slf4j for module [parquet-column] part 1
lw-lin Feb 3, 2016
6335d76
replace Log.java with slf4j for module [parquet-column] part 2
lw-lin Feb 3, 2016
3a99a6b
replace Log.java with slf4j for module [parquet-column] part 3
lw-lin Feb 3, 2016
af36699
replace Log.java with slf4j for module [parquet-column] part 4
lw-lin Feb 3, 2016
7c54e90
replace Log.java with slf4j for module [parquet-column] part 5
lw-lin Feb 3, 2016
bf9e0f6
replace Log.java with slf4j for module [parquet-column] part 6
lw-lin Feb 3, 2016
69079f5
replace Log.java with slf4j for module [parquet-hadoop] part 1
lw-lin Feb 3, 2016
01402bb
replace Log.java with slf4j for module [parquet-hadoop] part 2
lw-lin Feb 3, 2016
fd40acc
replace Log.java with slf4j for module [parquet-hadoop] part 3
lw-lin Feb 3, 2016
dd71021
replace Log.java with slf4j for module [parquet-avro]
lw-lin Feb 3, 2016
21c5994
replace Log.java with slf4j for module [parquet-column] part test
lw-lin Feb 3, 2016
c0058d3
replace Log.java with slf4j for module [parquet-encoding] part test
lw-lin Feb 3, 2016
8f50e96
replace Log.java with slf4j for module [parquet-hadoop] part tes
lw-lin Feb 3, 2016
113fec5
replace Log.java with slf4j for module [parquet-pig]
lw-lin Feb 3, 2016
8175eba
replace Log.java with slf4j for module [parquet-protobuf]
lw-lin Feb 3, 2016
46be52d
replace Log.java with slf4j for module [parquet-thrift] part 1
lw-lin Feb 3, 2016
83665af
replace Log.java with slf4j for module [parquet-thrift] part 2
lw-lin Feb 3, 2016
44f42fc
replace Log.java with slf4j for module [parquet-hive]
lw-lin Feb 3, 2016
2927861
remove unused import of Log.java
lw-lin Feb 3, 2016
f0b18fb
replace LOG.DEBUG with LOGGER.isDebugEnabled() for RecordReaderImplem…
lw-lin Feb 3, 2016
ee5466d
Remove another 2 unused import of Log.java
lw-lin Feb 3, 2016
f781e40
Add LOGGER and replace Log.DEBUG with LOGGER.isDebugEnabled()
lw-lin Feb 3, 2016
ff86f17
Revert "Updates"
lw-lin Feb 3, 2016
f9a80f7
Merge remote-tracking branch 'refs/remotes/apache/master' into PARQUE…
lw-lin Feb 3, 2016
1bd0d39
empty commit to trigger CI
lw-lin Feb 3, 2016
40f41ed
Merge remote-tracking branch 'refs/remotes/apache/master' into PARQUE…
lw-lin Mar 8, 2016
a9f50b6
Add a static final field to optimize out LOGGER.isXXXEnabled() call
lw-lin Mar 11, 2016
d4d9bb7
Merge remote-tracking branch 'refs/remotes/apache/master' into PARQUE…
lw-lin Mar 11, 2016
ade4f52
Replace Log with slf.Logger for DictionaryFilter
lw-lin Mar 11, 2016
fa51e9c
Add a static final field to optimize out LOGGER.isXXXEnabled() call
lw-lin Mar 11, 2016
bf8ed47
Merge or remove some if (LOGGER.isXXXEnabled())
lw-lin Mar 11, 2016
10d6e32
Fix a minor compilation issue
lw-lin Mar 11, 2016
a718d67
Add static final fields to optimize out LOGGER.isXXXEnabled() call
lw-lin Mar 31, 2016
f4a1ea4
Merge remote-tracking branch 'apache/master' into PARQUET-401--Deprec…
lw-lin Apr 17, 2016
f97fe4c
Merge remote-tracking branch 'refs/remotes/apache/master' into p401
lw-lin Aug 24, 2016
0cc0eff
Fix LOGGER
lw-lin Aug 30, 2016
199398c
Fix Log
lw-lin Aug 30, 2016
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -36,14 +36,16 @@
import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
import org.junit.Test;
import org.apache.parquet.Log;

import static java.lang.Thread.sleep;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNull;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

public class TestInputOutputFormat {
private static final Log LOG = Log.getLog(TestInputOutputFormat.class);
private static final Logger LOGGER = LoggerFactory.getLogger(TestInputOutputFormat.class);

private static Schema avroSchema;
static {
Expand Down Expand Up @@ -132,10 +134,10 @@ public void testReadWrite() throws Exception {
private void waitForJob(Job job) throws Exception {
job.submit();
while (!job.isComplete()) {
LOG.debug("waiting for job " + job.getJobName());
LOGGER.debug("waiting for job {}", job.getJobName());
sleep(100);
}
LOG.info("status for job " + job.getJobName() + ": " + (job.isSuccessful() ? "SUCCESS" : "FAILURE"));
LOGGER.info("status for job {}: {}", job.getJobName(), job.isSuccessful() ? "SUCCESS" : "FAILURE");
if (!job.isSuccessful()) {
throw new RuntimeException("job failed " + job.getJobName());
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,6 @@
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
import org.apache.parquet.Log;
import org.apache.parquet.column.ColumnReader;
import org.apache.parquet.filter.ColumnPredicates;
import org.apache.parquet.filter.ColumnRecordFilter;
Expand All @@ -54,9 +53,11 @@
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;

public class TestReflectInputOutputFormat {
private static final Log LOG = Log.getLog(TestReflectInputOutputFormat.class);
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

public class TestReflectInputOutputFormat {
private static final Logger LOGGER = LoggerFactory.getLogger(TestReflectInputOutputFormat.class);

public static class Service {
private long date;
Expand Down Expand Up @@ -477,10 +478,10 @@ public void testReadWriteChangedCar() throws Exception {
private void waitForJob(Job job) throws Exception {
job.submit();
while (!job.isComplete()) {
LOG.debug("waiting for job " + job.getJobName());
LOGGER.debug("waiting for job {}", job.getJobName());
sleep(100);
}
LOG.info("status for job " + job.getJobName() + ": " + (job.isSuccessful() ? "SUCCESS" : "FAILURE"));
LOGGER.info("status for job {}: {}", job.getJobName(), job.isSuccessful() ? "SUCCESS" : "FAILURE");
if (!job.isSuccessful()) {
throw new RuntimeException("job failed " + job.getJobName());
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -39,15 +39,17 @@
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import org.apache.parquet.Log;
import org.apache.parquet.column.ColumnReader;
import org.apache.parquet.filter.ColumnPredicates;
import org.apache.parquet.filter.ColumnRecordFilter;
import org.apache.parquet.filter.RecordFilter;
import org.apache.parquet.filter.UnboundRecordFilter;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

public class TestSpecificInputOutputFormat {
private static final Log LOG = Log.getLog(TestSpecificInputOutputFormat.class);
private static final Logger LOGGER = LoggerFactory.getLogger(TestSpecificInputOutputFormat.class);

public static Car nextRecord(int i) {
String vin = "1VXBR12EXCP000000";
Expand Down Expand Up @@ -268,10 +270,10 @@ public void testReadWriteChangedCar() throws Exception {
private void waitForJob(Job job) throws Exception {
job.submit();
while (!job.isComplete()) {
LOG.debug("waiting for job " + job.getJobName());
LOGGER.debug("waiting for job {}", job.getJobName());
sleep(100);
}
LOG.info("status for job " + job.getJobName() + ": " + (job.isSuccessful() ? "SUCCESS" : "FAILURE"));
LOGGER.info("status for job {}: {}", job.getJobName(), job.isSuccessful() ? "SUCCESS" : "FAILURE");
if (!job.isSuccessful()) {
throw new RuntimeException("job failed " + job.getJobName());
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,8 +21,11 @@
import org.apache.parquet.VersionParser.ParsedVersion;
import org.apache.parquet.column.Encoding;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

public class CorruptDeltaByteArrays {
private static final Log LOG = Log.getLog(CorruptStatistics.class);
private static final Logger LOGGER = LoggerFactory.getLogger(CorruptStatistics.class);

private static final SemanticVersion PARQUET_246_FIXED_VERSION =
new SemanticVersion(1, 8, 0);
Expand All @@ -42,8 +45,8 @@ public static boolean requiresSequentialReads(ParsedVersion version, Encoding en
}

if (!version.hasSemanticVersion()) {
LOG.warn("Requiring sequential reads because created_by did not " +
"contain a valid version (see PARQUET-246): " + version.version);
LOGGER.warn("Requiring sequential reads because created_by did not " +
"contain a valid version (see PARQUET-246): {}", version.version);
return true;
}

Expand All @@ -60,8 +63,8 @@ public static boolean requiresSequentialReads(SemanticVersion semver, Encoding e
}

if (semver.compareTo(PARQUET_246_FIXED_VERSION) < 0) {
LOG.info("Requiring sequential reads because this file was created " +
"prior to " + PARQUET_246_FIXED_VERSION + ". See PARQUET-246" );
LOGGER.info("Requiring sequential reads because this file was created " +
"prior to {}. See PARQUET-246", PARQUET_246_FIXED_VERSION);
return true;
}

Expand All @@ -75,7 +78,7 @@ public static boolean requiresSequentialReads(String createdBy, Encoding encodin
}

if (Strings.isNullOrEmpty(createdBy)) {
LOG.info("Requiring sequential reads because file version is empty. " +
LOGGER.info("Requiring sequential reads because file version is empty. " +
"See PARQUET-246");
return true;
}
Expand All @@ -93,7 +96,7 @@ public static boolean requiresSequentialReads(String createdBy, Encoding encodin
}

private static void warnParseError(String createdBy, Throwable e) {
LOG.warn("Requiring sequential reads because created_by could not be " +
"parsed (see PARQUET-246): " + createdBy, e);
LOGGER.warn("Requiring sequential reads because created_by could not be " +
"parsed (see PARQUET-246): {}", createdBy, e);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,9 @@
import org.apache.parquet.VersionParser.VersionParseException;
import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

/**
* There was a bug (PARQUET-251) that caused the statistics metadata
* for binary columns to be corrupted in the write path.
Expand All @@ -35,7 +38,7 @@
public class CorruptStatistics {
private static final AtomicBoolean alreadyLogged = new AtomicBoolean(false);

private static final Log LOG = Log.getLog(CorruptStatistics.class);
private static final Logger LOGGER = LoggerFactory.getLogger(CorruptStatistics.class);

// the version in which the bug described by jira: PARQUET-251 was fixed
// the bug involved writing invalid binary statistics, so stats written prior to this
Expand Down Expand Up @@ -108,13 +111,13 @@ public static boolean shouldIgnoreStatistics(String createdBy, PrimitiveTypeName

private static void warnParseErrorOnce(String createdBy, Throwable e) {
if(!alreadyLogged.getAndSet(true)) {
LOG.warn("Ignoring statistics because created_by could not be parsed (see PARQUET-251): " + createdBy, e);
LOGGER.warn("Ignoring statistics because created_by could not be parsed (see PARQUET-251): {}", createdBy, e);
}
}

private static void warnOnce(String message) {
if(!alreadyLogged.getAndSet(true)) {
LOG.warn(message);
LOGGER.warn(message);
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,6 @@
package org.apache.parquet.column.impl;

import static java.lang.String.format;
import static org.apache.parquet.Log.DEBUG;
import static org.apache.parquet.Preconditions.checkNotNull;
import static org.apache.parquet.column.ValuesType.DEFINITION_LEVEL;
import static org.apache.parquet.column.ValuesType.REPETITION_LEVEL;
Expand All @@ -30,7 +29,6 @@
import java.nio.ByteBuffer;

import org.apache.parquet.CorruptDeltaByteArrays;
import org.apache.parquet.Log;
import org.apache.parquet.VersionParser.ParsedVersion;
import org.apache.parquet.bytes.BytesInput;
import org.apache.parquet.bytes.BytesUtils;
Expand All @@ -52,14 +50,17 @@
import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName;
import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeNameConverter;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

/**
* ColumnReader implementation
*
* @author Julien Le Dem
*
*/
public class ColumnReaderImpl implements ColumnReader {
private static final Log LOG = Log.getLog(ColumnReaderImpl.class);
private static final Logger LOGGER = LoggerFactory.getLogger(ColumnReaderImpl.class);

/**
* binds the lower level page decoder to the record converter materializing the records
Expand Down Expand Up @@ -523,7 +524,7 @@ private void readRepetitionAndDefinitionLevels() {
private void checkRead() {
if (isPageFullyConsumed()) {
if (isFullyConsumed()) {
if (DEBUG) LOG.debug("end reached");
LOGGER.debug("end reached");
repetitionLevel = 0; // the next repetition level
return;
}
Expand All @@ -533,7 +534,7 @@ private void checkRead() {
}

private void readPage() {
if (DEBUG) LOG.debug("loading page");
LOGGER.debug("loading page");
DataPage page = pageReader.readPage();
page.accept(new DataPage.Visitor<Void>() {
@Override
Expand Down Expand Up @@ -590,14 +591,14 @@ private void readPageV1(DataPageV1 page) {
this.definitionLevelColumn = new ValuesReaderIntIterator(dlReader);
try {
ByteBuffer bytes = page.getBytes().toByteBuffer();
if (DEBUG) LOG.debug("page size " + bytes.remaining() + " bytes and " + pageValueCount + " records");
if (DEBUG) LOG.debug("reading repetition levels at 0");
LOGGER.debug("page size {} bytes and {} records", bytes.remaining(), pageValueCount);
LOGGER.debug("reading repetition levels at 0");
rlReader.initFromPage(pageValueCount, bytes, 0);
int next = rlReader.getNextOffset();
if (DEBUG) LOG.debug("reading definition levels at " + next);
LOGGER.debug("reading definition levels at {}", next);
dlReader.initFromPage(pageValueCount, bytes, next);
next = dlReader.getNextOffset();
if (DEBUG) LOG.debug("reading data at " + next);
LOGGER.debug("reading data at {}", next);
initDataReader(page.getValueEncoding(), bytes, next, page.getValueCount());
} catch (IOException e) {
throw new ParquetDecodingException("could not read page " + page + " in col " + path, e);
Expand All @@ -608,7 +609,7 @@ private void readPageV2(DataPageV2 page) {
this.repetitionLevelColumn = newRLEIterator(path.getMaxRepetitionLevel(), page.getRepetitionLevels());
this.definitionLevelColumn = newRLEIterator(path.getMaxDefinitionLevel(), page.getDefinitionLevels());
try {
if (DEBUG) LOG.debug("page data size " + page.getData().size() + " bytes and " + pageValueCount + " records");
LOGGER.debug("page data size {} bytes and {} records", page.getData().size(), pageValueCount);
initDataReader(page.getDataEncoding(), page.getData().toByteBuffer(), 0, page.getValueCount());
} catch (IOException e) {
throw new ParquetDecodingException("could not read page " + page + " in col " + path, e);
Expand Down
Loading