Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,7 @@ public void execute(CommandLine options) throws Exception {
ParquetReader<SimpleRecord> reader = null;
try {
PrintWriter writer = new PrintWriter(Main.out, true);
reader = new ParquetReader<SimpleRecord>(new Path(input), new SimpleReadSupport());
reader = ParquetReader.builder(new SimpleReadSupport(), new Path(input)).build();
for (SimpleRecord value = reader.read(); value != null; value = reader.read()) {
if (options.hasOption('j')) {
value.prettyPrintJson(writer);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@
import org.apache.parquet.column.page.DictionaryPage;
import org.apache.parquet.column.page.PageReadStore;
import org.apache.parquet.column.page.PageReader;
import org.apache.parquet.format.converter.ParquetMetadataConverter;
import org.apache.parquet.hadoop.ParquetFileReader;
import org.apache.parquet.hadoop.metadata.BlockMetaData;
import org.apache.parquet.hadoop.metadata.ColumnChunkMetaData;
Expand All @@ -62,6 +63,8 @@

import com.google.common.base.Joiner;

import static org.apache.parquet.format.converter.ParquetMetadataConverter.NO_FILTER;

public class DumpCommand extends ArgsOnlyCommand {
private static final Charset UTF8 = Charset.forName("UTF-8");
private static final CharsetDecoder UTF8_DECODER = UTF8.newDecoder();
Expand Down Expand Up @@ -115,7 +118,7 @@ public void execute(CommandLine options) throws Exception {
Configuration conf = new Configuration();
Path inpath = new Path(input);

ParquetMetadata metaData = ParquetFileReader.readFooter(conf, inpath);
ParquetMetadata metaData = ParquetFileReader.readFooter(conf, inpath, NO_FILTER);
MessageType schema = metaData.getFileMetaData().getSchema();

PrettyPrintWriter out = PrettyPrintWriter.stdoutPrettyPrinter()
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -78,7 +78,7 @@ public void execute(CommandLine options) throws Exception {
ParquetReader<SimpleRecord> reader = null;
try {
PrintWriter writer = new PrintWriter(Main.out, true);
reader = new ParquetReader<SimpleRecord>(new Path(input), new SimpleReadSupport());
reader = ParquetReader.builder(new SimpleReadSupport(), new Path(input)).build();
for (SimpleRecord value = reader.read(); value != null && num-- > 0; value = reader.read()) {
value.prettyPrint(writer);
writer.println();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,8 @@
import org.apache.parquet.tools.util.MetadataUtils;
import org.apache.parquet.tools.util.PrettyPrintWriter;

import static org.apache.parquet.format.converter.ParquetMetadataConverter.NO_FILTER;

public class ShowSchemaCommand extends ArgsOnlyCommand {
public static final String[] USAGE = new String[] {
"<input>",
Expand Down Expand Up @@ -88,7 +90,7 @@ public void execute(CommandLine options) throws Exception {
} else {
file = path;
}
metaData = ParquetFileReader.readFooter(conf, file);
metaData = ParquetFileReader.readFooter(conf, file, NO_FILTER);
MessageType schema = metaData.getFileMetaData().getSchema();

Main.out.println(schema);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,9 @@
package org.apache.parquet.tools.read;

import com.google.common.collect.Maps;
import org.codehaus.jackson.node.BinaryNode;

import java.util.Arrays;
import java.util.Map;

public class SimpleMapRecord extends SimpleRecord {
Expand All @@ -30,14 +32,55 @@ protected Object toJsonObject() {
String key = null;
Object val = null;
for (NameValue kv : ((SimpleRecord) value.getValue()).values) {
if (kv.getName().equals("key")) {
key = (String) kv.getValue();
} else if (kv.getName().equals("value")) {
val = toJsonValue(kv.getValue());
String kvName = kv.getName();
Object kvValue = kv.getValue();
if (kvName.equals("key")) {
key = keyToString(kvValue);
} else if (kvName.equals("value")) {
val = toJsonValue(kvValue);
}
}
result.put(key, val);
}
return result;
}

String keyToString(Object kvValue) {
if (kvValue == null) {
return "null";
}

Class<?> type = kvValue.getClass();
if (type.isArray()) {
if (type.getComponentType() == boolean.class) {
return Arrays.toString((boolean[]) kvValue);
}
else if (type.getComponentType() == byte.class) {
return new BinaryNode((byte[]) kvValue).asText();
}
else if (type.getComponentType() == char.class) {
return Arrays.toString((char[]) kvValue);
}
else if (type.getComponentType() == double.class) {
return Arrays.toString((double[]) kvValue);
}
else if (type.getComponentType() == float.class) {
return Arrays.toString((float[]) kvValue);
}
else if (type.getComponentType() == int.class) {
return Arrays.toString((int[]) kvValue);
}
else if (type.getComponentType() == long.class) {
return Arrays.toString((long[]) kvValue);
}
else if (type.getComponentType() == short.class) {
return Arrays.toString((short[]) kvValue);
}
else {
return Arrays.toString((Object[]) kvValue);
}
} else {
return String.valueOf(kvValue);
}
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,56 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.parquet.tools.read;
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Looks like you forgot to add a license header to this file and that's causing Travis CI to fail.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

added



import org.junit.Assert;
import org.junit.Test;

public class TestSimpleMapRecord {

class TestRecord {
private int x;
private int y;

public TestRecord(int x, int y) {
this.x = x;
this.y = y;
}

@Override
public String toString() {
return "TestRecord {" + x + "," + y + "}";
}
}

@Test
public void testBinary() {
SimpleMapRecord r = new SimpleMapRecord();
Assert.assertEquals("null", r.keyToString(null));
Assert.assertEquals("[true, false, true]", r.keyToString(new boolean[]{true, false, true}));
Assert.assertEquals("[a, z]", r.keyToString(new char[] { 'a', 'z' }));
Assert.assertEquals("[1.0, 3.0]", r.keyToString(new double[]{1.0, 3.0 }));
Assert.assertEquals("[2.0, 4.0]", r.keyToString(new float[]{2.0f, 4.0f }));
Assert.assertEquals("[100, 999]", r.keyToString(new int[]{100, 999 }));
Assert.assertEquals("[23, 37]", r.keyToString(new long[] { 23l, 37l }));
Assert.assertEquals("[-1, -2]", r.keyToString(new short[]{(short) -1, (short) -2}));
Assert.assertEquals("dGVzdA==", r.keyToString("test".getBytes()));
Assert.assertEquals("TestRecord {222,333}", r.keyToString(new TestRecord(222, 333)));
}
}