Throw IOException when the text file line is over the maxLineLength l…

…imit
prestodb · Sep 10, 2018 · 56d2f8b · 56d2f8b
1 parent f417683
commit 56d2f8b
Show file tree

Hide file tree

Showing 2 changed files with 35 additions and 4 deletions.
diff --git a/pom.xml b/pom.xml
@@ -367,6 +367,24 @@
                 </configuration>
             </plugin>
 
+            <plugin>
+                <groupId>com.ning.maven.plugins</groupId>
+                <artifactId>maven-duplicate-finder-plugin</artifactId>
+                <version>1.0.4</version>
+                <executions>
+                    <execution>
+                        <id>default</id>
+                        <phase>verify</phase>
+                        <goals>
+                            <goal>check</goal>
+                        </goals>
+                    </execution>
+                </executions>
+                <configuration>
+                    <failBuildInCaseOfEqualContentConflict>true</failBuildInCaseOfEqualContentConflict>
+                </configuration>
+            </plugin>
+
             <plugin>
                 <groupId>com.ning.maven.plugins</groupId>
                 <artifactId>maven-dependency-versions-check-plugin</artifactId>
@@ -543,7 +561,6 @@
                                     <artifact>org.apache.hadoop:hadoop-common</artifact>
                                     <excludes>
                                         <exclude>*.h</exclude>
-                                        <exclude>org/apache/hadoop/util/LineReader.class</exclude>
                                     </excludes>
                                 </filter>
                                 <filter>

diff --git a/src/main/java/org/apache/hadoop/util/LineReader.java b/src/main/java/org/apache/hadoop/util/LineReader.java
@@ -244,6 +244,10 @@ private int readDefaultLine(Text str, int maxLineLength, int maxBytesToConsume)
             int appendLength = readLength - newlineLength;
             if (appendLength > maxLineLength - txtLength) {
                 appendLength = maxLineLength - txtLength;
+                if (appendLength > 0) {
+                    // We want to fail the query when the line length is over the limit.
+                    throw new IOException("Too many bytes before newline:  " + maxLineLength);
+                }
             }
             if (appendLength > 0) {
                 int newTxtLength = txtLength + appendLength;
@@ -257,8 +261,10 @@ private int readDefaultLine(Text str, int maxLineLength, int maxBytesToConsume)
             }
         } while (newlineLength == 0 && bytesConsumed < maxBytesToConsume);
 
-        if (bytesConsumed > Integer.MAX_VALUE) {
-
+        if (newlineLength == 0 && bytesConsumed >= maxBytesToConsume) {
+            // It is possible that bytesConsumed is over the maxBytesToConsume but we
+            // didn't append anything to str.buffer. If we have consumed over maxBytesToConsume
+            // bytes but still haven't seen a line terminator, we will fail the query.
             throw new IOException("Too many bytes before newline: " + bytesConsumed);
         }
         return (int)bytesConsumed;
@@ -343,6 +349,10 @@ private int readCustomLine(Text str, int maxLineLength, int maxBytesToConsume)
             int appendLength = readLength - delPosn;
             if (appendLength > maxLineLength - txtLength) {
                 appendLength = maxLineLength - txtLength;
+                if (appendLength > 0) {
+                    // We want to fail the query when the line length is over the limit.
+                    throw new IOException("Too many bytes before newline:  " + maxLineLength);
+                }
             }
             bytesConsumed += ambiguousByteCount;
             if (appendLength >= 0 && ambiguousByteCount > 0) {
@@ -372,7 +382,11 @@ private int readCustomLine(Text str, int maxLineLength, int maxBytesToConsume)
             }
         } while (delPosn < recordDelimiterBytes.length
                 && bytesConsumed < maxBytesToConsume);
-        if (bytesConsumed > Integer.MAX_VALUE) {
+        if (delPosn < recordDelimiterBytes.length
+                && bytesConsumed >= maxBytesToConsume) {
+            // It is possible that bytesConsumed is over the maxBytesToConsume but we
+            // didn't append anything to str.buffer. If we have consumed over maxBytesToConsume
+            // bytes but still haven't seen a line terminator, we will fail the query.
             throw new IOException("Too many bytes before delimiter: " + bytesConsumed);
         }
         return (int) bytesConsumed;