Skip to content

Commit 5bed26f

Browse files
author
Gary Gregory
committed
Improved performance of IOUtils.contentEquals(InputStream, InputStream).
This is based on the PR #118 by XenoAmess but only for this one method.
1 parent b6bca11 commit 5bed26f

File tree

6 files changed

+781
-13
lines changed

6 files changed

+781
-13
lines changed

‎src/changes/changes.xml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -194,6 +194,9 @@ The <action> type attribute can be add,update,fix,remove.
194194
<action dev="ggregory" type="update" due-to="Dependabot">
195195
Bump jimfs from 1.1 to 1.2 #183.
196196
</action>
197+
<action dev="ggregory" type="update" due-to="XenoAmess, Gary Gregory">
198+
Improved performance of IOUtils.contentEquals(InputStream, InputStream).
199+
</action>
197200
</release>
198201
<!-- The release date is the date RC is cut -->
199202
<release version="2.8.0" date="2020-09-05" description="Java 8 required.">

‎src/main/java/org/apache/commons/io/IOUtils.java‎

Lines changed: 36 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -744,26 +744,49 @@ public static long consume(final InputStream input)
744744
* @throws NullPointerException if either input is null
745745
* @throws IOException if an I/O error occurs
746746
*/
747-
@SuppressWarnings("resource")
748-
public static boolean contentEquals(final InputStream input1, final InputStream input2)
749-
throws IOException {
747+
public static boolean contentEquals(final InputStream input1, final InputStream input2) throws IOException {
748+
// Before making any changes, please test with
749+
// org.apache.commons.io.jmh.IOUtilsContentEqualsInputStreamsBenchmark
750750
if (input1 == input2) {
751751
return true;
752752
}
753-
if (input1 == null ^ input2 == null) {
753+
if (input1 == null || input2 == null) {
754754
return false;
755755
}
756-
final BufferedInputStream bufferedInput1 = buffer(input1);
757-
final BufferedInputStream bufferedInput2 = buffer(input2);
758-
int ch = bufferedInput1.read();
759-
while (EOF != ch) {
760-
final int ch2 = bufferedInput2.read();
761-
if (ch != ch2) {
762-
return false;
756+
757+
final byte[] array1 = new byte[DEFAULT_BUFFER_SIZE];
758+
final byte[] array2 = new byte[DEFAULT_BUFFER_SIZE];
759+
int pos1;
760+
int pos2;
761+
int count1;
762+
int count2;
763+
while (true) {
764+
pos1 = 0;
765+
pos2 = 0;
766+
for (int index = 0; index < DEFAULT_BUFFER_SIZE; index++) {
767+
if (pos1 == index) {
768+
do {
769+
count1 = input1.read(array1, pos1, DEFAULT_BUFFER_SIZE - pos1);
770+
} while (count1 == 0);
771+
if (count1 == EOF) {
772+
return pos2 == index && input2.read() == EOF;
773+
}
774+
pos1 += count1;
775+
}
776+
if (pos2 == index) {
777+
do {
778+
count2 = input2.read(array2, pos2, DEFAULT_BUFFER_SIZE - pos2);
779+
} while (count2 == 0);
780+
if (count2 == EOF) {
781+
return pos1 == index && input1.read() == EOF;
782+
}
783+
pos2 += count2;
784+
}
785+
if (array1[index] != array2[index]) {
786+
return false;
787+
}
763788
}
764-
ch = bufferedInput1.read();
765789
}
766-
return bufferedInput2.read() == EOF;
767790
}
768791

769792
/**

‎src/test/java/org/apache/commons/io/IOUtilsTestCase.java‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -548,6 +548,14 @@ public void testContentEquals_InputStream_InputStream() throws Exception {
548548
new ByteArrayInputStream(bytes2XDefaultA2)));
549549
assertTrue(IOUtils.contentEquals(new ByteArrayInputStream(bytes2XDefaultA),
550550
new ByteArrayInputStream(bytes2XDefaultA)));
551+
// FileInputStream a bit more than 16 k.
552+
try (
553+
final FileInputStream input1 = new FileInputStream(
554+
"src/test/resources/org/apache/commons/io/abitmorethan16k.txt");
555+
final FileInputStream input2 = new FileInputStream(
556+
"src/test/resources/org/apache/commons/io/abitmorethan16kcopy.txt")) {
557+
assertTrue(IOUtils.contentEquals(input1, input1));
558+
}
551559
}
552560

553561
@Test

‎src/test/java/org/apache/commons/io/jmh/IOUtilsContentEqualsInputStreamsBenchmark.java‎

Lines changed: 238 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,238 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright ownership.
5+
* The ASF licenses this file to You under the Apache License, Version 2.0
6+
* (the "License"); you may not use this file except in compliance with
7+
* the License. You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
18+
package org.apache.commons.io.jmh;
19+
20+
import static org.apache.commons.io.IOUtils.DEFAULT_BUFFER_SIZE;
21+
import static org.apache.commons.io.IOUtils.EOF;
22+
import static org.apache.commons.io.IOUtils.buffer;
23+
24+
import java.io.BufferedInputStream;
25+
import java.io.IOException;
26+
import java.io.InputStream;
27+
import java.nio.charset.Charset;
28+
import java.util.concurrent.TimeUnit;
29+
30+
import org.apache.commons.io.IOUtils;
31+
import org.apache.commons.lang3.StringUtils;
32+
import org.openjdk.jmh.annotations.Benchmark;
33+
import org.openjdk.jmh.annotations.BenchmarkMode;
34+
import org.openjdk.jmh.annotations.Fork;
35+
import org.openjdk.jmh.annotations.Measurement;
36+
import org.openjdk.jmh.annotations.Mode;
37+
import org.openjdk.jmh.annotations.OutputTimeUnit;
38+
import org.openjdk.jmh.annotations.Scope;
39+
import org.openjdk.jmh.annotations.State;
40+
import org.openjdk.jmh.annotations.Warmup;
41+
import org.openjdk.jmh.infra.Blackhole;
42+
43+
/**
44+
* Test different implementations of {@link IOUtils#contentEquals(InputStream, InputStream)}.
45+
*
46+
* <pre>
47+
* Benchmark Mode Cnt Score Error Units
48+
* IOUtilsContentEqualsInputStreamsBenchmark.testFileCurrent avgt 5 1518342.821 ▒ 201890.705 ns/op
49+
* IOUtilsContentEqualsInputStreamsBenchmark.testFilePr118 avgt 5 1578606.938 ▒ 66980.718 ns/op
50+
* IOUtilsContentEqualsInputStreamsBenchmark.testFileRelease_2_8_0 avgt 5 2439163.068 ▒ 265765.294 ns/op
51+
* IOUtilsContentEqualsInputStreamsBenchmark.testStringCurrent avgt 5 10389834700.000 ▒ 330301175.219 ns/op
52+
* IOUtilsContentEqualsInputStreamsBenchmark.testStringPr118 avgt 5 10890915400.000 ▒ 3251289634.067 ns/op
53+
* IOUtilsContentEqualsInputStreamsBenchmark.testStringRelease_2_8_0 avgt 5 12522802960.000 ▒ 111147669.527 ns/op
54+
* </pre>
55+
*/
56+
@BenchmarkMode(Mode.AverageTime)
57+
@OutputTimeUnit(TimeUnit.NANOSECONDS)
58+
@State(Scope.Thread)
59+
@Warmup(iterations = 5, time = 10, timeUnit = TimeUnit.SECONDS)
60+
@Measurement(iterations = 5, time = 10, timeUnit = TimeUnit.SECONDS)
61+
@Fork(value = 1, jvmArgs = {"-server"})
62+
public class IOUtilsContentEqualsInputStreamsBenchmark {
63+
64+
private static final String TEST_PATH_A = "/org/apache/commons/io/testfileBOM.xml";
65+
private static final String TEST_PATH_16K_A = "/org/apache/commons/io/abitmorethan16k.txt";
66+
private static final String TEST_PATH_16K_A_COPY = "/org/apache/commons/io/abitmorethan16kcopy.txt";
67+
private static final String TEST_PATH_B = "/org/apache/commons/io/testfileNoBOM.xml";
68+
private static final Charset DEFAULT_CHARSET = Charset.defaultCharset();
69+
static String[] STRINGS = new String[5];
70+
71+
static {
72+
STRINGS[0] = StringUtils.repeat("ab", 1 << 24);
73+
STRINGS[1] = STRINGS[0] + 'c';
74+
STRINGS[2] = STRINGS[0] + 'd';
75+
STRINGS[3] = StringUtils.repeat("ab\rab\n", 1 << 24);
76+
STRINGS[4] = StringUtils.repeat("ab\r\nab\r", 1 << 24);
77+
}
78+
79+
static String SPECIAL_CASE_STRING_0 = StringUtils.repeat(StringUtils.repeat("ab", 1 << 24) + '\n', 2);
80+
static String SPECIAL_CASE_STRING_1 = StringUtils.repeat(StringUtils.repeat("cd", 1 << 24) + '\n', 2);
81+
82+
@SuppressWarnings("resource")
83+
public static boolean contentEquals_release_2_8_0(final InputStream input1, final InputStream input2)
84+
throws IOException {
85+
if (input1 == input2) {
86+
return true;
87+
}
88+
if (input1 == null ^ input2 == null) {
89+
return false;
90+
}
91+
final BufferedInputStream bufferedInput1 = buffer(input1);
92+
final BufferedInputStream bufferedInput2 = buffer(input2);
93+
int ch = bufferedInput1.read();
94+
while (EOF != ch) {
95+
final int ch2 = bufferedInput2.read();
96+
if (ch != ch2) {
97+
return false;
98+
}
99+
ch = bufferedInput1.read();
100+
}
101+
return bufferedInput2.read() == EOF;
102+
103+
}
104+
105+
public static boolean contentEqualsPr118(final InputStream input1, final InputStream input2) throws IOException {
106+
if (input1 == input2) {
107+
return true;
108+
}
109+
if (input1 == null || input2 == null) {
110+
return false;
111+
}
112+
113+
final byte[] array1 = new byte[DEFAULT_BUFFER_SIZE];
114+
final byte[] array2 = new byte[DEFAULT_BUFFER_SIZE];
115+
int pos1;
116+
int pos2;
117+
int count1;
118+
int count2;
119+
while (true) {
120+
pos1 = 0;
121+
pos2 = 0;
122+
for (int index = 0; index < DEFAULT_BUFFER_SIZE; index++) {
123+
if (pos1 == index) {
124+
do {
125+
count1 = input1.read(array1, pos1, DEFAULT_BUFFER_SIZE - pos1);
126+
} while (count1 == 0);
127+
if (count1 == EOF) {
128+
return pos2 == index && input2.read() == EOF;
129+
}
130+
pos1 += count1;
131+
}
132+
if (pos2 == index) {
133+
do {
134+
count2 = input2.read(array2, pos2, DEFAULT_BUFFER_SIZE - pos2);
135+
} while (count2 == 0);
136+
if (count2 == EOF) {
137+
return pos1 == index && input1.read() == EOF;
138+
}
139+
pos2 += count2;
140+
}
141+
if (array1[index] != array2[index]) {
142+
return false;
143+
}
144+
}
145+
}
146+
}
147+
148+
@Benchmark
149+
public boolean[] testFileCurrent() throws IOException {
150+
final boolean[] res = new boolean[3];
151+
try (InputStream inputStream1 = this.getClass().getResourceAsStream(TEST_PATH_A);
152+
InputStream inputStream2 = this.getClass().getResourceAsStream(TEST_PATH_B)) {
153+
res[0] = IOUtils.contentEquals(inputStream1, inputStream1);
154+
}
155+
try (InputStream inputStream1 = this.getClass().getResourceAsStream(TEST_PATH_A);
156+
InputStream inputStream2 = this.getClass().getResourceAsStream(TEST_PATH_A);) {
157+
res[1] = IOUtils.contentEquals(inputStream1, inputStream2);
158+
}
159+
try (InputStream inputStream1 = this.getClass().getResourceAsStream(TEST_PATH_16K_A);
160+
InputStream inputStream2 = this.getClass().getResourceAsStream(TEST_PATH_16K_A_COPY);) {
161+
res[2] = IOUtils.contentEquals(inputStream1, inputStream2);
162+
}
163+
return res;
164+
}
165+
166+
@Benchmark
167+
public boolean[] testFilePr118() throws IOException {
168+
final boolean[] res = new boolean[3];
169+
try (InputStream inputStream1 = this.getClass().getResourceAsStream(TEST_PATH_A);
170+
InputStream inputStream2 = this.getClass().getResourceAsStream(TEST_PATH_B)) {
171+
res[0] = contentEqualsPr118(inputStream1, inputStream1);
172+
}
173+
try (InputStream inputStream1 = this.getClass().getResourceAsStream(TEST_PATH_A);
174+
InputStream inputStream2 = this.getClass().getResourceAsStream(TEST_PATH_A);) {
175+
res[1] = contentEqualsPr118(inputStream1, inputStream2);
176+
}
177+
try (InputStream inputStream1 = this.getClass().getResourceAsStream(TEST_PATH_16K_A);
178+
InputStream inputStream2 = this.getClass().getResourceAsStream(TEST_PATH_16K_A_COPY);) {
179+
res[2] = contentEqualsPr118(inputStream1, inputStream2);
180+
}
181+
return res;
182+
}
183+
184+
@Benchmark
185+
public boolean[] testFileRelease_2_8_0() throws IOException {
186+
final boolean[] res = new boolean[3];
187+
try (InputStream inputStream1 = this.getClass().getResourceAsStream(TEST_PATH_A);
188+
InputStream inputStream2 = this.getClass().getResourceAsStream(TEST_PATH_B)) {
189+
res[0] = contentEquals_release_2_8_0(inputStream1, inputStream1);
190+
}
191+
try (InputStream inputStream1 = this.getClass().getResourceAsStream(TEST_PATH_A);
192+
InputStream inputStream2 = this.getClass().getResourceAsStream(TEST_PATH_A);) {
193+
res[1] = contentEquals_release_2_8_0(inputStream1, inputStream2);
194+
}
195+
try (InputStream inputStream1 = this.getClass().getResourceAsStream(TEST_PATH_16K_A);
196+
InputStream inputStream2 = this.getClass().getResourceAsStream(TEST_PATH_16K_A_COPY);) {
197+
res[2] = contentEquals_release_2_8_0(inputStream1, inputStream2);
198+
}
199+
return res;
200+
}
201+
202+
@Benchmark
203+
public void testStringCurrent(final Blackhole blackhole) throws IOException {
204+
for (int i = 0; i < 5; i++) {
205+
for (int j = 0; j < 5; j++) {
206+
try (InputStream inputReader1 = IOUtils.toInputStream(STRINGS[i], DEFAULT_CHARSET);
207+
InputStream inputReader2 = IOUtils.toInputStream(STRINGS[j], DEFAULT_CHARSET)) {
208+
blackhole.consume(IOUtils.contentEquals(inputReader1, inputReader2));
209+
}
210+
}
211+
}
212+
}
213+
214+
@Benchmark
215+
public void testStringPr118(final Blackhole blackhole) throws IOException {
216+
for (int i = 0; i < 5; i++) {
217+
for (int j = 0; j < 5; j++) {
218+
try (InputStream inputReader1 = IOUtils.toInputStream(STRINGS[i], DEFAULT_CHARSET);
219+
InputStream inputReader2 = IOUtils.toInputStream(STRINGS[j], DEFAULT_CHARSET)) {
220+
blackhole.consume(contentEqualsPr118(inputReader1, inputReader2));
221+
}
222+
}
223+
}
224+
}
225+
226+
@Benchmark
227+
public void testStringRelease_2_8_0(final Blackhole blackhole) throws IOException {
228+
for (int i = 0; i < 5; i++) {
229+
for (int j = 0; j < 5; j++) {
230+
try (InputStream inputReader1 = IOUtils.toInputStream(STRINGS[i], DEFAULT_CHARSET);
231+
InputStream inputReader2 = IOUtils.toInputStream(STRINGS[j], DEFAULT_CHARSET)) {
232+
blackhole.consume(contentEquals_release_2_8_0(inputReader1, inputReader2));
233+
}
234+
}
235+
}
236+
}
237+
238+
}

0 commit comments

Comments
 (0)