diff --git a/lucene/benchmark-jmh/build.gradle b/lucene/benchmark-jmh/build.gradle index 1751a43d7a79..1142a472296d 100644 --- a/lucene/benchmark-jmh/build.gradle +++ b/lucene/benchmark-jmh/build.gradle @@ -24,6 +24,7 @@ description = 'Lucene JMH micro-benchmarking module' dependencies { moduleImplementation project(':lucene:core') moduleImplementation project(':lucene:expressions') + moduleTestImplementation project(':lucene:test-framework') moduleImplementation deps.jmh.core annotationProcessor deps.jmh.annprocess diff --git a/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/DocIdEncodingBenchmark.java b/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/DocIdEncodingBenchmark.java new file mode 100644 index 000000000000..a6b09d4ea1b5 --- /dev/null +++ b/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/DocIdEncodingBenchmark.java @@ -0,0 +1,651 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.benchmark.jmh; + +import java.io.IOException; +import java.lang.reflect.InvocationTargetException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Random; +import java.util.concurrent.TimeUnit; +import java.util.stream.Stream; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.FSDirectory; +import org.apache.lucene.store.IOContext; +import org.apache.lucene.store.IndexInput; +import org.apache.lucene.store.IndexOutput; +import org.apache.lucene.util.Constants; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Level; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.TearDown; +import org.openjdk.jmh.annotations.Warmup; + +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(TimeUnit.MILLISECONDS) +@State(Scope.Benchmark) +@Warmup(iterations = 3, time = 5) +@Measurement(iterations = 5, time = 8) +@Fork(value = 1) +public class DocIdEncodingBenchmark { + + private static final long BPV_21_MASK = 0x1FFFFFL; + + private static List DOC_ID_SEQUENCES = new ArrayList<>(); + + private static int INPUT_SCALE_FACTOR; + + static { + parseInput(); + } + + @Param({ + "Bit21With3StepsEncoder", + "Bit21With2StepsEncoder", + "Bit24Encoder", + "Bit21HybridEncoder", + "Bit21With2StepsOnlyRWLongEncoder", + "Bit21With3StepsEncoderOnlyRWLongEncoder" + }) + String encoderName; + + @Param({"encode", "decode"}) + String methodName; + + private DocIdEncoder docIdEncoder; + + private Path tmpDir; + + private final int[] scratch = new int[512]; + + private String decoderInputFile; + + @Setup(Level.Trial) + public void init() throws IOException { + tmpDir = Files.createTempDirectory("docIdJmh"); + docIdEncoder = DocIdEncoder.SingletonFactory.fromName(encoderName); + decoderInputFile = + String.join("_", "docIdJmhData", docIdEncoder.getClass().getSimpleName(), "DecoderInput"); + // Create a file for decoders ( once per trial ) to read in every JMH iteration + if (methodName.equalsIgnoreCase("decode")) { + try (Directory dir = FSDirectory.open(tmpDir); + IndexOutput out = dir.createOutput(decoderInputFile, IOContext.DEFAULT)) { + encode(out, docIdEncoder, DOC_ID_SEQUENCES, INPUT_SCALE_FACTOR); + } + } + } + + @TearDown(Level.Trial) + public void finish() throws IOException { + if (methodName.equalsIgnoreCase("decode")) { + Files.delete(tmpDir.resolve(decoderInputFile)); + } + Files.delete(tmpDir); + } + + @Benchmark + public void executeEncodeOrDecode() throws IOException { + if (methodName.equalsIgnoreCase("encode")) { + String outputFile = + String.join( + "_", + "docIdJmhData", + docIdEncoder.getClass().getSimpleName(), + String.valueOf(System.nanoTime())); + try (Directory dir = FSDirectory.open(tmpDir); + IndexOutput out = dir.createOutput(outputFile, IOContext.DEFAULT)) { + encode(out, docIdEncoder, DOC_ID_SEQUENCES, INPUT_SCALE_FACTOR); + } finally { + Files.delete(tmpDir.resolve(outputFile)); + } + } else if (methodName.equalsIgnoreCase("decode")) { + try (Directory dir = FSDirectory.open(tmpDir); + IndexInput in = dir.openInput(decoderInputFile, IOContext.DEFAULT)) { + for (int[] docIdSequence : DOC_ID_SEQUENCES) { + for (int i = 1; i <= INPUT_SCALE_FACTOR; i++) { + docIdEncoder.decode(in, 0, docIdSequence.length, scratch); + } + } + } + } else { + throw new IllegalArgumentException("Unknown method: " + methodName); + } + } + + private void encode( + IndexOutput out, DocIdEncoder docIdEncoder, List docIdSequences, int inputScaleFactor) + throws IOException { + for (int[] docIdSequence : docIdSequences) { + for (int i = 1; i <= inputScaleFactor; i++) { + docIdEncoder.encode(out, 0, docIdSequence.length, docIdSequence); + } + } + } + + /** + * Extend this interface to add a new implementation used for DocId Encoding and Decoding. These + * are taken from org.apache.lucene.util.bkd.DocIdsWriter. + */ + public interface DocIdEncoder { + + void encode(IndexOutput out, int start, int count, int[] docIds) throws IOException; + + void decode(IndexInput in, int start, int count, int[] docIds) throws IOException; + + class SingletonFactory { + + private static final Map ENCODER_NAME_TO_INSTANCE_MAPPING = + new HashMap<>(); + + static { + initialiseEncoders(); + } + + private static String parsedClazzName(Class clazz) { + return clazz.getSimpleName().toLowerCase(Locale.ROOT); + } + + private static void initialiseEncoders() { + Class[] allImplementations = DocIdEncoder.class.getDeclaredClasses(); + for (Class clazz : allImplementations) { + boolean isADocIdEncoder = + Arrays.asList(clazz.getInterfaces()).contains(DocIdEncoder.class); + if (isADocIdEncoder) { + try { + ENCODER_NAME_TO_INSTANCE_MAPPING.put( + parsedClazzName(clazz), (DocIdEncoder) clazz.getConstructor().newInstance()); + } catch (InstantiationException + | IllegalAccessException + | InvocationTargetException + | NoSuchMethodException e) { + throw new RuntimeException(e); + } + } + } + } + + public static DocIdEncoder fromName(String encoderName) { + String parsedEncoderName = encoderName.trim().toLowerCase(Locale.ROOT); + return getInternal(parsedEncoderName); + } + + public static List getAllExcept( + List> excludeClasses) { + return ENCODER_NAME_TO_INSTANCE_MAPPING.values().stream() + .filter(x -> !excludeClasses.contains(x.getClass())) + .toList(); + } + + private static DocIdEncoder getInternal(String parsedEncoderName) { + if (ENCODER_NAME_TO_INSTANCE_MAPPING.containsKey(parsedEncoderName)) { + return ENCODER_NAME_TO_INSTANCE_MAPPING.get(parsedEncoderName); + } else { + throw new IllegalArgumentException( + String.format(Locale.ROOT, "Unknown DocIdEncoder [%s]", parsedEncoderName)); + } + } + } + + class Bit24Encoder implements DocIdEncoder { + @Override + public void encode(IndexOutput out, int start, int count, int[] docIds) throws IOException { + int i; + for (i = 0; i < count - 7; i += 8) { + int doc1 = docIds[i]; + int doc2 = docIds[i + 1]; + int doc3 = docIds[i + 2]; + int doc4 = docIds[i + 3]; + int doc5 = docIds[i + 4]; + int doc6 = docIds[i + 5]; + int doc7 = docIds[i + 6]; + int doc8 = docIds[i + 7]; + long l1 = (doc1 & 0xffffffL) << 40 | (doc2 & 0xffffffL) << 16 | ((doc3 >>> 8) & 0xffffL); + long l2 = + (doc3 & 0xffL) << 56 + | (doc4 & 0xffffffL) << 32 + | (doc5 & 0xffffffL) << 8 + | ((doc6 >> 16) & 0xffL); + long l3 = (doc6 & 0xffffL) << 48 | (doc7 & 0xffffffL) << 24 | (doc8 & 0xffffffL); + out.writeLong(l1); + out.writeLong(l2); + out.writeLong(l3); + } + for (; i < count; ++i) { + out.writeShort((short) (docIds[i] >>> 8)); + out.writeByte((byte) docIds[i]); + } + } + + @Override + public void decode(IndexInput in, int start, int count, int[] docIDs) throws IOException { + int i; + for (i = 0; i < count - 7; i += 8) { + long l1 = in.readLong(); + long l2 = in.readLong(); + long l3 = in.readLong(); + docIDs[i] = (int) (l1 >>> 40); + docIDs[i + 1] = (int) (l1 >>> 16) & 0xffffff; + docIDs[i + 2] = (int) (((l1 & 0xffff) << 8) | (l2 >>> 56)); + docIDs[i + 3] = (int) (l2 >>> 32) & 0xffffff; + docIDs[i + 4] = (int) (l2 >>> 8) & 0xffffff; + docIDs[i + 5] = (int) (((l2 & 0xff) << 16) | (l3 >>> 48)); + docIDs[i + 6] = (int) (l3 >>> 24) & 0xffffff; + docIDs[i + 7] = (int) l3 & 0xffffff; + } + for (; i < count; ++i) { + docIDs[i] = + (Short.toUnsignedInt(in.readShort()) << 8) | Byte.toUnsignedInt(in.readByte()); + } + } + } + + /** + * Uses 21 bits to represent an integer and can store 3 docIds within a long. This is the + * simplified version which is faster in encoding in aarch64 + */ + class Bit21With2StepsEncoder implements DocIdEncoder { + @Override + public void encode(IndexOutput out, int start, int count, int[] docIds) throws IOException { + int i = 0; + for (; i < count - 2; i += 3) { + long packedLong = + ((docIds[i] & BPV_21_MASK) << 42) + | ((docIds[i + 1] & BPV_21_MASK) << 21) + | (docIds[i + 2] & BPV_21_MASK); + out.writeLong(packedLong); + } + for (; i < count; i++) { + out.writeInt(docIds[i]); + } + } + + @Override + public void decode(IndexInput in, int start, int count, int[] docIDs) throws IOException { + int i = 0; + for (; i < count - 2; i += 3) { + long packedLong = in.readLong(); + docIDs[i] = (int) (packedLong >>> 42); + docIDs[i + 1] = (int) ((packedLong >>> 21) & BPV_21_MASK); + docIDs[i + 2] = (int) (packedLong & BPV_21_MASK); + } + for (; i < count; i++) { + docIDs[i] = in.readInt(); + } + } + } + + class Bit21With2StepsOnlyRWLongEncoder implements DocIdEncoder { + @Override + public void encode(IndexOutput out, int start, int count, int[] docIds) throws IOException { + int i = 0; + for (; i < count - 2; i += 3) { + long packedLong = + ((docIds[i] & BPV_21_MASK) << 42) + | ((docIds[i + 1] & BPV_21_MASK) << 21) + | (docIds[i + 2] & BPV_21_MASK); + out.writeLong(packedLong); + } + for (; i < count; i++) { + out.writeLong(docIds[i]); + } + } + + @Override + public void decode(IndexInput in, int start, int count, int[] docIDs) throws IOException { + int i = 0; + for (; i < count - 2; i += 3) { + long packedLong = in.readLong(); + docIDs[i] = (int) (packedLong >>> 42); + docIDs[i + 1] = (int) ((packedLong >>> 21) & BPV_21_MASK); + docIDs[i + 2] = (int) (packedLong & BPV_21_MASK); + } + for (; i < count; i++) { + docIDs[i] = (int) in.readLong(); + } + } + } + + /** + * Variation of @{@link Bit21With2StepsEncoder} but uses 3 loops to decode the array of DocIds. + * Comparatively better in decoding than @{@link Bit21With2StepsEncoder} on aarch64 with JDK 22 + * whereas poorer in encoding. + */ + class Bit21With3StepsEncoder implements DocIdEncoder { + + @Override + public void encode(IndexOutput out, int start, int count, int[] docIds) throws IOException { + int i = 0; + for (; i < count - 8; i += 9) { + long l1 = + ((docIds[i] & BPV_21_MASK) << 42) + | ((docIds[i + 1] & BPV_21_MASK) << 21) + | (docIds[i + 2] & BPV_21_MASK); + long l2 = + ((docIds[i + 3] & BPV_21_MASK) << 42) + | ((docIds[i + 4] & BPV_21_MASK) << 21) + | (docIds[i + 5] & BPV_21_MASK); + long l3 = + ((docIds[i + 6] & BPV_21_MASK) << 42) + | ((docIds[i + 7] & BPV_21_MASK) << 21) + | (docIds[i + 8] & BPV_21_MASK); + out.writeLong(l1); + out.writeLong(l2); + out.writeLong(l3); + } + for (; i < count - 2; i += 3) { + long packedLong = + ((docIds[i] & BPV_21_MASK) << 42) + | ((docIds[i + 1] & BPV_21_MASK) << 21) + | (docIds[i + 2] & BPV_21_MASK); + out.writeLong(packedLong); + } + for (; i < count; i++) { + out.writeInt(docIds[i]); + } + } + + @Override + public void decode(IndexInput in, int start, int count, int[] docIDs) throws IOException { + int i = 0; + for (; i < count - 8; i += 9) { + long l1 = in.readLong(); + long l2 = in.readLong(); + long l3 = in.readLong(); + docIDs[i] = (int) (l1 >>> 42); + docIDs[i + 1] = (int) ((l1 >>> 21) & BPV_21_MASK); + docIDs[i + 2] = (int) (l1 & BPV_21_MASK); + docIDs[i + 3] = (int) (l2 >>> 42); + docIDs[i + 4] = (int) ((l2 >>> 21) & BPV_21_MASK); + docIDs[i + 5] = (int) (l2 & BPV_21_MASK); + docIDs[i + 6] = (int) (l3 >>> 42); + docIDs[i + 7] = (int) ((l3 >>> 21) & BPV_21_MASK); + docIDs[i + 8] = (int) (l3 & BPV_21_MASK); + } + for (; i < count - 2; i += 3) { + long packedLong = in.readLong(); + docIDs[i] = (int) (packedLong >>> 42); + docIDs[i + 1] = (int) ((packedLong >>> 21) & BPV_21_MASK); + docIDs[i + 2] = (int) (packedLong & BPV_21_MASK); + } + for (; i < count; i++) { + docIDs[i] = in.readInt(); + } + } + } + + class Bit21With3StepsEncoderOnlyRWLongEncoder implements DocIdEncoder { + + @Override + public void encode(IndexOutput out, int start, int count, int[] docIds) throws IOException { + int i = 0; + for (; i < count - 8; i += 9) { + long l1 = + ((docIds[i] & BPV_21_MASK) << 42) + | ((docIds[i + 1] & BPV_21_MASK) << 21) + | (docIds[i + 2] & BPV_21_MASK); + long l2 = + ((docIds[i + 3] & BPV_21_MASK) << 42) + | ((docIds[i + 4] & BPV_21_MASK) << 21) + | (docIds[i + 5] & BPV_21_MASK); + long l3 = + ((docIds[i + 6] & BPV_21_MASK) << 42) + | ((docIds[i + 7] & BPV_21_MASK) << 21) + | (docIds[i + 8] & BPV_21_MASK); + out.writeLong(l1); + out.writeLong(l2); + out.writeLong(l3); + } + for (; i < count - 2; i += 3) { + long packedLong = + ((docIds[i] & BPV_21_MASK) << 42) + | ((docIds[i + 1] & BPV_21_MASK) << 21) + | (docIds[i + 2] & BPV_21_MASK); + out.writeLong(packedLong); + } + for (; i < count; i++) { + out.writeLong(docIds[i]); + } + } + + @Override + public void decode(IndexInput in, int start, int count, int[] docIDs) throws IOException { + int i = 0; + for (; i < count - 8; i += 9) { + long l1 = in.readLong(); + long l2 = in.readLong(); + long l3 = in.readLong(); + docIDs[i] = (int) (l1 >>> 42); + docIDs[i + 1] = (int) ((l1 >>> 21) & BPV_21_MASK); + docIDs[i + 2] = (int) (l1 & BPV_21_MASK); + docIDs[i + 3] = (int) (l2 >>> 42); + docIDs[i + 4] = (int) ((l2 >>> 21) & BPV_21_MASK); + docIDs[i + 5] = (int) (l2 & BPV_21_MASK); + docIDs[i + 6] = (int) (l3 >>> 42); + docIDs[i + 7] = (int) ((l3 >>> 21) & BPV_21_MASK); + docIDs[i + 8] = (int) (l3 & BPV_21_MASK); + } + for (; i < count - 2; i += 3) { + long packedLong = in.readLong(); + docIDs[i] = (int) (packedLong >>> 42); + docIDs[i + 1] = (int) ((packedLong >>> 21) & BPV_21_MASK); + docIDs[i + 2] = (int) (packedLong & BPV_21_MASK); + } + for (; i < count; i++) { + docIDs[i] = (int) in.readLong(); + } + } + } + + class Bit21HybridEncoder implements DocIdEncoder { + + private final DocIdEncoder encoder; + private final DocIdEncoder decoder; + + public Bit21HybridEncoder() { + if (Constants.OS_ARCH.equals("aarch64")) { + this.encoder = this.decoder = new Bit21With2StepsEncoder(); + } else { + this.encoder = this.decoder = new Bit21With3StepsEncoderOnlyRWLongEncoder(); + } + } + + @Override + public void encode(IndexOutput out, int start, int count, int[] docIds) throws IOException { + encoder.encode(out, start, count, docIds); + } + + @Override + public void decode(IndexInput in, int start, int count, int[] docIds) throws IOException { + decoder.decode(in, start, count, docIds); + } + } + + /** + * Last fallback in org.apache.lucene.util.bkd.DocIdsWriter#writeDocIds() when no optimisation + * works + */ + class Bit32Encoder implements DocIdEncoder { + + @Override + public void encode(IndexOutput out, int start, int count, int[] docIds) throws IOException { + for (int i = 0; i < count; i++) { + out.writeInt(docIds[i]); + } + } + + @Override + public void decode(IndexInput in, int start, int count, int[] docIds) throws IOException { + for (int i = 0; i < count; i++) { + docIds[i] = in.readInt(); + } + } + } + + /** Variation of @{@link Bit32Encoder} using readLong and writeLong methods. */ + class Bit32OnlyRWLongEncoder implements DocIdEncoder { + + @Override + public void encode(IndexOutput out, int start, int count, int[] docIds) throws IOException { + int i; + for (i = 0; i < count - 1; i += 2) { + long packedLong = (((long) docIds[i]) << 32) | docIds[i + 1]; + out.writeLong(packedLong); + } + for (; i < count; i++) { + out.writeLong(docIds[i]); + } + } + + @Override + public void decode(IndexInput in, int start, int count, int[] docIds) throws IOException { + int i; + for (i = 0; i < count - 1; i += 2) { + long packedLong = in.readLong(); + docIds[i] = (int) (packedLong >>> 32); + docIds[i + 1] = (int) (packedLong & 0xFFFFFFFFL); + } + for (; i < count; i++) { + docIds[i] = (int) in.readLong(); + } + } + } + } + + interface DocIdProvider { + + Map, Integer> ENCODER_TO_BPV_MAPPING = + Map.of( + DocIdEncodingBenchmark.DocIdEncoder.Bit21With2StepsEncoder.class, + 21, + DocIdEncodingBenchmark.DocIdEncoder.Bit21With3StepsEncoder.class, + 21, + DocIdEncodingBenchmark.DocIdEncoder.Bit21With2StepsOnlyRWLongEncoder.class, + 21, + DocIdEncodingBenchmark.DocIdEncoder.Bit21With3StepsEncoderOnlyRWLongEncoder.class, + 21, + DocIdEncodingBenchmark.DocIdEncoder.Bit21HybridEncoder.class, + 21, + DocIdEncodingBenchmark.DocIdEncoder.Bit24Encoder.class, + 24, + DocIdEncodingBenchmark.DocIdEncoder.Bit32Encoder.class, + 32, + DocIdEncoder.Bit32OnlyRWLongEncoder.class, + 32); + + /** + * We want to load all the docId sequences completely in memory to avoid including the time + * spent in fetching from disk or any other source in every iteration unless we can consistently + * prove otherwise.
+ * + * @param args : Data about the source of docId sequences depending on the underlying provider + * like a file or randomly generated sequences given size. + * @return : Loaded docIds + */ + List getDocIds(Object... args); + } + + static class DocIdsFromLocalFS implements DocIdProvider { + + @Override + public List getDocIds(Object... args) { + try (Stream lines = Files.lines(Path.of((String) args[0]))) { + return lines + .parallel() + .map(String::trim) + .filter(x -> !(x.startsWith("#") || x.isEmpty())) // Comments can start with a # + .map( + x -> + Arrays.stream(x.split(",")) + .mapToInt((y -> Integer.parseInt(y.trim()))) + .toArray()) + .toList(); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + } + + static class FixedBPVRandomDocIdProvider implements DocIdEncodingBenchmark.DocIdProvider { + + private static final Random RANDOM = new Random(); + + private static final Map, Double> ENCODER_POWERS_OF_2; + + static { + ENCODER_POWERS_OF_2 = new HashMap<>(ENCODER_TO_BPV_MAPPING.size()); + ENCODER_TO_BPV_MAPPING.forEach( + (encoderClazz, bitsUsed) -> + ENCODER_POWERS_OF_2.put(encoderClazz, Math.pow(2, bitsUsed) - 1)); + } + + @SuppressWarnings("unchecked") + @Override + public List getDocIds(Object... args) { + + Class encoderClass = (Class) args[0]; + int capacity = (int) args[1]; + int low = (int) args[2]; + int high = (int) args[3]; + + List docIdSequences = new ArrayList<>(capacity); + + for (int i = 1; i <= capacity; i++) { + docIdSequences.add( + RANDOM + .ints(0, ENCODER_POWERS_OF_2.get(encoderClass).intValue()) + .distinct() + .limit(RANDOM.nextInt(low, high)) + .toArray()); + } + return docIdSequences; + } + } + + private static void parseInput() { + + String inputScaleFactor = System.getProperty("docIdEncoding.inputScaleFactor"); + + if (inputScaleFactor != null && !inputScaleFactor.isEmpty()) { + INPUT_SCALE_FACTOR = Integer.parseInt(inputScaleFactor); + } else { + INPUT_SCALE_FACTOR = 10; + } + + String inputFilePath = System.getProperty("docIdEncoding.inputFile"); + if (inputFilePath != null && !inputFilePath.isEmpty()) { + DOC_ID_SEQUENCES = new DocIdsFromLocalFS().getDocIds(inputFilePath); + } else { + DOC_ID_SEQUENCES = + new FixedBPVRandomDocIdProvider() + .getDocIds(DocIdEncoder.Bit21With3StepsEncoder.class, 100, 100, 512); + } + } +} diff --git a/lucene/benchmark-jmh/src/test/org/apache/lucene/benchmark/jmh/TestDocIdEncoding.java b/lucene/benchmark-jmh/src/test/org/apache/lucene/benchmark/jmh/TestDocIdEncoding.java new file mode 100644 index 000000000000..74b151db5099 --- /dev/null +++ b/lucene/benchmark-jmh/src/test/org/apache/lucene/benchmark/jmh/TestDocIdEncoding.java @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.benchmark.jmh; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Collections; +import java.util.List; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.FSDirectory; +import org.apache.lucene.store.IOContext; +import org.apache.lucene.store.IndexInput; +import org.apache.lucene.store.IndexOutput; +import org.apache.lucene.tests.util.LuceneTestCase; +import org.apache.lucene.util.ArrayUtil; + +public class TestDocIdEncoding extends LuceneTestCase { + + @Override + public void setUp() throws Exception { + super.setUp(); + } + + public void testBPV21AndAbove() throws IOException { + + List encoders = + DocIdEncodingBenchmark.DocIdEncoder.SingletonFactory.getAllExcept(Collections.emptyList()); + + final int[] scratch = new int[512]; + + DocIdEncodingBenchmark.DocIdProvider docIdProvider = + new DocIdEncodingBenchmark.FixedBPVRandomDocIdProvider(); + + Path tempDir = null; + + try { + + tempDir = Files.createTempDirectory("DocIdEncoding_testBPV21AndAbove_"); + + for (DocIdEncodingBenchmark.DocIdEncoder encoder : encoders) { + + List docIdSequences = docIdProvider.getDocIds(encoder.getClass(), 100, 100, 512); + + String encoderFileName = "Encoder_" + encoder.getClass().getSimpleName(); + + try (Directory outDir = FSDirectory.open(tempDir); + IndexOutput out = outDir.createOutput(encoderFileName, IOContext.DEFAULT)) { + for (int[] sequence : docIdSequences) { + encoder.encode(out, 0, sequence.length, sequence); + } + } + + try (Directory inDir = FSDirectory.open(tempDir); + IndexInput in = inDir.openInput(encoderFileName, IOContext.DEFAULT)) { + for (int[] sequence : docIdSequences) { + encoder.decode(in, 0, sequence.length, scratch); + assertArrayEquals(sequence, ArrayUtil.copyOfSubArray(scratch, 0, sequence.length)); + } + } finally { + Files.delete(tempDir.resolve(encoderFileName)); + } + } + } finally { + if (tempDir != null) { + Files.delete(tempDir); + } + } + } +} diff --git a/lucene/core/src/java/org/apache/lucene/util/bkd/DocIdsWriter.java b/lucene/core/src/java/org/apache/lucene/util/bkd/DocIdsWriter.java index b9ea0d9aa08e..5c11ed88ad74 100644 --- a/lucene/core/src/java/org/apache/lucene/util/bkd/DocIdsWriter.java +++ b/lucene/core/src/java/org/apache/lucene/util/bkd/DocIdsWriter.java @@ -23,6 +23,7 @@ import org.apache.lucene.store.DataOutput; import org.apache.lucene.store.IndexInput; import org.apache.lucene.util.ArrayUtil; +import org.apache.lucene.util.Constants; import org.apache.lucene.util.DocBaseBitSetIterator; import org.apache.lucene.util.FixedBitSet; import org.apache.lucene.util.IntsRef; @@ -33,11 +34,15 @@ final class DocIdsWriter { private static final byte CONTINUOUS_IDS = (byte) -2; private static final byte BITSET_IDS = (byte) -1; private static final byte DELTA_BPV_16 = (byte) 16; + private static final byte BPV_21 = (byte) 21; private static final byte BPV_24 = (byte) 24; private static final byte BPV_32 = (byte) 32; // These signs are legacy, should no longer be used in the writing side. private static final byte LEGACY_DELTA_VINT = (byte) 0; + private static final long BPV_21_MASK = 0x1FFFFFL; + private static final boolean IS_ARCH_64 = Constants.OS_ARCH.equals("aarch64"); + private final int[] scratch; private final LongsRef scratchLongs = new LongsRef(); @@ -112,7 +117,47 @@ void writeDocIds(int[] docIds, int start, int count, DataOutput out) throws IOEx out.writeShort((short) scratch[count - 1]); } } else { - if (max <= 0xFFFFFF) { + if (max <= 0x001FFFFF) { + out.writeByte(BPV_21); + int i = 0; + // See + // @org.apache.lucene.benchmark.jmh.DocIdEncodingBenchmark$DocIdEncoder$Bit21With3StepsEncoder + if (!IS_ARCH_64) { + for (; i < count - 8; i += 9) { + long l1 = + ((docIds[i] & BPV_21_MASK) << 42) + | ((docIds[i + 1] & BPV_21_MASK) << 21) + | (docIds[i + 2] & BPV_21_MASK); + long l2 = + ((docIds[i + 3] & BPV_21_MASK) << 42) + | ((docIds[i + 4] & BPV_21_MASK) << 21) + | (docIds[i + 5] & BPV_21_MASK); + long l3 = + ((docIds[i + 6] & BPV_21_MASK) << 42) + | ((docIds[i + 7] & BPV_21_MASK) << 21) + | (docIds[i + 8] & BPV_21_MASK); + out.writeLong(l1); + out.writeLong(l2); + out.writeLong(l3); + } + } + for (; i < count - 2; i += 3) { + long packedLong = + ((docIds[i] & BPV_21_MASK) << 42) + | ((docIds[i + 1] & BPV_21_MASK) << 21) + | (docIds[i + 2] & BPV_21_MASK); + out.writeLong(packedLong); + } + if (IS_ARCH_64) { + for (; i < count; i++) { + out.writeInt(docIds[i]); + } + } else { + for (; i < count; i++) { + out.writeLong(docIds[i]); + } + } + } else if (max <= 0xFFFFFF) { out.writeByte(BPV_24); // write them the same way we are reading them. int i; @@ -195,6 +240,9 @@ void readInts(IndexInput in, int count, int[] docIDs) throws IOException { case DELTA_BPV_16: readDelta16(in, count, docIDs); break; + case BPV_21: + readInts21(in, count, docIDs); + break; case BPV_24: readInts24(in, count, docIDs); break; @@ -262,6 +310,45 @@ private static void readDelta16(IndexInput in, int count, int[] docIDs) throws I } } + private void readInts21(IndexInput in, int count, int[] docIDs) throws IOException { + int i = 0; + // We are always using + // org.apache.lucene.benchmark.jmh.DocIdEncodingBenchmark.DocIdEncoder.Bit21With3StepsEncoder + // over + // org.apache.lucene.benchmark.jmh.DocIdEncodingBenchmark.DocIdEncoder.Bit21With2StepsEncoder + // for decoding irrespective of architecture + // due to it's better performance in benchmarks like nyc taxis, big5, http_logs. + for (; i < count - 8; i += 9) { + long l1 = in.readLong(); + long l2 = in.readLong(); + long l3 = in.readLong(); + docIDs[i] = (int) (l1 >>> 42); + docIDs[i + 1] = (int) ((l1 >>> 21) & BPV_21_MASK); + docIDs[i + 2] = (int) (l1 & BPV_21_MASK); + docIDs[i + 3] = (int) (l2 >>> 42); + docIDs[i + 4] = (int) ((l2 >>> 21) & BPV_21_MASK); + docIDs[i + 5] = (int) (l2 & BPV_21_MASK); + docIDs[i + 6] = (int) (l3 >>> 42); + docIDs[i + 7] = (int) ((l3 >>> 21) & BPV_21_MASK); + docIDs[i + 8] = (int) (l3 & BPV_21_MASK); + } + for (; i < count - 2; i += 3) { + long packedLong = in.readLong(); + docIDs[i] = (int) (packedLong >>> 42); + docIDs[i + 1] = (int) ((packedLong >>> 21) & BPV_21_MASK); + docIDs[i + 2] = (int) (packedLong & BPV_21_MASK); + } + if (IS_ARCH_64) { + for (; i < count; i++) { + docIDs[i] = in.readInt(); + } + } else { + for (; i < count; i++) { + docIDs[i] = (int) in.readLong(); + } + } + } + private static void readInts24(IndexInput in, int count, int[] docIDs) throws IOException { int i; for (i = 0; i < count - 7; i += 8) { @@ -302,6 +389,9 @@ void readInts(IndexInput in, int count, IntersectVisitor visitor) throws IOExcep case DELTA_BPV_16: readDelta16(in, count, visitor); break; + case BPV_21: + readInts21(in, count, visitor); + break; case BPV_24: readInts24(in, count, visitor); break; @@ -348,6 +438,13 @@ private void readDelta16(IndexInput in, int count, IntersectVisitor visitor) thr visitor.visit(scratchIntsRef); } + private void readInts21(IndexInput in, int count, IntersectVisitor visitor) throws IOException { + readInts21(in, count, scratch); + scratchIntsRef.ints = scratch; + scratchIntsRef.length = count; + visitor.visit(scratchIntsRef); + } + private static void readInts24(IndexInput in, int count, IntersectVisitor visitor) throws IOException { int i; diff --git a/versions.lock b/versions.lock index 26de44f99e2d..76d344d1762a 100644 --- a/versions.lock +++ b/versions.lock @@ -27,7 +27,7 @@ "xerces:xercesImpl:2.12.0" : "5ce8cdc6,refs=2" }, "test_dependencies" : { - "com.carrotsearch.randomizedtesting:randomizedtesting-runner:2.8.1" : "b35e5d7a,refs=74", + "com.carrotsearch.randomizedtesting:randomizedtesting-runner:2.8.1" : "129da9bf,refs=76", "com.carrotsearch:procfork:1.0.6" : "b7ba1646,refs=2", "com.github.ben-manes.caffeine:caffeine:3.0.5" : "6897bc09,refs=38", "com.github.kevinstern:software-and-algorithms:1.0" : "6897bc09,refs=38", @@ -50,7 +50,7 @@ "io.github.java-diff-utils:java-diff-utils:4.0" : "6897bc09,refs=38", "io.sgr:s2-geometry-library-java:1.0.0" : "1d5a4b2b,refs=4", "javax.inject:javax.inject:1" : "6897bc09,refs=38", - "junit:junit:4.13.1" : "b35e5d7a,refs=74", + "junit:junit:4.13.1" : "129da9bf,refs=76", "net.sf.jopt-simple:jopt-simple:5.0.4" : "152d9f78,refs=3", "net.sourceforge.nekohtml:nekohtml:1.9.17" : "6f16ff86,refs=2", "org.antlr:antlr4-runtime:4.11.1" : "6fbc4021,refs=5", @@ -64,7 +64,7 @@ "org.checkerframework:checker-qual:3.19.0" : "6897bc09,refs=38", "org.checkerframework:dataflow-errorprone:3.27.0" : "6897bc09,refs=38", "org.eclipse.jgit:org.eclipse.jgit:4.4.1.201607150455-r" : "6897bc09,refs=38", - "org.hamcrest:hamcrest:2.2" : "b35e5d7a,refs=74", + "org.hamcrest:hamcrest:2.2" : "129da9bf,refs=76", "org.locationtech.jts:jts-core:1.17.0" : "180518e6,refs=2", "org.locationtech.spatial4j:spatial4j:0.8" : "1d5a4b2b,refs=4", "org.openjdk.jmh:jmh-core:1.37" : "152d9f78,refs=3", @@ -79,31 +79,23 @@ } }, "because" : { - "152d9f78" : [ - { - "configuration" : "annotationProcessor", - "projectPath" : ":lucene:benchmark-jmh" - }, + "129da9bf" : [ { "configuration" : "testCompileClasspath", - "projectPath" : ":lucene:benchmark-jmh" + "projectPath" : ":lucene:analysis.tests" }, { "configuration" : "testRuntimeClasspath", - "projectPath" : ":lucene:benchmark-jmh" - } - ], - "180518e6" : [ + "projectPath" : ":lucene:analysis.tests" + }, { "configuration" : "testCompileClasspath", - "projectPath" : ":lucene:spatial-extras" + "projectPath" : ":lucene:backward-codecs" }, { "configuration" : "testRuntimeClasspath", - "projectPath" : ":lucene:spatial-extras" - } - ], - "1d5a4b2b" : [ + "projectPath" : ":lucene:backward-codecs" + }, { "configuration" : "testCompileClasspath", "projectPath" : ":lucene:benchmark" @@ -114,280 +106,354 @@ }, { "configuration" : "testCompileClasspath", - "projectPath" : ":lucene:spatial-extras" + "projectPath" : ":lucene:benchmark-jmh" }, { "configuration" : "testRuntimeClasspath", - "projectPath" : ":lucene:spatial-extras" - } - ], - "2f760bab" : [ - { - "configuration" : "compileClasspath", - "projectPath" : ":lucene:luke" + "projectPath" : ":lucene:benchmark-jmh" }, { - "configuration" : "runtimeClasspath", - "projectPath" : ":lucene:luke" + "configuration" : "testCompileClasspath", + "projectPath" : ":lucene:classification" }, { - "configuration" : "compileClasspath", - "projectPath" : ":lucene:analysis:opennlp" + "configuration" : "testRuntimeClasspath", + "projectPath" : ":lucene:classification" }, { - "configuration" : "runtimeClasspath", - "projectPath" : ":lucene:analysis:opennlp" - } - ], - "47ea4550" : [ - { - "configuration" : "compileClasspath", - "projectPath" : ":lucene:benchmark" + "configuration" : "testCompileClasspath", + "projectPath" : ":lucene:codecs" }, { - "configuration" : "runtimeClasspath", - "projectPath" : ":lucene:benchmark" + "configuration" : "testRuntimeClasspath", + "projectPath" : ":lucene:codecs" }, { - "configuration" : "compileClasspath", - "projectPath" : ":lucene:luke" + "configuration" : "testCompileClasspath", + "projectPath" : ":lucene:core" }, { - "configuration" : "runtimeClasspath", - "projectPath" : ":lucene:luke" + "configuration" : "testRuntimeClasspath", + "projectPath" : ":lucene:core" }, { - "configuration" : "compileClasspath", - "projectPath" : ":lucene:analysis:icu" + "configuration" : "testCompileClasspath", + "projectPath" : ":lucene:core.tests" }, { - "configuration" : "runtimeClasspath", - "projectPath" : ":lucene:analysis:icu" - } - ], - "5ce8cdc6" : [ - { - "configuration" : "compileClasspath", - "projectPath" : ":lucene:benchmark" + "configuration" : "testRuntimeClasspath", + "projectPath" : ":lucene:core.tests" }, { - "configuration" : "runtimeClasspath", - "projectPath" : ":lucene:benchmark" - } - ], - "6897bc09" : [ - { - "configuration" : "annotationProcessor", - "projectPath" : ":lucene:analysis.tests" + "configuration" : "testCompileClasspath", + "projectPath" : ":lucene:demo" }, { - "configuration" : "annotationProcessor", - "projectPath" : ":lucene:backward-codecs" + "configuration" : "testRuntimeClasspath", + "projectPath" : ":lucene:demo" }, { - "configuration" : "annotationProcessor", - "projectPath" : ":lucene:benchmark" + "configuration" : "testCompileClasspath", + "projectPath" : ":lucene:distribution.tests" }, { - "configuration" : "annotationProcessor", - "projectPath" : ":lucene:benchmark-jmh" + "configuration" : "testRuntimeClasspath", + "projectPath" : ":lucene:distribution.tests" }, { - "configuration" : "annotationProcessor", - "projectPath" : ":lucene:classification" + "configuration" : "testCompileClasspath", + "projectPath" : ":lucene:expressions" }, { - "configuration" : "annotationProcessor", - "projectPath" : ":lucene:codecs" + "configuration" : "testRuntimeClasspath", + "projectPath" : ":lucene:expressions" }, { - "configuration" : "annotationProcessor", - "projectPath" : ":lucene:core" + "configuration" : "testCompileClasspath", + "projectPath" : ":lucene:facet" }, { - "configuration" : "annotationProcessor", - "projectPath" : ":lucene:core.tests" + "configuration" : "testRuntimeClasspath", + "projectPath" : ":lucene:facet" }, { - "configuration" : "annotationProcessor", - "projectPath" : ":lucene:demo" + "configuration" : "testCompileClasspath", + "projectPath" : ":lucene:grouping" }, { - "configuration" : "annotationProcessor", - "projectPath" : ":lucene:distribution.tests" + "configuration" : "testRuntimeClasspath", + "projectPath" : ":lucene:grouping" }, { - "configuration" : "annotationProcessor", - "projectPath" : ":lucene:expressions" + "configuration" : "testCompileClasspath", + "projectPath" : ":lucene:highlighter" }, { - "configuration" : "annotationProcessor", - "projectPath" : ":lucene:facet" + "configuration" : "testRuntimeClasspath", + "projectPath" : ":lucene:highlighter" }, { - "configuration" : "annotationProcessor", - "projectPath" : ":lucene:grouping" + "configuration" : "testCompileClasspath", + "projectPath" : ":lucene:join" }, { - "configuration" : "annotationProcessor", - "projectPath" : ":lucene:highlighter" + "configuration" : "testRuntimeClasspath", + "projectPath" : ":lucene:join" }, { - "configuration" : "annotationProcessor", - "projectPath" : ":lucene:join" + "configuration" : "testCompileClasspath", + "projectPath" : ":lucene:luke" }, { - "configuration" : "annotationProcessor", + "configuration" : "testRuntimeClasspath", "projectPath" : ":lucene:luke" }, { - "configuration" : "annotationProcessor", + "configuration" : "testCompileClasspath", "projectPath" : ":lucene:memory" }, { - "configuration" : "annotationProcessor", + "configuration" : "testRuntimeClasspath", + "projectPath" : ":lucene:memory" + }, + { + "configuration" : "testCompileClasspath", "projectPath" : ":lucene:misc" }, { - "configuration" : "annotationProcessor", + "configuration" : "testRuntimeClasspath", + "projectPath" : ":lucene:misc" + }, + { + "configuration" : "testCompileClasspath", "projectPath" : ":lucene:monitor" }, { - "configuration" : "annotationProcessor", + "configuration" : "testRuntimeClasspath", + "projectPath" : ":lucene:monitor" + }, + { + "configuration" : "testCompileClasspath", "projectPath" : ":lucene:queries" }, { - "configuration" : "annotationProcessor", + "configuration" : "testRuntimeClasspath", + "projectPath" : ":lucene:queries" + }, + { + "configuration" : "testCompileClasspath", "projectPath" : ":lucene:queryparser" }, { - "configuration" : "annotationProcessor", + "configuration" : "testRuntimeClasspath", + "projectPath" : ":lucene:queryparser" + }, + { + "configuration" : "testCompileClasspath", "projectPath" : ":lucene:replicator" }, { - "configuration" : "annotationProcessor", + "configuration" : "testRuntimeClasspath", + "projectPath" : ":lucene:replicator" + }, + { + "configuration" : "testCompileClasspath", "projectPath" : ":lucene:sandbox" }, { - "configuration" : "annotationProcessor", + "configuration" : "testRuntimeClasspath", + "projectPath" : ":lucene:sandbox" + }, + { + "configuration" : "testCompileClasspath", "projectPath" : ":lucene:spatial-extras" }, { - "configuration" : "annotationProcessor", + "configuration" : "testRuntimeClasspath", + "projectPath" : ":lucene:spatial-extras" + }, + { + "configuration" : "testCompileClasspath", "projectPath" : ":lucene:spatial-test-fixtures" }, { - "configuration" : "annotationProcessor", + "configuration" : "testRuntimeClasspath", + "projectPath" : ":lucene:spatial-test-fixtures" + }, + { + "configuration" : "testCompileClasspath", "projectPath" : ":lucene:spatial3d" }, { - "configuration" : "annotationProcessor", + "configuration" : "testRuntimeClasspath", + "projectPath" : ":lucene:spatial3d" + }, + { + "configuration" : "testCompileClasspath", "projectPath" : ":lucene:suggest" }, { - "configuration" : "annotationProcessor", + "configuration" : "testRuntimeClasspath", + "projectPath" : ":lucene:suggest" + }, + { + "configuration" : "testCompileClasspath", "projectPath" : ":lucene:test-framework" }, { - "configuration" : "annotationProcessor", + "configuration" : "testRuntimeClasspath", + "projectPath" : ":lucene:test-framework" + }, + { + "configuration" : "testCompileClasspath", "projectPath" : ":lucene:analysis:common" }, { - "configuration" : "annotationProcessor", + "configuration" : "testRuntimeClasspath", + "projectPath" : ":lucene:analysis:common" + }, + { + "configuration" : "testCompileClasspath", "projectPath" : ":lucene:analysis:icu" }, { - "configuration" : "annotationProcessor", + "configuration" : "testRuntimeClasspath", + "projectPath" : ":lucene:analysis:icu" + }, + { + "configuration" : "testCompileClasspath", "projectPath" : ":lucene:analysis:kuromoji" }, { - "configuration" : "annotationProcessor", + "configuration" : "testRuntimeClasspath", + "projectPath" : ":lucene:analysis:kuromoji" + }, + { + "configuration" : "testCompileClasspath", "projectPath" : ":lucene:analysis:morfologik" }, { - "configuration" : "annotationProcessor", + "configuration" : "testRuntimeClasspath", + "projectPath" : ":lucene:analysis:morfologik" + }, + { + "configuration" : "testCompileClasspath", "projectPath" : ":lucene:analysis:morfologik.tests" }, { - "configuration" : "annotationProcessor", + "configuration" : "testRuntimeClasspath", + "projectPath" : ":lucene:analysis:morfologik.tests" + }, + { + "configuration" : "testCompileClasspath", "projectPath" : ":lucene:analysis:nori" }, { - "configuration" : "annotationProcessor", + "configuration" : "testRuntimeClasspath", + "projectPath" : ":lucene:analysis:nori" + }, + { + "configuration" : "testCompileClasspath", "projectPath" : ":lucene:analysis:opennlp" }, { - "configuration" : "annotationProcessor", + "configuration" : "testRuntimeClasspath", + "projectPath" : ":lucene:analysis:opennlp" + }, + { + "configuration" : "testCompileClasspath", "projectPath" : ":lucene:analysis:phonetic" }, { - "configuration" : "annotationProcessor", + "configuration" : "testRuntimeClasspath", + "projectPath" : ":lucene:analysis:phonetic" + }, + { + "configuration" : "testCompileClasspath", "projectPath" : ":lucene:analysis:smartcn" }, { - "configuration" : "annotationProcessor", - "projectPath" : ":lucene:analysis:stempel" - } - ], - "6f16ff86" : [ + "configuration" : "testRuntimeClasspath", + "projectPath" : ":lucene:analysis:smartcn" + }, { "configuration" : "testCompileClasspath", - "projectPath" : ":lucene:benchmark" + "projectPath" : ":lucene:analysis:stempel" }, { "configuration" : "testRuntimeClasspath", - "projectPath" : ":lucene:benchmark" + "projectPath" : ":lucene:analysis:stempel" } ], - "6fbc4021" : [ + "152d9f78" : [ { - "configuration" : "testRuntimeClasspath", + "configuration" : "annotationProcessor", "projectPath" : ":lucene:benchmark-jmh" }, - { - "configuration" : "testRuntimeClasspath", - "projectPath" : ":lucene:demo" - }, { "configuration" : "testCompileClasspath", - "projectPath" : ":lucene:expressions" + "projectPath" : ":lucene:benchmark-jmh" }, { "configuration" : "testRuntimeClasspath", - "projectPath" : ":lucene:expressions" + "projectPath" : ":lucene:benchmark-jmh" + } + ], + "180518e6" : [ + { + "configuration" : "testCompileClasspath", + "projectPath" : ":lucene:spatial-extras" }, { "configuration" : "testRuntimeClasspath", - "projectPath" : ":lucene:queries" + "projectPath" : ":lucene:spatial-extras" } ], - "733734f0" : [ + "1d5a4b2b" : [ { "configuration" : "testCompileClasspath", - "projectPath" : ":lucene:analysis.tests" + "projectPath" : ":lucene:benchmark" }, { "configuration" : "testRuntimeClasspath", - "projectPath" : ":lucene:analysis.tests" + "projectPath" : ":lucene:benchmark" }, { "configuration" : "testCompileClasspath", - "projectPath" : ":lucene:luke" + "projectPath" : ":lucene:spatial-extras" }, { "configuration" : "testRuntimeClasspath", + "projectPath" : ":lucene:spatial-extras" + } + ], + "2f760bab" : [ + { + "configuration" : "compileClasspath", "projectPath" : ":lucene:luke" }, { - "configuration" : "testCompileClasspath", - "projectPath" : ":lucene:analysis:phonetic" + "configuration" : "runtimeClasspath", + "projectPath" : ":lucene:luke" }, { - "configuration" : "testRuntimeClasspath", - "projectPath" : ":lucene:analysis:phonetic" + "configuration" : "compileClasspath", + "projectPath" : ":lucene:analysis:opennlp" + }, + { + "configuration" : "runtimeClasspath", + "projectPath" : ":lucene:analysis:opennlp" } ], - "79af844b" : [ + "47ea4550" : [ + { + "configuration" : "compileClasspath", + "projectPath" : ":lucene:benchmark" + }, + { + "configuration" : "runtimeClasspath", + "projectPath" : ":lucene:benchmark" + }, { "configuration" : "compileClasspath", "projectPath" : ":lucene:luke" @@ -398,295 +464,225 @@ }, { "configuration" : "compileClasspath", - "projectPath" : ":lucene:analysis:morfologik" + "projectPath" : ":lucene:analysis:icu" }, { "configuration" : "runtimeClasspath", - "projectPath" : ":lucene:analysis:morfologik" + "projectPath" : ":lucene:analysis:icu" } ], - "85a1e4c6" : [ + "5ce8cdc6" : [ { "configuration" : "compileClasspath", - "projectPath" : ":lucene:benchmark-jmh" + "projectPath" : ":lucene:benchmark" }, { "configuration" : "runtimeClasspath", - "projectPath" : ":lucene:benchmark-jmh" + "projectPath" : ":lucene:benchmark" } ], - "b35e5d7a" : [ - { - "configuration" : "testCompileClasspath", - "projectPath" : ":lucene:analysis.tests" - }, + "6897bc09" : [ { - "configuration" : "testRuntimeClasspath", + "configuration" : "annotationProcessor", "projectPath" : ":lucene:analysis.tests" }, { - "configuration" : "testCompileClasspath", - "projectPath" : ":lucene:backward-codecs" - }, - { - "configuration" : "testRuntimeClasspath", + "configuration" : "annotationProcessor", "projectPath" : ":lucene:backward-codecs" }, { - "configuration" : "testCompileClasspath", - "projectPath" : ":lucene:benchmark" - }, - { - "configuration" : "testRuntimeClasspath", + "configuration" : "annotationProcessor", "projectPath" : ":lucene:benchmark" }, { - "configuration" : "testCompileClasspath", - "projectPath" : ":lucene:classification" + "configuration" : "annotationProcessor", + "projectPath" : ":lucene:benchmark-jmh" }, { - "configuration" : "testRuntimeClasspath", + "configuration" : "annotationProcessor", "projectPath" : ":lucene:classification" }, { - "configuration" : "testCompileClasspath", - "projectPath" : ":lucene:codecs" - }, - { - "configuration" : "testRuntimeClasspath", + "configuration" : "annotationProcessor", "projectPath" : ":lucene:codecs" }, { - "configuration" : "testCompileClasspath", - "projectPath" : ":lucene:core" - }, - { - "configuration" : "testRuntimeClasspath", + "configuration" : "annotationProcessor", "projectPath" : ":lucene:core" }, { - "configuration" : "testCompileClasspath", - "projectPath" : ":lucene:core.tests" - }, - { - "configuration" : "testRuntimeClasspath", + "configuration" : "annotationProcessor", "projectPath" : ":lucene:core.tests" }, { - "configuration" : "testCompileClasspath", - "projectPath" : ":lucene:demo" - }, - { - "configuration" : "testRuntimeClasspath", + "configuration" : "annotationProcessor", "projectPath" : ":lucene:demo" }, { - "configuration" : "testCompileClasspath", - "projectPath" : ":lucene:distribution.tests" - }, - { - "configuration" : "testRuntimeClasspath", + "configuration" : "annotationProcessor", "projectPath" : ":lucene:distribution.tests" }, { - "configuration" : "testCompileClasspath", - "projectPath" : ":lucene:expressions" - }, - { - "configuration" : "testRuntimeClasspath", + "configuration" : "annotationProcessor", "projectPath" : ":lucene:expressions" }, { - "configuration" : "testCompileClasspath", - "projectPath" : ":lucene:facet" - }, - { - "configuration" : "testRuntimeClasspath", + "configuration" : "annotationProcessor", "projectPath" : ":lucene:facet" }, { - "configuration" : "testCompileClasspath", - "projectPath" : ":lucene:grouping" - }, - { - "configuration" : "testRuntimeClasspath", + "configuration" : "annotationProcessor", "projectPath" : ":lucene:grouping" }, { - "configuration" : "testCompileClasspath", - "projectPath" : ":lucene:highlighter" - }, - { - "configuration" : "testRuntimeClasspath", + "configuration" : "annotationProcessor", "projectPath" : ":lucene:highlighter" }, { - "configuration" : "testCompileClasspath", - "projectPath" : ":lucene:join" - }, - { - "configuration" : "testRuntimeClasspath", + "configuration" : "annotationProcessor", "projectPath" : ":lucene:join" }, { - "configuration" : "testCompileClasspath", - "projectPath" : ":lucene:luke" - }, - { - "configuration" : "testRuntimeClasspath", + "configuration" : "annotationProcessor", "projectPath" : ":lucene:luke" }, { - "configuration" : "testCompileClasspath", - "projectPath" : ":lucene:memory" - }, - { - "configuration" : "testRuntimeClasspath", + "configuration" : "annotationProcessor", "projectPath" : ":lucene:memory" }, { - "configuration" : "testCompileClasspath", - "projectPath" : ":lucene:misc" - }, - { - "configuration" : "testRuntimeClasspath", + "configuration" : "annotationProcessor", "projectPath" : ":lucene:misc" }, { - "configuration" : "testCompileClasspath", - "projectPath" : ":lucene:monitor" - }, - { - "configuration" : "testRuntimeClasspath", + "configuration" : "annotationProcessor", "projectPath" : ":lucene:monitor" }, { - "configuration" : "testCompileClasspath", - "projectPath" : ":lucene:queries" - }, - { - "configuration" : "testRuntimeClasspath", + "configuration" : "annotationProcessor", "projectPath" : ":lucene:queries" }, { - "configuration" : "testCompileClasspath", - "projectPath" : ":lucene:queryparser" - }, - { - "configuration" : "testRuntimeClasspath", + "configuration" : "annotationProcessor", "projectPath" : ":lucene:queryparser" }, { - "configuration" : "testCompileClasspath", - "projectPath" : ":lucene:replicator" - }, - { - "configuration" : "testRuntimeClasspath", + "configuration" : "annotationProcessor", "projectPath" : ":lucene:replicator" }, { - "configuration" : "testCompileClasspath", - "projectPath" : ":lucene:sandbox" - }, - { - "configuration" : "testRuntimeClasspath", + "configuration" : "annotationProcessor", "projectPath" : ":lucene:sandbox" }, { - "configuration" : "testCompileClasspath", + "configuration" : "annotationProcessor", "projectPath" : ":lucene:spatial-extras" }, { - "configuration" : "testRuntimeClasspath", - "projectPath" : ":lucene:spatial-extras" + "configuration" : "annotationProcessor", + "projectPath" : ":lucene:spatial-test-fixtures" }, { - "configuration" : "testCompileClasspath", - "projectPath" : ":lucene:spatial-test-fixtures" + "configuration" : "annotationProcessor", + "projectPath" : ":lucene:spatial3d" }, { - "configuration" : "testRuntimeClasspath", - "projectPath" : ":lucene:spatial-test-fixtures" + "configuration" : "annotationProcessor", + "projectPath" : ":lucene:suggest" }, { - "configuration" : "testCompileClasspath", - "projectPath" : ":lucene:spatial3d" + "configuration" : "annotationProcessor", + "projectPath" : ":lucene:test-framework" }, { - "configuration" : "testRuntimeClasspath", - "projectPath" : ":lucene:spatial3d" + "configuration" : "annotationProcessor", + "projectPath" : ":lucene:analysis:common" }, { - "configuration" : "testCompileClasspath", - "projectPath" : ":lucene:suggest" + "configuration" : "annotationProcessor", + "projectPath" : ":lucene:analysis:icu" }, { - "configuration" : "testRuntimeClasspath", - "projectPath" : ":lucene:suggest" + "configuration" : "annotationProcessor", + "projectPath" : ":lucene:analysis:kuromoji" }, { - "configuration" : "testCompileClasspath", - "projectPath" : ":lucene:test-framework" + "configuration" : "annotationProcessor", + "projectPath" : ":lucene:analysis:morfologik" }, { - "configuration" : "testRuntimeClasspath", - "projectPath" : ":lucene:test-framework" + "configuration" : "annotationProcessor", + "projectPath" : ":lucene:analysis:morfologik.tests" }, { - "configuration" : "testCompileClasspath", - "projectPath" : ":lucene:analysis:common" + "configuration" : "annotationProcessor", + "projectPath" : ":lucene:analysis:nori" }, { - "configuration" : "testRuntimeClasspath", - "projectPath" : ":lucene:analysis:common" + "configuration" : "annotationProcessor", + "projectPath" : ":lucene:analysis:opennlp" }, { - "configuration" : "testCompileClasspath", - "projectPath" : ":lucene:analysis:icu" + "configuration" : "annotationProcessor", + "projectPath" : ":lucene:analysis:phonetic" }, { - "configuration" : "testRuntimeClasspath", - "projectPath" : ":lucene:analysis:icu" + "configuration" : "annotationProcessor", + "projectPath" : ":lucene:analysis:smartcn" }, + { + "configuration" : "annotationProcessor", + "projectPath" : ":lucene:analysis:stempel" + } + ], + "6f16ff86" : [ { "configuration" : "testCompileClasspath", - "projectPath" : ":lucene:analysis:kuromoji" + "projectPath" : ":lucene:benchmark" }, { "configuration" : "testRuntimeClasspath", - "projectPath" : ":lucene:analysis:kuromoji" - }, + "projectPath" : ":lucene:benchmark" + } + ], + "6fbc4021" : [ { - "configuration" : "testCompileClasspath", - "projectPath" : ":lucene:analysis:morfologik" + "configuration" : "testRuntimeClasspath", + "projectPath" : ":lucene:benchmark-jmh" }, { "configuration" : "testRuntimeClasspath", - "projectPath" : ":lucene:analysis:morfologik" + "projectPath" : ":lucene:demo" }, { "configuration" : "testCompileClasspath", - "projectPath" : ":lucene:analysis:morfologik.tests" + "projectPath" : ":lucene:expressions" }, { "configuration" : "testRuntimeClasspath", - "projectPath" : ":lucene:analysis:morfologik.tests" + "projectPath" : ":lucene:expressions" }, + { + "configuration" : "testRuntimeClasspath", + "projectPath" : ":lucene:queries" + } + ], + "733734f0" : [ { "configuration" : "testCompileClasspath", - "projectPath" : ":lucene:analysis:nori" + "projectPath" : ":lucene:analysis.tests" }, { "configuration" : "testRuntimeClasspath", - "projectPath" : ":lucene:analysis:nori" + "projectPath" : ":lucene:analysis.tests" }, { "configuration" : "testCompileClasspath", - "projectPath" : ":lucene:analysis:opennlp" + "projectPath" : ":lucene:luke" }, { "configuration" : "testRuntimeClasspath", - "projectPath" : ":lucene:analysis:opennlp" + "projectPath" : ":lucene:luke" }, { "configuration" : "testCompileClasspath", @@ -695,22 +691,34 @@ { "configuration" : "testRuntimeClasspath", "projectPath" : ":lucene:analysis:phonetic" + } + ], + "79af844b" : [ + { + "configuration" : "compileClasspath", + "projectPath" : ":lucene:luke" }, { - "configuration" : "testCompileClasspath", - "projectPath" : ":lucene:analysis:smartcn" + "configuration" : "runtimeClasspath", + "projectPath" : ":lucene:luke" }, { - "configuration" : "testRuntimeClasspath", - "projectPath" : ":lucene:analysis:smartcn" + "configuration" : "compileClasspath", + "projectPath" : ":lucene:analysis:morfologik" }, { - "configuration" : "testCompileClasspath", - "projectPath" : ":lucene:analysis:stempel" + "configuration" : "runtimeClasspath", + "projectPath" : ":lucene:analysis:morfologik" + } + ], + "85a1e4c6" : [ + { + "configuration" : "compileClasspath", + "projectPath" : ":lucene:benchmark-jmh" }, { - "configuration" : "testRuntimeClasspath", - "projectPath" : ":lucene:analysis:stempel" + "configuration" : "runtimeClasspath", + "projectPath" : ":lucene:benchmark-jmh" } ], "b7ba1646" : [