Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Remove the HPPC dependency from all modules and add the required classes to the hppc fork. #13422

Merged
merged 10 commits into from
May 27, 2024
5 changes: 5 additions & 0 deletions lucene/CHANGES.txt
Original file line number Diff line number Diff line change
Expand Up @@ -236,6 +236,11 @@ API Changes

* GITHUB#13145: Deprecate ByteBufferIndexInput as it will be removed in Lucene 10.0. (Uwe Schindler)

* GITHUB#13422: an explicit dependency on the HPPC library is removed in favor of an internal repackaged copy in
oal.internal.hppc. If you relied on HPPC as a transitive dependency, you'll have to add it to your project explicitly.
The HPPC classes now bundled in Lucene core are internal and will have restricted access in future releases, please do
not use them. (Bruno Roustant, Dawid Weiss, Uwe Schindler, Chris Hegarty)

New Features
---------------------

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,11 +20,11 @@
import java.io.Reader;
import org.apache.lucene.analysis.CharFilter; // javadocs
import org.apache.lucene.analysis.util.RollingCharBuffer;
import org.apache.lucene.internal.hppc.CharObjectHashMap;
import org.apache.lucene.util.CharsRef;
import org.apache.lucene.util.fst.CharSequenceOutputs;
import org.apache.lucene.util.fst.FST;
import org.apache.lucene.util.fst.Outputs;
import org.apache.lucene.util.hppc.CharObjectHashMap;

/**
* Simplistic {@link CharFilter} that applies the mappings contained in a {@link NormalizeCharMap}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,14 +19,14 @@
import java.io.IOException;
import java.util.Map;
import java.util.TreeMap;
import org.apache.lucene.internal.hppc.CharObjectHashMap;
import org.apache.lucene.util.CharsRef;
import org.apache.lucene.util.IntsRefBuilder;
import org.apache.lucene.util.fst.CharSequenceOutputs;
import org.apache.lucene.util.fst.FST;
import org.apache.lucene.util.fst.FSTCompiler;
import org.apache.lucene.util.fst.Outputs;
import org.apache.lucene.util.fst.Util;
import org.apache.lucene.util.hppc.CharObjectHashMap;

// TODO: save/load?

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -26,10 +26,10 @@
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
import org.apache.lucene.analysis.tokenattributes.PositionLengthAttribute;
import org.apache.lucene.internal.hppc.IntArrayList;
import org.apache.lucene.internal.hppc.IntCursor;
import org.apache.lucene.util.AttributeSource;
import org.apache.lucene.util.RollingBuffer;
import org.apache.lucene.util.hppc.IntArrayList;
import org.apache.lucene.util.hppc.IntCursor;

/**
* Converts an incoming graph token stream, such as one from {@link SynonymGraphFilter}, into a flat
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,9 @@
import java.util.stream.Collectors;
import org.apache.lucene.analysis.hunspell.SortingStrategy.EntryAccumulator;
import org.apache.lucene.analysis.hunspell.SortingStrategy.EntrySupplier;
import org.apache.lucene.internal.hppc.CharHashSet;
import org.apache.lucene.internal.hppc.IntArrayList;
import org.apache.lucene.internal.hppc.IntCursor;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.IntsRef;
Expand All @@ -58,9 +61,6 @@
import org.apache.lucene.util.fst.FSTCompiler;
import org.apache.lucene.util.fst.IntSequenceOutputs;
import org.apache.lucene.util.fst.Util;
import org.apache.lucene.util.hppc.CharHashSet;
import org.apache.lucene.util.hppc.IntArrayList;
import org.apache.lucene.util.hppc.IntCursor;

/** In-memory structure for the dictionary (.dic) and affix (.aff) data of a hunspell dictionary. */
public class Dictionary {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,10 +17,10 @@
package org.apache.lucene.analysis.hunspell;

import java.util.function.Consumer;
import org.apache.lucene.internal.hppc.IntObjectHashMap;
import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.CharsRef;
import org.apache.lucene.util.IntsRef;
import org.apache.lucene.util.hppc.IntObjectHashMap;

/**
* A cache allowing for CPU-cache-friendlier iteration over {@link WordStorage} entries that can be
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -39,11 +39,11 @@
import java.util.stream.Collectors;
import java.util.stream.Stream;
import org.apache.lucene.analysis.hunspell.AffixedWord.Affix;
import org.apache.lucene.internal.hppc.CharHashSet;
import org.apache.lucene.internal.hppc.CharObjectHashMap;
import org.apache.lucene.util.IntsRef;
import org.apache.lucene.util.fst.FST;
import org.apache.lucene.util.fst.IntsRefFSTEnum;
import org.apache.lucene.util.hppc.CharHashSet;
import org.apache.lucene.util.hppc.CharObjectHashMap;

/**
* A utility class used for generating possible word forms by adding affixes to stems ({@link
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
import java.util.ArrayList;
import java.util.List;
import java.util.function.Consumer;
import org.apache.lucene.internal.hppc.IntArrayList;
import org.apache.lucene.store.ByteArrayDataInput;
import org.apache.lucene.store.ByteArrayDataOutput;
import org.apache.lucene.store.DataOutput;
Expand All @@ -28,7 +29,6 @@
import org.apache.lucene.util.IntsRef;
import org.apache.lucene.util.IntsRefBuilder;
import org.apache.lucene.util.fst.IntSequenceOutputs;
import org.apache.lucene.util.hppc.IntArrayList;

/**
* A data structure for memory-efficient word storage and fast lookup/enumeration. Each dictionary
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,11 +20,11 @@
import java.util.Arrays;
import java.util.Collections;
import java.util.EnumMap;
import org.apache.lucene.internal.hppc.IntArrayList;
import org.apache.lucene.internal.hppc.IntCursor;
import org.apache.lucene.internal.hppc.IntIntHashMap;
import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.fst.FST;
import org.apache.lucene.util.hppc.IntArrayList;
import org.apache.lucene.util.hppc.IntCursor;
import org.apache.lucene.util.hppc.IntIntHashMap;

/** {@link Viterbi} subclass for n-best path calculation. */
public abstract class ViterbiNBest<T extends Token, U extends MorphData>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,9 +21,9 @@
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
import org.apache.lucene.internal.hppc.IntArrayList;
import org.apache.lucene.util.AttributeFactory;
import org.apache.lucene.util.IgnoreRandomChains;
import org.apache.lucene.util.hppc.IntArrayList;

/**
* Tokenizer for domain-like hierarchies.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,8 @@
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
import org.apache.lucene.internal.hppc.IntArrayList;
import org.apache.lucene.internal.hppc.IntHashSet;
import org.apache.lucene.store.ByteArrayDataOutput;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.BytesRefBuilder;
Expand All @@ -37,8 +39,6 @@
import org.apache.lucene.util.fst.FST;
import org.apache.lucene.util.fst.FSTCompiler;
import org.apache.lucene.util.fst.Util;
import org.apache.lucene.util.hppc.IntArrayList;
import org.apache.lucene.util.hppc.IntHashSet;

/**
* A map of synonyms, keys and values are phrases.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,11 +27,11 @@
import java.util.List;
import java.util.Map;
import org.apache.lucene.analysis.TokenizerFactory;
import org.apache.lucene.internal.hppc.IntObjectHashMap;
import org.apache.lucene.util.AttributeFactory;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.ResourceLoader;
import org.apache.lucene.util.ResourceLoaderAware;
import org.apache.lucene.util.hppc.IntObjectHashMap;

/**
* Factory for {@link ICUTokenizer}. Words are broken across script boundaries, then segmented
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
package org.apache.lucene.analysis.ja;

import java.util.Map;
import org.apache.lucene.util.hppc.CharObjectHashMap;
import org.apache.lucene.internal.hppc.CharObjectHashMap;

/** Utility methods for Japanese filters. */
class JapaneseFilterUtil {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@
import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.util.hppc.CharObjectHashMap;
import org.apache.lucene.internal.hppc.CharObjectHashMap;

/**
* A {@link TokenFilter} that normalizes small letters (捨て仮名) in hiragana into normal letters. For
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@
import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.util.hppc.CharObjectHashMap;
import org.apache.lucene.internal.hppc.CharObjectHashMap;

/**
* A {@link TokenFilter} that normalizes small letters (捨て仮名) in katakana into normal letters. For
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,9 +19,9 @@
import java.util.ArrayList;
import java.util.List;
import org.apache.lucene.analysis.cn.smart.Utility;
import org.apache.lucene.util.hppc.IntArrayList;
import org.apache.lucene.util.hppc.IntObjectHashMap;
import org.apache.lucene.util.hppc.ObjectCursor;
import org.apache.lucene.internal.hppc.IntArrayList;
import org.apache.lucene.internal.hppc.IntObjectHashMap;
import org.apache.lucene.internal.hppc.ObjectCursor;

/**
* Graph representing possible token pairs (bigrams) at each start offset in the sentence.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@

import java.util.ArrayList;
import java.util.List;
import org.apache.lucene.util.hppc.IntObjectHashMap;
import org.apache.lucene.internal.hppc.IntObjectHashMap;

/**
* Graph representing possible tokens at each start offset in the sentence.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@ created by Leo Galambos (Leo.G@seznam.cz).
import java.util.Arrays;
import java.util.Iterator;
import java.util.List;
import org.apache.lucene.util.hppc.ObjectCursor;
import org.apache.lucene.internal.hppc.ObjectCursor;

/**
* The Gener object helps in the discarding of nodes which break the reduction effort and defend the
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@ created by Leo Galambos (Leo.G@seznam.cz).
import java.util.Arrays;
import java.util.Iterator;
import java.util.List;
import org.apache.lucene.util.hppc.ObjectCursor;
import org.apache.lucene.internal.hppc.ObjectCursor;

/**
* The Lift class is a data structure that is a variation of a Patricia trie.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@ created by Leo Galambos (Leo.G@seznam.cz).
import java.util.Arrays;
import java.util.Iterator;
import java.util.List;
import org.apache.lucene.util.hppc.CharCursor;
import org.apache.lucene.internal.hppc.CharCursor;

/**
* The Optimizer class is a Trie that will be reduced (have empty rows removed).
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -58,8 +58,8 @@ created by Leo Galambos (Leo.G@seznam.cz).
import java.util.Arrays;
import java.util.Iterator;
import java.util.List;
import org.apache.lucene.util.hppc.CharCursor;
import org.apache.lucene.util.hppc.ObjectCursor;
import org.apache.lucene.internal.hppc.CharCursor;
import org.apache.lucene.internal.hppc.ObjectCursor;

/** The Reduce object is used to remove gaps in a Trie which stores a dictionary. */
public class Reduce {
Expand Down
6 changes: 3 additions & 3 deletions lucene/analysis/stempel/src/java/org/egothor/stemmer/Row.java
Original file line number Diff line number Diff line change
Expand Up @@ -59,9 +59,9 @@ created by Leo Galambos (Leo.G@seznam.cz).
import java.io.IOException;
import java.io.PrintStream;
import java.util.Iterator;
import org.apache.lucene.util.hppc.CharCursor;
import org.apache.lucene.util.hppc.CharObjectHashMap;
import org.apache.lucene.util.hppc.ObjectCursor;
import org.apache.lucene.internal.hppc.CharCursor;
import org.apache.lucene.internal.hppc.CharObjectHashMap;
import org.apache.lucene.internal.hppc.ObjectCursor;

/** The Row class represents a row in a matrix representation of a trie. */
public class Row {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,14 +23,14 @@
import org.apache.lucene.benchmark.byTask.utils.Config;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.internal.hppc.IntObjectHashMap;
import org.apache.lucene.spatial.SpatialStrategy;
import org.apache.lucene.spatial.composite.CompositeSpatialStrategy;
import org.apache.lucene.spatial.prefix.RecursivePrefixTreeStrategy;
import org.apache.lucene.spatial.prefix.tree.PackedQuadPrefixTree;
import org.apache.lucene.spatial.prefix.tree.SpatialPrefixTree;
import org.apache.lucene.spatial.prefix.tree.SpatialPrefixTreeFactory;
import org.apache.lucene.spatial.serialized.SerializedDVStrategy;
import org.apache.lucene.util.hppc.IntObjectHashMap;
import org.locationtech.spatial4j.context.SpatialContext;
import org.locationtech.spatial4j.context.SpatialContextFactory;
import org.locationtech.spatial4j.shape.Point;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@
import java.util.Map;
import java.util.Properties;
import java.util.StringTokenizer;
import org.apache.lucene.util.hppc.IntArrayList;
import org.apache.lucene.internal.hppc.IntArrayList;

/**
* Perf run configuration properties.
Expand Down
19 changes: 11 additions & 8 deletions lucene/core/src/java/module-info.java
Original file line number Diff line number Diff line change
Expand Up @@ -23,36 +23,39 @@
requires java.logging;
requires static jdk.management; // this is optional but explicit declaration is recommended

exports org.apache.lucene.analysis;
exports org.apache.lucene.analysis.standard;
exports org.apache.lucene.analysis.tokenattributes;
exports org.apache.lucene.codecs;
exports org.apache.lucene.analysis;
exports org.apache.lucene.codecs.compressing;
exports org.apache.lucene.codecs.lucene90.blocktree;
exports org.apache.lucene.codecs.lucene90.compressing;
exports org.apache.lucene.codecs.lucene90;
exports org.apache.lucene.codecs.lucene94;
exports org.apache.lucene.codecs.lucene95;
exports org.apache.lucene.codecs.lucene99;
exports org.apache.lucene.codecs.lucene90.blocktree;
exports org.apache.lucene.codecs.lucene90.compressing;
exports org.apache.lucene.codecs.perfield;
exports org.apache.lucene.codecs;
exports org.apache.lucene.document;
exports org.apache.lucene.geo;
exports org.apache.lucene.index;
exports org.apache.lucene.search;
exports org.apache.lucene.search.comparators;
exports org.apache.lucene.search.similarities;
exports org.apache.lucene.search.knn;
exports org.apache.lucene.search.similarities;
exports org.apache.lucene.search;
exports org.apache.lucene.store;
exports org.apache.lucene.util;
exports org.apache.lucene.util.automaton;
exports org.apache.lucene.util.bkd;
exports org.apache.lucene.util.compress;
exports org.apache.lucene.util.fst;
exports org.apache.lucene.util.graph;
exports org.apache.lucene.util.hnsw;
exports org.apache.lucene.util.hppc;
exports org.apache.lucene.util.mutable;
exports org.apache.lucene.util.packed;
exports org.apache.lucene.util;

// Temporarily export HPPC to all modules (eventually, this
// should be restricted to only Lucene modules)
exports org.apache.lucene.internal.hppc;

// Only export internal packages to the test framework.
exports org.apache.lucene.internal.tests to
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,11 +25,11 @@
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
import org.apache.lucene.analysis.tokenattributes.PositionLengthAttribute;
import org.apache.lucene.internal.hppc.IntArrayList;
import org.apache.lucene.internal.hppc.IntCursor;
import org.apache.lucene.internal.hppc.IntIntHashMap;
import org.apache.lucene.util.automaton.Automaton;
import org.apache.lucene.util.automaton.Transition;
import org.apache.lucene.util.hppc.IntArrayList;
import org.apache.lucene.util.hppc.IntCursor;
import org.apache.lucene.util.hppc.IntIntHashMap;

/** Converts an Automaton into a TokenStream. */
public class AutomatonToTokenStream {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,8 @@
import org.apache.lucene.index.SortedNumericDocValues;
import org.apache.lucene.index.SortedSetDocValues;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.internal.hppc.LongHashSet;
import org.apache.lucene.internal.hppc.LongIntHashMap;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.SortedSetSelector;
import org.apache.lucene.store.ByteArrayDataOutput;
Expand All @@ -50,8 +52,6 @@
import org.apache.lucene.util.MathUtil;
import org.apache.lucene.util.StringHelper;
import org.apache.lucene.util.compress.LZ4;
import org.apache.lucene.util.hppc.LongHashSet;
import org.apache.lucene.util.hppc.LongIntHashMap;
import org.apache.lucene.util.packed.DirectMonotonicWriter;
import org.apache.lucene.util.packed.DirectWriter;

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -29,11 +29,11 @@
import org.apache.lucene.index.IndexFileNames;
import org.apache.lucene.index.NumericDocValues;
import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.internal.hppc.IntObjectHashMap;
import org.apache.lucene.store.ChecksumIndexInput;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.RandomAccessInput;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.hppc.IntObjectHashMap;

/** Reader for {@link Lucene90NormsFormat} */
final class Lucene90NormsProducer extends NormsProducer implements Cloneable {
Expand Down
Loading