Upgrade Anserini to Lucene (and Solr) 9.0.0 (#1875)

castorini · Aug 1, 2022 · 5af657d · 5af657d
1 parent 38d8581
commit 5af657d
Show file tree

Hide file tree

Showing 22 changed files with 148 additions and 121 deletions.
diff --git a/pom.xml b/pom.xml
@@ -26,7 +26,8 @@
   </developers>
 
   <properties>
-    <lucene.version>8.11.0</lucene.version>
+    <lucene.version>9.0.0</lucene.version>
+    <solr.version>9.0.0</solr.version>
     <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
   </properties>
 
@@ -297,12 +298,17 @@
     </dependency>
     <dependency>
       <groupId>org.apache.lucene</groupId>
-      <artifactId>lucene-analyzers-kuromoji</artifactId>
+      <artifactId>lucene-queries</artifactId>
+      <version>${lucene.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.lucene</groupId>
+      <artifactId>lucene-analysis-kuromoji</artifactId>
       <version>${lucene.version}</version>
     </dependency>
     <dependency> <!-- polish and ukrainian -->
       <groupId>org.apache.lucene</groupId>
-      <artifactId>lucene-analyzers-morfologik</artifactId>
+      <artifactId>lucene-analysis-morfologik</artifactId>
       <version>${lucene.version}</version>
     </dependency>
     <dependency> <!-- only needed for testing -->
@@ -320,13 +326,62 @@
     <dependency>
       <groupId>org.apache.solr</groupId>
       <artifactId>solr-solrj</artifactId>
-      <version>${lucene.version}</version>
+      <version>${solr.version}</version>
+      <exclusions>
+        <exclusion>
+          <groupId>org.apache.lucene</groupId>
+          <artifactId>lucene-core</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>org.apache.lucene</groupId>
+          <artifactId>lucene-analysis-common</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>org.apache.lucene</groupId>
+          <artifactId>lucene-queries</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>org.slf4j</groupId>
+          <artifactId>slf4j-api</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>org.apache.logging.log4j</groupId>
+          <artifactId>log4j-slf4j-impl</artifactId>
+        </exclusion>
+      </exclusions>
     </dependency>
     <dependency> <!-- only needed for testing -->
       <groupId>org.apache.solr</groupId>
       <artifactId>solr-test-framework</artifactId>
-      <version>${lucene.version}</version>
+      <version>${solr.version}</version>
       <scope>test</scope>
+      <exclusions>
+        <exclusion>
+          <groupId>org.apache.lucene</groupId>
+          <artifactId>lucene-core</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>org.apache.lucene</groupId>
+          <artifactId>lucene-analysis-common</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>org.apache.lucene</groupId>
+          <artifactId>lucene-queries</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>org.slf4j</groupId>
+          <artifactId>slf4j-api</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>org.apache.logging.log4j</groupId>
+          <artifactId>log4j-slf4j-impl</artifactId>
+        </exclusion>
+      </exclusions>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.lucene</groupId>
+      <artifactId>lucene-codecs</artifactId>
+      <version>${lucene.version}</version>
     </dependency>
     <dependency>
       <groupId>org.elasticsearch.client</groupId>

diff --git a/src/main/java/io/anserini/index/IndexCollection.java b/src/main/java/io/anserini/index/IndexCollection.java
@@ -85,6 +85,7 @@
 import org.apache.lucene.store.FSDirectory;
 import org.apache.solr.client.solrj.SolrClient;
 import org.apache.solr.client.solrj.impl.CloudSolrClient;
+import org.apache.solr.client.solrj.impl.Http2SolrClient;
 import org.apache.solr.common.SolrInputDocument;
 import org.elasticsearch.action.DocWriteRequest;
 import org.elasticsearch.action.bulk.BulkRequest;
@@ -411,8 +412,7 @@ private class SolrClientFactory extends BasePooledObjectFactory<SolrClient> {
     @Override
     public SolrClient create() {
       return new CloudSolrClient.Builder(Splitter.on(',').splitToList(args.zkUrl), Optional.of(args.zkChroot))
-          .withConnectionTimeout(TIMEOUT)
-          .withSocketTimeout(TIMEOUT)
+          .withInternalClientBuilder(new Http2SolrClient.Builder().connectionTimeout(TIMEOUT))
           .build();
     }
 

diff --git a/src/main/java/io/anserini/index/generator/AclAnthologyGenerator.java b/src/main/java/io/anserini/index/generator/AclAnthologyGenerator.java
@@ -23,11 +23,11 @@
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.analysis.CharArraySet;
 import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.document.BinaryDocValuesField;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
 import org.apache.lucene.document.FieldType;
 import org.apache.lucene.document.IntPoint;
-import org.apache.lucene.document.SortedDocValuesField;
 import org.apache.lucene.document.StoredField;
 import org.apache.lucene.document.StringField;
 import org.apache.lucene.index.IndexOptions;
@@ -102,7 +102,7 @@ public Document createDocument(AclAnthology.Document aclDoc) throws GeneratorExc
     // Store the collection docid.
     doc.add(new StringField(IndexArgs.ID, id, Field.Store.YES));
     // This is needed to break score ties by docid.
-    doc.add(new SortedDocValuesField(IndexArgs.ID, new BytesRef(id)));
+    doc.add(new BinaryDocValuesField(IndexArgs.ID, new BytesRef(id)));
 
     if (args.storeRaw) {
       doc.add(new StoredField(IndexArgs.RAW, aclDoc.raw()));

diff --git a/src/main/java/io/anserini/index/generator/BibtexGenerator.java b/src/main/java/io/anserini/index/generator/BibtexGenerator.java
@@ -22,11 +22,11 @@
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.analysis.CharArraySet;
 import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.document.BinaryDocValuesField;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
 import org.apache.lucene.document.FieldType;
 import org.apache.lucene.document.IntPoint;
-import org.apache.lucene.document.SortedDocValuesField;
 import org.apache.lucene.document.StoredField;
 import org.apache.lucene.document.StringField;
 import org.apache.lucene.index.IndexOptions;
@@ -101,7 +101,7 @@ public Document createDocument(BibtexCollection.Document bibtexDoc) throws Gener
     // Store the collection docid.
     doc.add(new StringField(IndexArgs.ID, id, Field.Store.YES));
     // This is needed to break score ties by docid.
-    doc.add(new SortedDocValuesField(IndexArgs.ID, new BytesRef(id)));
+    doc.add(new BinaryDocValuesField(IndexArgs.ID, new BytesRef(id)));
     // Store the collection's bibtex type
     doc.add(new StringField(TYPE, type, Field.Store.YES));
 

diff --git a/src/main/java/io/anserini/index/generator/Cord19Generator.java b/src/main/java/io/anserini/index/generator/Cord19Generator.java
@@ -24,11 +24,11 @@
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.analysis.CharArraySet;
 import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.document.BinaryDocValuesField;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
 import org.apache.lucene.document.FieldType;
 import org.apache.lucene.document.IntPoint;
-import org.apache.lucene.document.SortedDocValuesField;
 import org.apache.lucene.document.StoredField;
 import org.apache.lucene.document.StringField;
 import org.apache.lucene.index.IndexOptions;
@@ -115,7 +115,7 @@ public Document createDocument(Cord19BaseDocument covidDoc) throws GeneratorExce
     // Store the collection docid.
     doc.add(new StringField(IndexArgs.ID, id, Field.Store.YES));
     // This is needed to break score ties by docid.
-    doc.add(new SortedDocValuesField(IndexArgs.ID, new BytesRef(id)));
+    doc.add(new BinaryDocValuesField(IndexArgs.ID, new BytesRef(id)));
 
     if (args.storeRaw) {
       doc.add(new StoredField(IndexArgs.RAW, raw));

diff --git a/src/main/java/io/anserini/index/generator/CoreGenerator.java b/src/main/java/io/anserini/index/generator/CoreGenerator.java
@@ -23,11 +23,11 @@
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.analysis.CharArraySet;
 import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.document.BinaryDocValuesField;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
 import org.apache.lucene.document.FieldType;
 import org.apache.lucene.document.IntPoint;
-import org.apache.lucene.document.SortedDocValuesField;
 import org.apache.lucene.document.StoredField;
 import org.apache.lucene.document.StringField;
 import org.apache.lucene.index.IndexOptions;
@@ -106,7 +106,7 @@ public Document createDocument(CoreCollection.Document coreDoc) throws Generator
     // Store the collection docid.
     doc.add(new StringField(IndexArgs.ID, id, Field.Store.YES));
     // This is needed to break score ties by docid.
-    doc.add(new SortedDocValuesField(IndexArgs.ID, new BytesRef(id)));
+    doc.add(new BinaryDocValuesField(IndexArgs.ID, new BytesRef(id)));
 
     if (args.storeRaw) {
       doc.add(new StoredField(IndexArgs.RAW, coreDoc.raw()));
@@ -165,10 +165,10 @@ private void addDocumentField(Document doc, String key, JsonNode value, FieldTyp
       // index as numeric value to allow range queries
       try {
         doc.add(new IntPoint(key, Integer.parseInt(valueText)));
+        doc.add(new StoredField(key, valueText));
       } catch(Exception e) {
         // year is not numeric value
       }
-      doc.add(new StoredField(key, valueText));
     } else {
       doc.add(new Field(key, valueText, fieldType));
     }

diff --git a/src/main/java/io/anserini/index/generator/DefaultLuceneDocumentGenerator.java b/src/main/java/io/anserini/index/generator/DefaultLuceneDocumentGenerator.java
@@ -21,10 +21,10 @@
 import io.anserini.collection.SourceDocument;
 import io.anserini.index.IndexArgs;
 import org.apache.commons.lang3.ArrayUtils;
+import org.apache.lucene.document.BinaryDocValuesField;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
 import org.apache.lucene.document.FieldType;
-import org.apache.lucene.document.SortedDocValuesField;
 import org.apache.lucene.document.StoredField;
 import org.apache.lucene.document.StringField;
 import org.apache.lucene.index.IndexOptions;
@@ -74,7 +74,7 @@ public Document createDocument(T src) throws GeneratorException {
     // Store the collection docid.
     document.add(new StringField(IndexArgs.ID, id, Field.Store.YES));
     // This is needed to break score ties by docid.
-    document.add(new SortedDocValuesField(IndexArgs.ID, new BytesRef(id)));
+    document.add(new BinaryDocValuesField(IndexArgs.ID, new BytesRef(id)));
 
     if (args.storeRaw) {
       document.add(new StoredField(IndexArgs.RAW, src.raw()));

diff --git a/src/main/java/io/anserini/index/generator/EpidemicQAGenerator.java b/src/main/java/io/anserini/index/generator/EpidemicQAGenerator.java
@@ -22,10 +22,10 @@
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.analysis.CharArraySet;
 import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.document.BinaryDocValuesField;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
 import org.apache.lucene.document.FieldType;
-import org.apache.lucene.document.SortedDocValuesField;
 import org.apache.lucene.document.StoredField;
 import org.apache.lucene.document.StringField;
 import org.apache.lucene.index.IndexOptions;
@@ -76,7 +76,7 @@ public Document createDocument(EpidemicQACollection.Document covidDoc) throws Ge
     // Store the collection docid.
     doc.add(new StringField(IndexArgs.ID, id, Field.Store.YES));
     // This is needed to break score ties by docid.
-    doc.add(new SortedDocValuesField(IndexArgs.ID, new BytesRef(id)));
+    doc.add(new BinaryDocValuesField(IndexArgs.ID, new BytesRef(id)));
 
     if (args.storeRaw) {
       doc.add(new StoredField(IndexArgs.RAW, raw));

diff --git a/src/main/java/io/anserini/rerank/ScoredDocuments.java b/src/main/java/io/anserini/rerank/ScoredDocuments.java
@@ -17,12 +17,12 @@
 package io.anserini.rerank;
 
 import io.anserini.index.IndexArgs;
+import org.apache.lucene.document.BinaryDocValuesField;
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
 import org.apache.lucene.document.StringField;
-import org.apache.lucene.document.SortedDocValuesField;
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.TermQuery;
@@ -97,7 +97,7 @@ public static ScoredDocuments fromSolrDocs(SolrDocumentList rs) {
       // Store the collection docid.
       document.add(new StringField(IndexArgs.ID, id, Field.Store.YES));
       // This is needed to break score ties by docid.
-      document.add(new SortedDocValuesField(IndexArgs.ID, new BytesRef(id)));
+      document.add(new BinaryDocValuesField(IndexArgs.ID, new BytesRef(id)));
       scoredDocs.documents[i] = document;
       scoredDocs.scores[i] = score;
       scoredDocs.ids[i] = i; // no internal Lucene ID available, use index as placeholder
@@ -130,7 +130,7 @@ public static ScoredDocuments fromESDocs(SearchHits rs) {
       // Store the collection docid.
       document.add(new StringField(IndexArgs.ID, id, Field.Store.YES));
       // This is needed to break score ties by docid.
-      document.add(new SortedDocValuesField(IndexArgs.ID, new BytesRef(id)));
+      document.add(new BinaryDocValuesField(IndexArgs.ID, new BytesRef(id)));
       scoredDocs.documents[i] = document;
       scoredDocs.scores[i] = score;
       scoredDocs.ids[i] = i; // no internal Lucene ID available, use index as placeholder

diff --git a/src/main/java/io/anserini/search/SearchSolr.java b/src/main/java/io/anserini/search/SearchSolr.java
@@ -31,6 +31,7 @@
 import org.apache.solr.client.solrj.SolrQuery;
 import org.apache.solr.client.solrj.SolrQuery.SortClause;
 import org.apache.solr.client.solrj.impl.CloudSolrClient;
+import org.apache.solr.client.solrj.impl.Http2SolrClient;
 import org.apache.solr.client.solrj.response.QueryResponse;
 import org.apache.solr.common.SolrDocumentList;
 import org.kohsuke.args4j.CmdLineException;
@@ -165,8 +166,7 @@ public SearchSolr(Args args) throws IOException {
     LOG.info("Solr ZooKeeper URL: " + args.zkUrl);
     this.client = new CloudSolrClient.Builder(Splitter.on(',')
             .splitToList(args.zkUrl), Optional.of(args.zkChroot))
-            .withConnectionTimeout(TIMEOUT)
-            .withSocketTimeout(TIMEOUT)
+            .withInternalClientBuilder(new Http2SolrClient.Builder().connectionTimeout(TIMEOUT))
             .build();
   }
 

diff --git a/src/main/java/io/anserini/search/query/SdmQueryGenerator.java b/src/main/java/io/anserini/search/query/SdmQueryGenerator.java
@@ -19,14 +19,14 @@
 import io.anserini.analysis.AnalyzerUtils;
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.index.Term;
+import org.apache.lucene.queries.spans.SpanNearQuery;
+import org.apache.lucene.queries.spans.SpanQuery;
+import org.apache.lucene.queries.spans.SpanTermQuery;
 import org.apache.lucene.search.BooleanClause;
 import org.apache.lucene.search.BooleanQuery;
 import org.apache.lucene.search.BoostQuery;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.TermQuery;
-import org.apache.lucene.search.spans.SpanNearQuery;
-import org.apache.lucene.search.spans.SpanQuery;
-import org.apache.lucene.search.spans.SpanTermQuery;
 
 import java.util.List;