From e1d6e70efd380c05a159ce897d82ced4b720cb76 Mon Sep 17 00:00:00 2001 From: Christine Poerschke Date: Tue, 10 Sep 2024 09:50:15 +0100 Subject: [PATCH 1/8] add TFSimilarity class --- .../search/similarities/TFSimilarity.java | 41 ++++++++ .../lucene/search/TestConjunctions.java | 23 +---- .../search/similarities/TestTFSimilarity.java | 93 +++++++++++++++++++ 3 files changed, 135 insertions(+), 22 deletions(-) create mode 100644 lucene/core/src/java/org/apache/lucene/search/similarities/TFSimilarity.java create mode 100644 lucene/core/src/test/org/apache/lucene/search/similarities/TestTFSimilarity.java diff --git a/lucene/core/src/java/org/apache/lucene/search/similarities/TFSimilarity.java b/lucene/core/src/java/org/apache/lucene/search/similarities/TFSimilarity.java new file mode 100644 index 000000000000..bd4089b5c214 --- /dev/null +++ b/lucene/core/src/java/org/apache/lucene/search/similarities/TFSimilarity.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.search.similarities; + +import org.apache.lucene.index.FieldInvertState; +import org.apache.lucene.search.CollectionStatistics; +import org.apache.lucene.search.TermStatistics; + +/** Similarity that returns the TF as score */ +public class TFSimilarity extends Similarity { + + @Override + public long computeNorm(FieldInvertState state) { + return 1; // we dont care + } + + @Override + public SimScorer scorer( + float boost, CollectionStatistics collectionStats, TermStatistics... termStats) { + return new SimScorer() { + @Override + public float score(float freq, long norm) { + return freq; + } + }; + } +} diff --git a/lucene/core/src/test/org/apache/lucene/search/TestConjunctions.java b/lucene/core/src/test/org/apache/lucene/search/TestConjunctions.java index 675aa959b665..6f1c0deb6865 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestConjunctions.java +++ b/lucene/core/src/test/org/apache/lucene/search/TestConjunctions.java @@ -29,12 +29,11 @@ import org.apache.lucene.document.StringField; import org.apache.lucene.document.TextField; import org.apache.lucene.index.DirectoryReader; -import org.apache.lucene.index.FieldInvertState; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.index.Term; -import org.apache.lucene.search.similarities.Similarity; +import org.apache.lucene.search.similarities.TFSimilarity; import org.apache.lucene.store.Directory; import org.apache.lucene.tests.analysis.MockAnalyzer; import org.apache.lucene.tests.index.RandomIndexWriter; @@ -93,26 +92,6 @@ public void tearDown() throws Exception { super.tearDown(); } - // Similarity that returns the TF as score - private static class TFSimilarity extends Similarity { - - @Override - public long computeNorm(FieldInvertState state) { - return 1; // we dont care - } - - @Override - public SimScorer scorer( - float boost, CollectionStatistics collectionStats, TermStatistics... termStats) { - return new SimScorer() { - @Override - public float score(float freq, long norm) { - return freq; - } - }; - } - } - public void testScorerGetChildren() throws Exception { Directory dir = newDirectory(); IndexWriter w = new IndexWriter(dir, newIndexWriterConfig()); diff --git a/lucene/core/src/test/org/apache/lucene/search/similarities/TestTFSimilarity.java b/lucene/core/src/test/org/apache/lucene/search/similarities/TestTFSimilarity.java new file mode 100644 index 000000000000..90c438f22475 --- /dev/null +++ b/lucene/core/src/test/org/apache/lucene/search/similarities/TestTFSimilarity.java @@ -0,0 +1,93 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.search.similarities; + +import java.io.IOException; +import java.util.Random; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.Term; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.TermQuery; +import org.apache.lucene.search.TopDocs; +import org.apache.lucene.store.Directory; +import org.apache.lucene.tests.search.similarities.BaseSimilarityTestCase; +import org.apache.lucene.tests.util.LuceneTestCase; +import org.apache.lucene.util.IOUtils; + +public class TestTFSimilarity extends BaseSimilarityTestCase { + + private Directory directory; + private IndexReader indexReader; + private IndexSearcher indexSearcher; + + @Override + protected Similarity getSimilarity(Random random) { + return new TFSimilarity(); + } + + @Override + public void setUp() throws Exception { + super.setUp(); + directory = newDirectory(); + try (IndexWriter indexWriter = new IndexWriter(directory, newIndexWriterConfig())) { + final Document document1 = new Document(); + final Document document2 = new Document(); + final Document document3 = new Document(); + document1.add(LuceneTestCase.newTextField("test", "one", Field.Store.YES)); + document2.add(LuceneTestCase.newTextField("test", "two two", Field.Store.YES)); + document3.add(LuceneTestCase.newTextField("test", "three three three", Field.Store.YES)); + indexWriter.addDocument(document1); + indexWriter.addDocument(document2); + indexWriter.addDocument(document3); + indexWriter.commit(); + } + indexReader = DirectoryReader.open(directory); + indexSearcher = newSearcher(indexReader); + indexSearcher.setSimilarity(new TFSimilarity()); + } + + @Override + public void tearDown() throws Exception { + IOUtils.close(indexReader, directory); + super.tearDown(); + } + + public void testOne() throws IOException { + implTest("one", 1f); + } + + public void testTwo() throws IOException { + implTest("two", 2f); + } + + public void testThree() throws IOException { + implTest("three", 3f); + } + + private void implTest(String text, float expectedScore) throws IOException { + Query query = new TermQuery(new Term("test", text)); + TopDocs topDocs = indexSearcher.search(query, 1); + assertEquals(1, topDocs.totalHits.value); + assertEquals(1, topDocs.scoreDocs.length); + assertEquals(expectedScore, topDocs.scoreDocs[0].score, 0.0); + } +} From 81775ab6db0ffba0d73ef2ec7df7ad3b2b3864f1 Mon Sep 17 00:00:00 2001 From: Christine Poerschke Date: Tue, 10 Sep 2024 11:26:59 +0100 Subject: [PATCH 2/8] Update lucene/core/src/java/org/apache/lucene/search/similarities/TFSimilarity.java --- .../org/apache/lucene/search/similarities/TFSimilarity.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lucene/core/src/java/org/apache/lucene/search/similarities/TFSimilarity.java b/lucene/core/src/java/org/apache/lucene/search/similarities/TFSimilarity.java index bd4089b5c214..3d66b3cf46ee 100644 --- a/lucene/core/src/java/org/apache/lucene/search/similarities/TFSimilarity.java +++ b/lucene/core/src/java/org/apache/lucene/search/similarities/TFSimilarity.java @@ -34,7 +34,7 @@ public SimScorer scorer( return new SimScorer() { @Override public float score(float freq, long norm) { - return freq; + return boost * freq; } }; } From d2a91c6668d98e1b6b1fc097c6bb8f87f4a3aac8 Mon Sep 17 00:00:00 2001 From: Christine Poerschke Date: Tue, 10 Sep 2024 11:42:03 +0100 Subject: [PATCH 3/8] add TestTFSimilarity.testBoostQuery() --- .../lucene/search/similarities/TestTFSimilarity.java | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/lucene/core/src/test/org/apache/lucene/search/similarities/TestTFSimilarity.java b/lucene/core/src/test/org/apache/lucene/search/similarities/TestTFSimilarity.java index 90c438f22475..6e7502e3cb06 100644 --- a/lucene/core/src/test/org/apache/lucene/search/similarities/TestTFSimilarity.java +++ b/lucene/core/src/test/org/apache/lucene/search/similarities/TestTFSimilarity.java @@ -24,6 +24,7 @@ import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.Term; +import org.apache.lucene.search.BoostQuery; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Query; import org.apache.lucene.search.TermQuery; @@ -90,4 +91,13 @@ private void implTest(String text, float expectedScore) throws IOException { assertEquals(1, topDocs.scoreDocs.length); assertEquals(expectedScore, topDocs.scoreDocs[0].score, 0.0); } + + public void testBoostQuery() throws IOException { + Query query = new TermQuery(new Term("test", "three")); + float boost = 14f; + TopDocs topDocs = indexSearcher.search(new BoostQuery(query, boost), 1); + assertEquals(1, topDocs.totalHits.value); + assertEquals(1, topDocs.scoreDocs.length); + assertEquals(42f, topDocs.scoreDocs[0].score, 0.0); + } } From 64559fff9c2c61a743c74e3b3b8f0696a2c180c5 Mon Sep 17 00:00:00 2001 From: Christine Poerschke Date: Wed, 11 Sep 2024 08:21:27 +0100 Subject: [PATCH 4/8] replace TestBooleanQueryVisitSubscorers.CountingSimilarity with TFSimilarity --- .../TestBooleanQueryVisitSubscorers.java | 25 ++----------------- 1 file changed, 2 insertions(+), 23 deletions(-) diff --git a/lucene/core/src/test/org/apache/lucene/search/TestBooleanQueryVisitSubscorers.java b/lucene/core/src/test/org/apache/lucene/search/TestBooleanQueryVisitSubscorers.java index b291a96c8024..f9055c3bf5a9 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestBooleanQueryVisitSubscorers.java +++ b/lucene/core/src/test/org/apache/lucene/search/TestBooleanQueryVisitSubscorers.java @@ -29,14 +29,13 @@ import org.apache.lucene.document.Document; import org.apache.lucene.document.Field.Store; import org.apache.lucene.document.TextField; -import org.apache.lucene.index.FieldInvertState; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.Term; import org.apache.lucene.search.BooleanClause.Occur; import org.apache.lucene.search.similarities.ClassicSimilarity; -import org.apache.lucene.search.similarities.Similarity; +import org.apache.lucene.search.similarities.TFSimilarity; import org.apache.lucene.store.Directory; import org.apache.lucene.tests.analysis.MockAnalyzer; import org.apache.lucene.tests.index.RandomIndexWriter; @@ -75,7 +74,7 @@ public void setUp() throws Exception { searcher = newSearcher(reader, true, false); searcher.setSimilarity(new ClassicSimilarity()); scorerSearcher = new ScorerIndexSearcher(reader); - scorerSearcher.setSimilarity(new CountingSimilarity()); + scorerSearcher.setSimilarity(new TFSimilarity()); } @Override @@ -345,24 +344,4 @@ private static StringBuilder indent(final StringBuilder builder, final int inden return builder; } } - - // Similarity that just returns the frequency as the score - private static class CountingSimilarity extends Similarity { - - @Override - public long computeNorm(FieldInvertState state) { - return 1; - } - - @Override - public SimScorer scorer( - float boost, CollectionStatistics collectionStats, TermStatistics... termStats) { - return new SimScorer() { - @Override - public float score(float freq, long norm) { - return freq; - } - }; - } - } } From b8feaa2664f36d79384f0a5ef35dc6f16bccfc63 Mon Sep 17 00:00:00 2001 From: Christine Poerschke Date: Wed, 11 Sep 2024 08:27:16 +0100 Subject: [PATCH 5/8] action code review feedback w.r.t. TFSimilarity.computeNorm --- .../search/similarities/TFSimilarity.java | 37 ++++++++++++++++++- 1 file changed, 36 insertions(+), 1 deletion(-) diff --git a/lucene/core/src/java/org/apache/lucene/search/similarities/TFSimilarity.java b/lucene/core/src/java/org/apache/lucene/search/similarities/TFSimilarity.java index 3d66b3cf46ee..0b5d9139dbc4 100644 --- a/lucene/core/src/java/org/apache/lucene/search/similarities/TFSimilarity.java +++ b/lucene/core/src/java/org/apache/lucene/search/similarities/TFSimilarity.java @@ -17,15 +17,50 @@ package org.apache.lucene.search.similarities; import org.apache.lucene.index.FieldInvertState; +import org.apache.lucene.index.IndexOptions; import org.apache.lucene.search.CollectionStatistics; import org.apache.lucene.search.TermStatistics; +import org.apache.lucene.util.SmallFloat; /** Similarity that returns the TF as score */ public class TFSimilarity extends Similarity { + private final boolean discountOverlaps; + + /** + * TFSimilarity with these default values: + * + *
    + *
  • {@code discountOverlaps = true} + *
+ */ + public TFSimilarity() { + this(true); + } + + /** + * @param discountOverlaps True if overlap tokens (tokens with a position of increment of zero) + * are discounted from the document's length. + */ + public TFSimilarity(boolean discountOverlaps) { + this.discountOverlaps = discountOverlaps; + } + + /** Returns true if overlap tokens are discounted from the document's length. */ + public boolean getDiscountOverlaps() { + return discountOverlaps; + } @Override public long computeNorm(FieldInvertState state) { - return 1; // we dont care + final int numTerms; + if (state.getIndexOptions() == IndexOptions.DOCS && state.getIndexCreatedVersionMajor() >= 8) { + numTerms = state.getUniqueTermCount(); + } else if (discountOverlaps) { + numTerms = state.getLength() - state.getNumOverlap(); + } else { + numTerms = state.getLength(); + } + return SmallFloat.intToByte4(numTerms); } @Override From 861034b2316d1df12e1f75899524af10e036d41f Mon Sep 17 00:00:00 2001 From: Christine Poerschke Date: Wed, 11 Sep 2024 08:30:11 +0100 Subject: [PATCH 6/8] action code review feedback w.r.t. [Raw]TFSimilarity class name --- .../{TFSimilarity.java => RawTFSimilarity.java} | 10 +++++----- .../lucene/search/TestBooleanQueryVisitSubscorers.java | 4 ++-- .../org/apache/lucene/search/TestConjunctions.java | 4 ++-- ...{TestTFSimilarity.java => TestRawTFSimilarity.java} | 6 +++--- 4 files changed, 12 insertions(+), 12 deletions(-) rename lucene/core/src/java/org/apache/lucene/search/similarities/{TFSimilarity.java => RawTFSimilarity.java} (90%) rename lucene/core/src/test/org/apache/lucene/search/similarities/{TestTFSimilarity.java => TestRawTFSimilarity.java} (95%) diff --git a/lucene/core/src/java/org/apache/lucene/search/similarities/TFSimilarity.java b/lucene/core/src/java/org/apache/lucene/search/similarities/RawTFSimilarity.java similarity index 90% rename from lucene/core/src/java/org/apache/lucene/search/similarities/TFSimilarity.java rename to lucene/core/src/java/org/apache/lucene/search/similarities/RawTFSimilarity.java index 0b5d9139dbc4..7d237fbc093e 100644 --- a/lucene/core/src/java/org/apache/lucene/search/similarities/TFSimilarity.java +++ b/lucene/core/src/java/org/apache/lucene/search/similarities/RawTFSimilarity.java @@ -22,18 +22,18 @@ import org.apache.lucene.search.TermStatistics; import org.apache.lucene.util.SmallFloat; -/** Similarity that returns the TF as score */ -public class TFSimilarity extends Similarity { +/** Similarity that returns the raw TF as score. */ +public class RawTFSimilarity extends Similarity { private final boolean discountOverlaps; /** - * TFSimilarity with these default values: + * RawTFSimilarity with these default values: * *
    *
  • {@code discountOverlaps = true} *
*/ - public TFSimilarity() { + public RawTFSimilarity() { this(true); } @@ -41,7 +41,7 @@ public TFSimilarity() { * @param discountOverlaps True if overlap tokens (tokens with a position of increment of zero) * are discounted from the document's length. */ - public TFSimilarity(boolean discountOverlaps) { + public RawTFSimilarity(boolean discountOverlaps) { this.discountOverlaps = discountOverlaps; } diff --git a/lucene/core/src/test/org/apache/lucene/search/TestBooleanQueryVisitSubscorers.java b/lucene/core/src/test/org/apache/lucene/search/TestBooleanQueryVisitSubscorers.java index f9055c3bf5a9..7d511446283a 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestBooleanQueryVisitSubscorers.java +++ b/lucene/core/src/test/org/apache/lucene/search/TestBooleanQueryVisitSubscorers.java @@ -35,7 +35,7 @@ import org.apache.lucene.index.Term; import org.apache.lucene.search.BooleanClause.Occur; import org.apache.lucene.search.similarities.ClassicSimilarity; -import org.apache.lucene.search.similarities.TFSimilarity; +import org.apache.lucene.search.similarities.RawTFSimilarity; import org.apache.lucene.store.Directory; import org.apache.lucene.tests.analysis.MockAnalyzer; import org.apache.lucene.tests.index.RandomIndexWriter; @@ -74,7 +74,7 @@ public void setUp() throws Exception { searcher = newSearcher(reader, true, false); searcher.setSimilarity(new ClassicSimilarity()); scorerSearcher = new ScorerIndexSearcher(reader); - scorerSearcher.setSimilarity(new TFSimilarity()); + scorerSearcher.setSimilarity(new RawTFSimilarity()); } @Override diff --git a/lucene/core/src/test/org/apache/lucene/search/TestConjunctions.java b/lucene/core/src/test/org/apache/lucene/search/TestConjunctions.java index 6f1c0deb6865..6a2581647842 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestConjunctions.java +++ b/lucene/core/src/test/org/apache/lucene/search/TestConjunctions.java @@ -33,7 +33,7 @@ import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.index.Term; -import org.apache.lucene.search.similarities.TFSimilarity; +import org.apache.lucene.search.similarities.RawTFSimilarity; import org.apache.lucene.store.Directory; import org.apache.lucene.tests.analysis.MockAnalyzer; import org.apache.lucene.tests.index.RandomIndexWriter; @@ -66,7 +66,7 @@ public void setUp() throws Exception { reader = writer.getReader(); writer.close(); searcher = newSearcher(reader); - searcher.setSimilarity(new TFSimilarity()); + searcher.setSimilarity(new RawTFSimilarity()); } static Document doc(String v1, String v2) { diff --git a/lucene/core/src/test/org/apache/lucene/search/similarities/TestTFSimilarity.java b/lucene/core/src/test/org/apache/lucene/search/similarities/TestRawTFSimilarity.java similarity index 95% rename from lucene/core/src/test/org/apache/lucene/search/similarities/TestTFSimilarity.java rename to lucene/core/src/test/org/apache/lucene/search/similarities/TestRawTFSimilarity.java index 6e7502e3cb06..3f90609997e1 100644 --- a/lucene/core/src/test/org/apache/lucene/search/similarities/TestTFSimilarity.java +++ b/lucene/core/src/test/org/apache/lucene/search/similarities/TestRawTFSimilarity.java @@ -34,7 +34,7 @@ import org.apache.lucene.tests.util.LuceneTestCase; import org.apache.lucene.util.IOUtils; -public class TestTFSimilarity extends BaseSimilarityTestCase { +public class TestRawTFSimilarity extends BaseSimilarityTestCase { private Directory directory; private IndexReader indexReader; @@ -42,7 +42,7 @@ public class TestTFSimilarity extends BaseSimilarityTestCase { @Override protected Similarity getSimilarity(Random random) { - return new TFSimilarity(); + return new RawTFSimilarity(); } @Override @@ -63,7 +63,7 @@ public void setUp() throws Exception { } indexReader = DirectoryReader.open(directory); indexSearcher = newSearcher(indexReader); - indexSearcher.setSimilarity(new TFSimilarity()); + indexSearcher.setSimilarity(new RawTFSimilarity()); } @Override From 17d6619dd2f698228dee6cd61482c1cdc0679198 Mon Sep 17 00:00:00 2001 From: Christine Poerschke Date: Wed, 11 Sep 2024 09:03:43 +0100 Subject: [PATCH 7/8] Update lucene/core/src/java/org/apache/lucene/search/similarities/RawTFSimilarity.java --- .../org/apache/lucene/search/similarities/RawTFSimilarity.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lucene/core/src/java/org/apache/lucene/search/similarities/RawTFSimilarity.java b/lucene/core/src/java/org/apache/lucene/search/similarities/RawTFSimilarity.java index 7d237fbc093e..232cea320676 100644 --- a/lucene/core/src/java/org/apache/lucene/search/similarities/RawTFSimilarity.java +++ b/lucene/core/src/java/org/apache/lucene/search/similarities/RawTFSimilarity.java @@ -38,6 +38,8 @@ public RawTFSimilarity() { } /** + * RawTFSimilarity + * * @param discountOverlaps True if overlap tokens (tokens with a position of increment of zero) * are discounted from the document's length. */ From e143faa4a3c2593a4388cd6b0c2c624ada392ba9 Mon Sep 17 00:00:00 2001 From: Christine Poerschke Date: Fri, 13 Sep 2024 09:52:42 +0100 Subject: [PATCH 8/8] post origin/main merge adjustments --- .../search/similarities/RawTFSimilarity.java | 41 ++----------------- .../similarities/TestRawTFSimilarity.java | 4 +- 2 files changed, 6 insertions(+), 39 deletions(-) diff --git a/lucene/core/src/java/org/apache/lucene/search/similarities/RawTFSimilarity.java b/lucene/core/src/java/org/apache/lucene/search/similarities/RawTFSimilarity.java index 232cea320676..043d831dbe19 100644 --- a/lucene/core/src/java/org/apache/lucene/search/similarities/RawTFSimilarity.java +++ b/lucene/core/src/java/org/apache/lucene/search/similarities/RawTFSimilarity.java @@ -16,53 +16,20 @@ */ package org.apache.lucene.search.similarities; -import org.apache.lucene.index.FieldInvertState; -import org.apache.lucene.index.IndexOptions; import org.apache.lucene.search.CollectionStatistics; import org.apache.lucene.search.TermStatistics; -import org.apache.lucene.util.SmallFloat; /** Similarity that returns the raw TF as score. */ public class RawTFSimilarity extends Similarity { - private final boolean discountOverlaps; - /** - * RawTFSimilarity with these default values: - * - *
    - *
  • {@code discountOverlaps = true} - *
- */ + /** Default constructor: parameter-free */ public RawTFSimilarity() { - this(true); + super(); } - /** - * RawTFSimilarity - * - * @param discountOverlaps True if overlap tokens (tokens with a position of increment of zero) - * are discounted from the document's length. - */ + /** Primary constructor. */ public RawTFSimilarity(boolean discountOverlaps) { - this.discountOverlaps = discountOverlaps; - } - - /** Returns true if overlap tokens are discounted from the document's length. */ - public boolean getDiscountOverlaps() { - return discountOverlaps; - } - - @Override - public long computeNorm(FieldInvertState state) { - final int numTerms; - if (state.getIndexOptions() == IndexOptions.DOCS && state.getIndexCreatedVersionMajor() >= 8) { - numTerms = state.getUniqueTermCount(); - } else if (discountOverlaps) { - numTerms = state.getLength() - state.getNumOverlap(); - } else { - numTerms = state.getLength(); - } - return SmallFloat.intToByte4(numTerms); + super(discountOverlaps); } @Override diff --git a/lucene/core/src/test/org/apache/lucene/search/similarities/TestRawTFSimilarity.java b/lucene/core/src/test/org/apache/lucene/search/similarities/TestRawTFSimilarity.java index 3f90609997e1..2631058834eb 100644 --- a/lucene/core/src/test/org/apache/lucene/search/similarities/TestRawTFSimilarity.java +++ b/lucene/core/src/test/org/apache/lucene/search/similarities/TestRawTFSimilarity.java @@ -87,7 +87,7 @@ public void testThree() throws IOException { private void implTest(String text, float expectedScore) throws IOException { Query query = new TermQuery(new Term("test", text)); TopDocs topDocs = indexSearcher.search(query, 1); - assertEquals(1, topDocs.totalHits.value); + assertEquals(1, topDocs.totalHits.value()); assertEquals(1, topDocs.scoreDocs.length); assertEquals(expectedScore, topDocs.scoreDocs[0].score, 0.0); } @@ -96,7 +96,7 @@ public void testBoostQuery() throws IOException { Query query = new TermQuery(new Term("test", "three")); float boost = 14f; TopDocs topDocs = indexSearcher.search(new BoostQuery(query, boost), 1); - assertEquals(1, topDocs.totalHits.value); + assertEquals(1, topDocs.totalHits.value()); assertEquals(1, topDocs.scoreDocs.length); assertEquals(42f, topDocs.scoreDocs[0].score, 0.0); }