Merge pull request apache#5 from apache/master

pull from upstream again

dev-tools/idea/lucene/join/join.iml

@@ -14,7 +14,6 @@
  <orderEntry type="library" scope="TEST" name="JUnit" level="project" />
  <orderEntry type="module" scope="TEST" module-name="lucene-test-framework" />
  <orderEntry type="module" module-name="grouping" />
- <orderEntry type="module" module-name="backward-codecs" />
  <orderEntry type="module" module-name="lucene-core" />
  </component>
 </module>

dev-tools/idea/lucene/queryparser/queryparser.iml

@@ -17,6 +17,5 @@
  <orderEntry type="module" module-name="lucene-core" />
  <orderEntry type="module" module-name="queries" />
  <orderEntry type="module" module-name="sandbox" />
- <orderEntry type="module" module-name="backward-codecs" />
  </component>
 </module>

dev-tools/scripts/addVersion.py

@@ -217,7 +217,9 @@ def main():
  update_changes('lucene/CHANGES.txt', c.version)
  update_changes('solr/CHANGES.txt', c.version, get_solr_init_changes())
 
- if current_version.is_back_compat_with(c.version):
+ is_back_compat = current_version.major == c.version.major or current_version.is_back_compat_with(c.version)
+
+ if is_back_compat:
  add_constant(c.version, not c.is_latest_version)
  else:
  print('\nNot adding constant for version %s because it is no longer supported' % c.version)
@@ -232,7 +234,7 @@ def main():
  print('\nTODO: ')
  print(' - Move backcompat oldIndexes to unsupportedIndexes in TestBackwardsCompatibility')
  print(' - Update IndexFormatTooOldException throw cases')
- elif current_version.is_back_compat_with(c.version):
+ elif is_back_compat:
  print('\nTesting changes')
  check_lucene_version_tests()
  check_solr_version_tests()

dev-tools/scripts/buildAndPushRelease.py

@@ -218,7 +218,7 @@ def check_cmdline_tools(): # Fail fast if there are cmdline tool problems
  if os.system('git --version >/dev/null 2>/dev/null'):
  raise RuntimeError('"git --version" returned a non-zero exit code.')
  antVersion = os.popen('ant -version').read().strip()
- if not antVersion.startswith('Apache Ant(TM) version 1.8'):
+ if not antVersion.startswith('Apache Ant(TM) version 1.8') and not antVersion.startswith('Apache Ant(TM) version 1.9'):
  raise RuntimeError('ant version is not 1.8.X: "%s"' % antVersion)
 
 def main():

dev-tools/scripts/smokeTestRelease.py

@@ -190,16 +190,23 @@ def checkJARMetaData(desc, jarFile, gitRevision, version):
  'Implementation-Vendor: The Apache Software Foundation',
  # Make sure 1.8 compiler was used to build release bits:
  'X-Compile-Source-JDK: 8',
- # Make sure 1.8 ant was used to build release bits: (this will match 1.8+)
- 'Ant-Version: Apache Ant 1.8',
+ # Make sure 1.8 or 1.9 ant was used to build release bits: (this will match 1.8.x, 1.9.x)
+ ('Ant-Version: Apache Ant 1.8', 'Ant-Version: Apache Ant 1.9'),
  # Make sure .class files are 1.8 format:
  'X-Compile-Target-JDK: 8',
  'Specification-Version: %s' % version,
  # Make sure the release was compiled with 1.8:
  'Created-By: 1.8'):
- if s.find(verify) == -1:
- raise RuntimeError('%s is missing "%s" inside its META-INF/MANIFEST.MF' % \
- (desc, verify))
+ if type(verify) is not tuple:
+ verify = (verify,)
+ for x in verify:
+ if s.find(x) != -1:
+ break
+ else:
+ if len(verify) == 1:
+ raise RuntimeError('%s is missing "%s" inside its META-INF/MANIFEST.MF' % (desc, verify[0]))
+ else:
+ raise RuntimeError('%s is missing one of "%s" inside its META-INF/MANIFEST.MF' % (desc, verify))
 
  if gitRevision != 'skip':
  # Make sure this matches the version and git revision we think we are releasing:

lucene/CHANGES.txt

@@ -30,6 +30,9 @@ Other
 
 * LUCENE-7360: Remove Explanation.toHtml() (Alan Woodward)
 
+======================= Lucene 6.3.0 =======================
+(No Changes)
+
 ======================= Lucene 6.2.0 =======================
 
 API Changes
@@ -91,6 +94,9 @@ Bug Fixes
 * SOLR-9413: Fix analysis/kuromoji's CSVUtil.quoteEscape logic, add TestCSVUtil test.
  (AppChecker, Christine Poerschke)
 
+* LUCENE-7419: Fix performance bug with TokenStream.end(), where it would lookup
+ PositionIncrementAttribute every time. (Mike McCandless, Robert Muir)
+
 Improvements
 
 * LUCENE-7323: Compound file writing now verifies the incoming

lucene/core/src/java/org/apache/lucene/analysis/TokenStream.java

@@ -22,7 +22,6 @@
 import java.lang.reflect.Modifier;
 
 import org.apache.lucene.analysis.tokenattributes.PackedTokenAttributeImpl;
-import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
 import org.apache.lucene.index.IndexWriter;
@@ -176,11 +175,7 @@ private boolean assertFinal() {
  * @throws IOException If an I/O error occurs
  */
  public void end() throws IOException {
- clearAttributes(); // LUCENE-3849: don't consume dirty atts
- PositionIncrementAttribute posIncAtt = getAttribute(PositionIncrementAttribute.class);
- if (posIncAtt != null) {
- posIncAtt.setPositionIncrement(0);
- }
+ endAttributes(); // LUCENE-3849: don't consume dirty atts
  }
 
  /**

lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/PackedTokenAttributeImpl.java

@@ -138,6 +138,17 @@ public void clear() {
  startOffset = endOffset = 0;
  type = DEFAULT_TYPE;
  }
+
+ /** Resets the attributes at end
+ */
+ @Override
+ public void end() {
+ super.end();
+ positionIncrement = 0;
+ positionLength = 1;
+ startOffset = endOffset = 0;
+ type = DEFAULT_TYPE;
+ }
 
  @Override
  public PackedTokenAttributeImpl clone() {

lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/PositionIncrementAttributeImpl.java

@@ -46,6 +46,11 @@ public void clear() {
  this.positionIncrement = 1;
  }
 
+ @Override
+ public void end() {
+ this.positionIncrement = 0;
+ }
+
  @Override
  public boolean equals(Object other) {
  if (other == this) {

lucene/core/src/java/org/apache/lucene/util/AttributeImpl.java

@@ -32,6 +32,17 @@ public abstract class AttributeImpl implements Cloneable, Attribute {
  */
  public abstract void clear();
 
+ /**
+ * Clears the values in this AttributeImpl and resets it to its value
+ * at the end of the field. If this implementation implements more than one Attribute interface
+ * it clears all.
+ * <p>
+ * The default implementation simply calls {@link #clear()}
+ */
+ public void end() {
+ clear();
+ }
+
  /**
  * This method returns the current attribute values as a string in the following format
  * by calling the {@link #reflectWith(AttributeReflector)} method:

lucene/core/src/java/org/apache/lucene/util/AttributeSource.java

@@ -270,6 +270,16 @@ public final void clearAttributes() {
  state.attribute.clear();
  }
  }
+
+ /**
+ * Resets all Attributes in this AttributeSource by calling
+ * {@link AttributeImpl#end()} on each Attribute implementation.
+ */
+ public final void endAttributes() {
+ for (State state = getCurrentState(); state != null; state = state.next) {
+ state.attribute.end();
+ }
+ }
 
  /**
  * Removes all attributes and their implementations from this AttributeSource.

lucene/core/src/java/org/apache/lucene/util/Version.java

@@ -59,6 +59,13 @@ public final class Version {
  @Deprecated
  public static final Version LUCENE_6_2_0 = new Version(6, 2, 0);
 
+ /**
+ * Match settings and bugs in Lucene's 6.3.0 release.
+ * @deprecated Use latest
+ */
+ @Deprecated
+ public static final Version LUCENE_6_3_0 = new Version(6, 3, 0);
+
  /**
  * Match settings and bugs in Lucene's 7.0.0 release.
  * <p>

lucene/core/src/test/org/apache/lucene/search/TestSearcherManager.java

@@ -26,6 +26,7 @@
 import java.util.concurrent.Executors;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicReference;
 
 import org.apache.lucene.analysis.MockAnalyzer;
 import org.apache.lucene.document.Document;
@@ -43,6 +44,7 @@
 import org.apache.lucene.index.ThreadedIndexingAndSearchingTestCase;
 import org.apache.lucene.store.AlreadyClosedException;
 import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.LineFileDocs;
 import org.apache.lucene.util.LuceneTestCase.SuppressCodecs;
 import org.apache.lucene.util.LuceneTestCase;
 import org.apache.lucene.util.NamedThreadFactory;
@@ -533,4 +535,144 @@ public IndexSearcher newSearcher(IndexReader reader, IndexReader previousReader)
  sm.close();
  dir.close();
  }
+
+ public void testConcurrentIndexCloseSearchAndRefresh() throws Exception {
+ final Directory dir = newFSDirectory(createTempDir());
+ AtomicReference<IndexWriter> writerRef = new AtomicReference<>();
+ writerRef.set(new IndexWriter(dir, newIndexWriterConfig()));
+
+ AtomicReference<SearcherManager> mgrRef = new AtomicReference<>();
+ mgrRef.set(new SearcherManager(writerRef.get(), null));
+ final AtomicBoolean stop = new AtomicBoolean();
+
+ Thread indexThread = new Thread() {
+ @Override
+ public void run() {
+ try {
+ LineFileDocs docs = new LineFileDocs(random());
+ long runTimeSec = TEST_NIGHTLY ? atLeast(10) : atLeast(2);
+ long endTime = System.nanoTime() + runTimeSec * 1000000000;
+ while (System.nanoTime() < endTime) {
+ IndexWriter w = writerRef.get();
+ w.addDocument(docs.nextDoc());
+ if (random().nextInt(1000) == 17) {
+ if (random().nextBoolean()) {
+ w.close();
+ } else {
+ w.rollback();
+ }
+ writerRef.set(new IndexWriter(dir, newIndexWriterConfig()));
+ }
+ }
+ docs.close();
+ stop.set(true);
+ if (VERBOSE) {
+ System.out.println("TEST: index count=" + writerRef.get().maxDoc());
+ }
+ } catch (IOException ioe) {
+ throw new RuntimeException(ioe);
+ }
+ }
+ };
+
+ Thread searchThread = new Thread() {
+ @Override
+ public void run() {
+ try {
+ long totCount = 0;
+ while (stop.get() == false) {
+ SearcherManager mgr = mgrRef.get();
+ if (mgr != null) {
+ IndexSearcher searcher;
+ try {
+ searcher = mgr.acquire();
+ } catch (AlreadyClosedException ace) {
+ // ok
+ continue;
+ }
+ totCount += searcher.getIndexReader().maxDoc();
+ mgr.release(searcher);
+ }
+ }
+ if (VERBOSE) {
+ System.out.println("TEST: search totCount=" + totCount);
+ }
+ } catch (IOException ioe) {
+ throw new RuntimeException(ioe);
+ }
+ }
+ };
+
+ Thread refreshThread = new Thread() {
+ @Override
+ public void run() {
+ try {
+ int refreshCount = 0;
+ int aceCount = 0;
+ while (stop.get() == false) {
+ SearcherManager mgr = mgrRef.get();
+ if (mgr != null) {
+ refreshCount++;
+ try {
+ mgr.maybeRefreshBlocking();
+ } catch (AlreadyClosedException ace) {
+ // ok
+ aceCount++;
+ continue;
+ }
+ }
+ }
+ if (VERBOSE) {
+ System.out.println("TEST: refresh count=" + refreshCount + " aceCount=" + aceCount);
+ }
+ } catch (IOException ioe) {
+ throw new RuntimeException(ioe);
+ }
+ }
+ };
+
+ Thread closeThread = new Thread() {
+ @Override
+ public void run() {
+ try {
+ int closeCount = 0;
+ int aceCount = 0;
+ while (stop.get() == false) {
+ SearcherManager mgr = mgrRef.get();
+ assert mgr != null;
+ mgr.close();
+ closeCount++;
+ while (stop.get() == false) {
+ try {
+ mgrRef.set(new SearcherManager(writerRef.get(), null));
+ break;
+ } catch (AlreadyClosedException ace) {
+ // ok
+ aceCount++;
+ }
+ }
+ }
+ if (VERBOSE) {
+ System.out.println("TEST: close count=" + closeCount + " aceCount=" + aceCount);
+ }
+ } catch (IOException ioe) {
+ throw new RuntimeException(ioe);
+ }
+ }
+ };
+
+ indexThread.start();
+ searchThread.start();
+ refreshThread.start();
+ closeThread.start();
+
+ indexThread.join();
+ searchThread.join();
+ refreshThread.join();
+ closeThread.join();
+
+ mgrRef.get().close();
+ writerRef.get().close();
+ dir.close();
+ }
 }

lucene/core/src/test/org/apache/lucene/store/TestMmapDirectory.java

@@ -22,7 +22,7 @@
 import java.util.Random;
 import java.util.concurrent.CountDownLatch;
 
-// import org.junit.Ignore;
+import org.junit.Ignore;
 
 /**
  * Tests MMapDirectory
@@ -43,7 +43,7 @@ public void setUp() throws Exception {
  MMapDirectory.UNMAP_SUPPORTED);
  }
 
- // TODO: @Ignore("This test is for JVM testing purposes. There are no guarantees that it may not fail with SIGSEGV!")
+ @Ignore("This test is for JVM testing purposes. There are no guarantees that it may not fail with SIGSEGV!")
  public void testAceWithThreads() throws Exception {
  for (int iter = 0; iter < 10; iter++) {
  Directory dir = getDirectory(createTempDir("testAceWithThreads"));

lucene/join/build.xml

@@ -26,7 +26,6 @@
 
  <path id="classpath">
  <pathelement path="${grouping.jar}"/>
- <pathelement path="${backward-codecs.jar}"/>
  <path refid="base.classpath"/>
  </path>
 
@@ -35,14 +34,13 @@
  <pathelement location="${build.dir}/classes/java"/>
  </path>
 
- <target name="init" depends="module-build.init,jar-grouping,jar-backward-codecs"/>
+ <target name="init" depends="module-build.init,jar-grouping"/>
 
- <target name="javadocs" depends="javadocs-grouping,javadocs-backward-codecs,compile-core,check-javadocs-uptodate"
+ <target name="javadocs" depends="javadocs-grouping,compile-core,check-javadocs-uptodate"
  unless="javadocs-uptodate-${name}">
  <invoke-module-javadoc>
  <links>
  <link href="../grouping"/>
- <link href="../backward-codecs"/>
  </links>
  </invoke-module-javadoc>
  </target>