merge from main

.github/workflows/run-checks-all.yml

@@ -10,7 +10,7 @@ on:
   push:
     branches:
       - 'main'
-      - 'branch_9x'
+      - 'branch_10x'
 
 env:
   GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}

.github/workflows/run-checks-gradle-upgrade.yml

@@ -6,15 +6,15 @@ on:
   pull_request:
     branches:
       - 'main'
-      - 'branch_9x'
+      - 'branch_10x'
     paths:
       - '.github/workflows/run-checks-gradle-upgrade.yml'
       - 'gradle/wrapper/**'
 
   push:
     branches:
       - 'main'
-      - 'branch_9x'
+      - 'branch_10x'
     paths:
       - '.github/workflows/run-checks-gradle-upgrade.yml'
       - 'gradle/wrapper/**'
@@ -30,7 +30,7 @@ jobs:
     strategy:
       matrix:
         os: [ ubuntu-latest ]
-        java-version: [ '22' ]
+        java-version: [ '23-ea' ]
         uses-alt-java: [ true, false ]
 
     runs-on: ${{ matrix.os }}
@@ -61,15 +61,24 @@ jobs:
           # https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#setting-an-environment-variable
           echo "RUNTIME_JAVA_HOME=${{ env.ALT_JAVA_DIR }}" >> "$GITHUB_ENV"
 
-      - run: ./gradlew -p lucene/core check -x test
+      - name: ./gradlew tidy
+        run: |
+          ./gradlew tidy
+          if [ ! -z "$(git status --porcelain)" ]; then
+          echo ":warning: **tidy left local checkout in modified state**" >> $GITHUB_STEP_SUMMARY
+          echo '```' >> $GITHUB_STEP_SUMMARY
+          git status --porcelain >> $GITHUB_STEP_SUMMARY
+          echo '```' >> $GITHUB_STEP_SUMMARY
+          git reset --hard && git clean -xfd .
+          fi
 
       - name: ./gradlew regenerate
         run: |
           # add this package for generateEmojiTokenizationTestChecksumLoad.
           sudo apt-get install libwww-perl
           ./gradlew regenerate -x generateUAX29URLEmailTokenizerInternal --rerun-tasks
           if [ ! -z "$(git status --porcelain)" ]; then 
-            echo ":warning: **regenerateleft local checkout in modified state**" >> $GITHUB_STEP_SUMMARY 
+            echo ":warning: **regenerate left local checkout in modified state**" >> $GITHUB_STEP_SUMMARY 
             echo '```' >> $GITHUB_STEP_SUMMARY
             git status --porcelain >> $GITHUB_STEP_SUMMARY
             echo '```' >> $GITHUB_STEP_SUMMARY
@@ -79,8 +88,7 @@ jobs:
       - run: ./gradlew testOpts
       - run: ./gradlew helpWorkflow
       - run: ./gradlew licenses updateLicenses
-      - run: ./gradlew tidy
-      - run: ./gradlew check -x test
+      - run: ./gradlew check -x test -Pvalidation.git.failOnModified=false
       - run: ./gradlew assembleRelease mavenToLocal
 
       # Conserve resources: only run these in non-alt-java mode.

.github/workflows/run-checks-mod-analysis-common.yml

@@ -6,15 +6,15 @@ on:
   pull_request:
     branches:
       - 'main'
-      - 'branch_9x'
+      - 'branch_10x'
     paths:
       - '.github/workflows/run-checks-mod-analysis-common.yml'
       - 'lucene/analysis/common/**'
 
   push:
     branches:
       - 'main'
-      - 'branch_9x'
+      - 'branch_10x'
     paths:
       - '.github/workflows/run-checks-mod-analysis-common.yml'
       - 'lucene/analysis/common/**'

.github/workflows/run-checks-mod-distribution.tests.yml

@@ -6,12 +6,12 @@ on:
   pull_request:
     branches:
       - 'main'
-      - 'branch_9x'
+      - 'branch_10x'
 
   push:
     branches:
       - 'main'
-      - 'branch_9x'
+      - 'branch_10x'
 
 env:
   GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}

.github/workflows/run-nightly-smoketester.yml

@@ -18,7 +18,7 @@ jobs:
     strategy:
       matrix:
         os: [ ubuntu-latest ]
-        java-version: [ '21', '22' ]
+        java-version: [ '21', '22', '23-ea' ]
 
     runs-on: ${{ matrix.os }}
 
@@ -72,3 +72,4 @@ jobs:
           name: smoke-tester-logs-jdk-${{ matrix.java-version }}
           path: |
             ${{ env.TMP_DIR }}/**/*.log
+            /tmp/release.log

NOTICE.txt

@@ -1,5 +1,5 @@
 Apache Lucene
-Copyright 2001-2022 The Apache Software Foundation
+Copyright 2001-2024 The Apache Software Foundation
 
 This product includes software developed at
 The Apache Software Foundation (http://www.apache.org/).

build-tools/build-infra/src/main/java/org/apache/lucene/gradle/ProfileResults.java

@@ -41,7 +41,7 @@
  */
 public class ProfileResults {
   /** Formats a frame to a formatted line. This is deduplicated on! */
-  static String frameToString(RecordedFrame frame, boolean lineNumbers) {
+  static String frameToString(RecordedFrame frame, boolean lineNumbers, boolean frameTypes) {
     StringBuilder builder = new StringBuilder();
     RecordedMethod method = frame.getMethod();
     RecordedClass clazz = method.getType();
@@ -55,13 +55,14 @@ static String frameToString(RecordedFrame frame, boolean lineNumbers) {
     builder.append("#");
     builder.append(method.getName());
     builder.append("()");
-    if (lineNumbers) {
+    if (lineNumbers && frame.getLineNumber() != -1) {
       builder.append(":");
-      if (frame.getLineNumber() == -1) {
-        builder.append("(" + frame.getType() + " code)");
-      } else {
-        builder.append(frame.getLineNumber());
-      }
+      builder.append(frame.getLineNumber());
+    }
+    if (clazz != null && frameTypes) {
+      builder.append(" [");
+      builder.append(frame.getType());
+      builder.append(" code]");
     }
     return builder.toString();
   }
@@ -77,6 +78,8 @@ static String frameToString(RecordedFrame frame, boolean lineNumbers) {
   public static final String COUNT_DEFAULT = "10";
   public static final String LINENUMBERS_KEY = "tests.profile.linenumbers";
   public static final String LINENUMBERS_DEFAULT = "false";
+  public static final String FRAMETYPES_KEY = "tests.profile.frametypes";
+  public static final String FRAMETYPES_DEFAULT = "true";
 
   /**
    * Driver method, for testing standalone.
@@ -92,7 +95,8 @@ public static void main(String[] args) throws IOException {
         System.getProperty(MODE_KEY, MODE_DEFAULT),
         Integer.parseInt(System.getProperty(STACKSIZE_KEY, STACKSIZE_DEFAULT)),
         Integer.parseInt(System.getProperty(COUNT_KEY, COUNT_DEFAULT)),
-        Boolean.parseBoolean(System.getProperty(LINENUMBERS_KEY, LINENUMBERS_DEFAULT)));
+        Boolean.parseBoolean(System.getProperty(LINENUMBERS_KEY, LINENUMBERS_DEFAULT)),
+        Boolean.parseBoolean(System.getProperty(FRAMETYPES_KEY, FRAMETYPES_DEFAULT)));
   }
 
   /** true if we care about this event */
@@ -152,7 +156,12 @@ private static String pad(String input) {
 
   /** Process all the JFR files passed in args and print a merged summary. */
   public static void printReport(
-      List<String> files, String mode, int stacksize, int count, boolean lineNumbers)
+      List<String> files,
+      String mode,
+      int stacksize,
+      int count,
+      boolean lineNumbers,
+      boolean frameTypes)
       throws IOException {
     if (!"cpu".equals(mode) && !"heap".equals(mode)) {
       throw new IllegalArgumentException("tests.profile.mode must be one of (cpu,heap)");
@@ -181,7 +190,7 @@ public static void printReport(
               if (stack.length() > 0) {
                 stack.append("\n").append(framePadding).append("  at ");
               }
-              stack.append(frameToString(trace.getFrames().get(i), lineNumbers));
+              stack.append(frameToString(trace.getFrames().get(i), lineNumbers, frameTypes));
             }
             String line = stack.toString();
             SimpleEntry<String, Long> entry =

build-tools/build-infra/src/main/java/org/apache/lucene/gradle/WrapperDownloader.java

@@ -60,8 +60,8 @@ public static void main(String[] args) {
 
   public static void checkVersion() {
     int major = Runtime.version().feature();
-    if (major != 21 && major != 22) {
-      throw new IllegalStateException("java version must be 21 or 22, your version: " + major);
+    if (major != 21 && major != 22 && major != 23) {
+      throw new IllegalStateException("java version must be 21, 22 or 23, your version: " + major);
     }
   }
 

build-tools/missing-doclet/src/main/java/org/apache/lucene/missingdoclet/MissingDoclet.java

@@ -231,8 +231,8 @@ private void check(Element element) {
       case PACKAGE:
         checkComment(element);
         break;
-        // class-like elements, check them, then recursively check their children (fields and
-        // methods)
+      // class-like elements, check them, then recursively check their children (fields and
+      // methods)
       case CLASS:
       case INTERFACE:
       case ENUM:
@@ -257,7 +257,7 @@ && level(element) >= METHOD) {
           }
         }
         break;
-        // method-like elements, check them if we are configured to do so
+      // method-like elements, check them if we are configured to do so
       case METHOD:
       case CONSTRUCTOR:
       case FIELD:

build.gradle

@@ -41,7 +41,7 @@ apply from: file('gradle/globals.gradle')
 // Calculate project version:
 version = {
   // Release manager: update base version here after release:
-  String baseVersion = '10.0.0'
+  String baseVersion = '11.0.0'
 
   // On a release explicitly set release version in one go:
   //  -Dversion.release=x.y.z
@@ -80,6 +80,9 @@ ext {
   // Minimum Java version required to compile and run Lucene.
   minJavaVersion = JavaVersion.toVersion(deps.versions.minJava.get())
 
+  // also change this in extractor tool: ExtractForeignAPI
+  vectorIncubatorJavaVersions = [ JavaVersion.VERSION_21, JavaVersion.VERSION_22, JavaVersion.VERSION_23 ] as Set
+
   // snapshot build marker used in scripts.
   snapshotBuild = version.contains("SNAPSHOT")
 
@@ -117,10 +120,6 @@ apply from: file('gradle/generation/local-settings.gradle')
 // Make sure the build environment is consistent.
 apply from: file('gradle/validation/check-environment.gradle')
 
-// IDE support, settings and specials.
-apply from: file('gradle/ide/intellij-idea.gradle')
-apply from: file('gradle/ide/eclipse.gradle')
-
 // Set up defaults and configure aspects for certain modules or functionality
 // (java, tests)
 apply from: file('gradle/java/folder-layout.gradle')
@@ -133,6 +132,10 @@ apply from: file('gradle/testing/alternative-jdk-support.gradle')
 apply from: file('gradle/java/jar-manifest.gradle')
 apply from: file('gradle/java/modules.gradle')
 
+// IDE support, settings and specials.
+apply from: file('gradle/ide/intellij-idea.gradle')
+apply from: file('gradle/ide/eclipse.gradle')
+
 // Maven artifact publishing.
 apply from: file('gradle/maven/publications.gradle')
 

dev-docs/working-between-major-versions.adoc

@@ -51,7 +51,7 @@ cd lucene
 git clone git@github.com:apache/lucene.git main
 cd main
 # For each branch that you want a separate directory created for, add a worktree
-git worktree add ../9x branch_9x
+git worktree add ../10x branch_10x
 ----
 
 === Using the Worktrees

dev-tools/doap/lucene.rdf

@@ -67,13 +67,27 @@
     </maintainer>
 
     <!-- NOTE: please insert releases in numeric order, NOT chronologically. -->
+    <release>
+       <Version>
+         <name>lucene-10.0.0</name>
+         <created>2024-10-14</created>
+         <revision>10.0.0</revision>
+       </Version>
+    </release>
+    <release>
+       <Version>
+         <name>lucene-9.12.0</name>
+         <created>2024-09-28</created>
+         <revision>9.12.0</revision>
+       </Version>
+    </release>
     <release>
        <Version>
          <name>lucene-9.11.1</name>
          <created>2024-06-27</created>
          <revision>9.11.1</revision>
        </Version>
-    </release>. 
+    </release> 
     <release>
        <Version>
          <name>lucene-9.11.0</name>
@@ -186,6 +200,13 @@
          <revision>9.0.0</revision>
        </Version>
     </release>
+    <release>
+       <Version>
+         <name>lucene-8.11.4</name>
+         <created>2024-09-24</created>
+         <revision>8.11.4</revision>
+       </Version>
+    </release>
     <release>
        <Version>
          <name>lucene-8.11.3</name>

dev-tools/scripts/addBackcompatIndexes.py

@@ -40,7 +40,7 @@ def create_and_add_index(source, indextype, index_version, current_version, temp
       'cfs': 'index',
       'nocfs': 'index',
       'sorted': 'sorted',
-      'int8_hnsw': 'int8_hnsw',
+      'int7_hnsw': 'int7_hnsw',
       'moreterms': 'moreterms',
       'dvupdates': 'dvupdates',
       'emptyIndex': 'empty'
@@ -61,7 +61,7 @@ def create_and_add_index(source, indextype, index_version, current_version, temp
     'cfs': 'testCreateCFS',
     'nocfs': 'testCreateNoCFS',
     'sorted': 'testCreateSortedIndex',
-    'int8_hnsw': 'testCreateInt8HNSWIndices',
+    'int7_hnsw': 'testCreateInt7HNSWIndices',
     'moreterms': 'testCreateMoreTermsIndex',
     'dvupdates': 'testCreateIndexWithDocValuesUpdates',
     'emptyIndex': 'testCreateEmptyIndex'
@@ -206,7 +206,7 @@ def main():
   current_version = scriptutil.Version.parse(scriptutil.find_current_version())
   create_and_add_index(source, 'cfs', c.version, current_version, c.temp_dir)
   create_and_add_index(source, 'nocfs', c.version, current_version, c.temp_dir)
-  create_and_add_index(source, 'int8_hnsw', c.version, current_version, c.temp_dir)
+  create_and_add_index(source, 'int7_hnsw', c.version, current_version, c.temp_dir)
   should_make_sorted =     current_version.is_back_compat_with(c.version) \
                        and (c.version.major > 6 or (c.version.major == 6 and c.version.minor >= 2))
   if should_make_sorted:

dev-tools/scripts/buildAndPushRelease.py

@@ -112,16 +112,18 @@ def prepare(root, version, pause_before_sign, gpg_key_id, gpg_password, gpg_home
   checkDOAPfiles(version)
 
   if not dev_mode:
-    print('  ./gradlew --no-daemon clean check')
-    run('./gradlew --no-daemon clean check')
+    print('  ./gradlew --stacktrace --no-daemon clean')
+    run('./gradlew --stacktrace --no-daemon clean')
+    print('  ./gradlew --stacktrace --no-daemon check')
+    run('./gradlew --stacktrace --no-daemon check')
   else:
     print('  skipping precommit check due to dev-mode')
 
   if pause_before_sign:
     input("Tests complete! Please press ENTER to proceed to assembleRelease: ")
 
   print('  prepare-release')
-  cmd = './gradlew --no-daemon assembleRelease' \
+  cmd = './gradlew --stacktrace --no-daemon assembleRelease' \
         ' -Dversion.release=%s' % version
   if dev_mode:
     cmd += ' -Pvalidation.git.failOnModified=false'

dev-tools/scripts/releaseWizard.py

@@ -239,7 +239,7 @@ def maybe_remove_rc_from_svn():
                  logfile="svn_rm.log",
                  tee=True,
                  vars={
-                     'dist_folder': """lucene-{{ release_version }}-RC{{ rc_number }}-rev{{ build_rc.git_rev | default("<git_rev>", True) }}""",
+                     'dist_folder': """lucene-{{ release_version }}-RC{{ rc_number }}-rev-{{ build_rc.git_rev | default("<git_rev>", True) }}""",
                      'dist_url': "{{ dist_url_base }}/{{ dist_folder }}"
                  }
              )],

gradle/documentation/render-javadoc.gradle

@@ -32,7 +32,7 @@ allprojects {
       missingdoclet "org.apache.lucene.tools:missing-doclet"
     }
 
-    ext {
+    project.ext {
       relativeDocPath = project.path.replaceFirst(/:\w+:/, "").replace(':', '/')
     }