Move buildSrc into a composite included build (build-infra). Expose a…

… plugin with buildinfra extension.

TODO.txt

@@ -0,0 +1,11 @@
+- locate all other dependencies in build scripts and move them to toml.
+- min java version -> toml
+
+- reuse version catalog from composite builds
+
+- configure configuration groups for versions.lock, add documentation to help/
+- configure and document updateVersions/ tidy versions file.
+
+- apply spotless to composite builds
+
+- upgrade spotless and other plugins

buildSrc/build.gradle → build-tools/build-infra/build.gradle

@@ -15,26 +15,43 @@
  * limitations under the License.
  */
 
+plugins {
+  id "java-gradle-plugin"
+}
+
 repositories {
   mavenCentral()
 }
 
+group = "org.apache.lucene"
+
 ext {
-  // Minimum Java version required to compile buildSrc.
+  // Minimum Java version required to compile.
   minJavaVersion = JavaVersion.VERSION_21
 }
 
+gradlePlugin {
+  automatedPublishing = false
+
+  plugins {
+    buildInfra {
+      id = 'lucene.build-infra'
+      implementationClass = 'org.apache.lucene.gradle.buildinfra.BuildInfraPlugin'
+    }
+  }
+}
 
 // Make sure the build environment is consistent.
-apply from: file('../gradle/validation/check-environment.gradle')
+// apply from: file('../gradle/validation/check-environment.gradle')
 
 dependencies {
   implementation gradleApi()
   implementation localGroovy()
   implementation "commons-codec:commons-codec:1.13"
 }
 
-if (!rootProject.hasJavaFlightRecorder) {
+def hasJavaFlightRecorder = ModuleLayer.boot().findModule('jdk.jfr').map(this.class.module::canRead).orElse(false)
+if (!hasJavaFlightRecorder) {
   logger.warn('Module jdk.jfr is not available; skipping compilation of Java Flight Recorder support.')
   tasks.named('compileJava').configure {
     exclude('**/ProfileResults.java')

build-tools/build-infra/settings.gradle

@@ -0,0 +1,18 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+rootProject.name = 'build-infra'

buildSrc/src/main/java/org/apache/lucene/gradle/Checksum.java → build-tools/build-infra/src/main/java/org/apache/lucene/gradle/Checksum.java

@@ -48,7 +48,7 @@
 public class Checksum extends DefaultTask {
   private FileCollection files;
   private File outputDir;
-  private Algorithm algorithm;
+  private Algorithm algorithm = Algorithm.SHA512;
 
   public enum Algorithm {
     MD5(new DigestUtils(DigestUtils.getMd5Digest())),

build-tools/build-infra/src/main/java/org/apache/lucene/gradle/buildinfra/BuildInfraPlugin.java

@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.gradle.buildinfra;
+
+import org.apache.lucene.gradle.Checksum;
+import org.apache.lucene.gradle.ErrorReportingTestListener;
+import org.apache.lucene.gradle.datasets.ExtractReuters;
+import org.gradle.api.tasks.testing.TestDescriptor;
+import org.gradle.api.Plugin;
+import org.gradle.api.Project;
+import org.gradle.api.tasks.testing.logging.TestLogging;
+import java.nio.file.Path;
+import org.apache.commons.codec.digest.DigestUtils;
+
+public class BuildInfraPlugin implements Plugin<Project> {
+    @Override
+    public void apply(Project project) {
+      project
+        .getExtensions()
+        .create(
+            BuildInfraExtension.NAME,
+            BuildInfraExtension.class);
+    }
+
+    public static class BuildInfraExtension {
+      public static final String NAME = "buildinfra";
+
+      public ErrorReportingTestListener newErrorReportingTestListener(TestLogging testLogging, Path spillDir, Path outputsDir, boolean verboseMode) {
+          return new ErrorReportingTestListener(testLogging, spillDir, outputsDir, verboseMode);
+      }
+
+      public DigestUtils sha1Digest() {
+          return new DigestUtils(DigestUtils.getSha1Digest());
+      }
+
+      public void extractReuters(String reutersDir, String outputDir) throws Exception {
+          ExtractReuters.main(new String [] { reutersDir, outputDir } );
+      }
+
+      public String getOutputLogName(TestDescriptor suite) {
+          return ErrorReportingTestListener.getOutputLogName(suite);
+      }
+
+      public Class<?> checksumClass() {
+          return Checksum.class;
+      }
+    }
+}

build.gradle

@@ -20,6 +20,7 @@ import java.time.format.DateTimeFormatter
 
 plugins {
   id "base"
+  id "lucene.build-infra"
 
   alias(deps.plugins.dependencychecks)
   alias(deps.plugins.spotless) apply false

gradle/datasets/external-datasets.gradle

@@ -1,5 +1,3 @@
-import org.apache.lucene.gradle.datasets.ExtractReuters
-
 import java.nio.file.Files
 
 /*
@@ -164,7 +162,7 @@ configure(project(":lucene:benchmark")) {
 
       logger.lifecycle("Extracting ${ext.name} into ${ext.dst}...")
       ext.dst.deleteDir()
-      ExtractReuters.main(untarPath.toString(), ext.dst.toString())
+      buildinfra.extractReuters(untarPath.toString(), ext.dst.toString())
     }
   }
 

gradle/generation/regenerate.gradle

@@ -1,7 +1,5 @@
 import groovy.json.JsonOutput
 import groovy.json.JsonSlurper
-import org.apache.commons.codec.digest.DigestUtils
-
 import java.util.function.Function
 
 /*
@@ -58,7 +56,7 @@ def computeChecksummedEntries = { Task sourceTask ->
   allFiles.files.forEach { file ->
     allEntries.put(
         sourceTask.project.rootDir.relativePath(file),
-        file.exists() ? new DigestUtils(DigestUtils.sha1Digest).digestAsHex(file).trim() : "--")
+        file.exists() ? buildinfra.sha1Digest().digestAsHex(file).trim() : "--")
   }
 
   return allEntries

gradle/testing/defaults-tests.gradle

@@ -18,7 +18,6 @@
 import org.apache.tools.ant.taskdefs.condition.Os
 import org.apache.tools.ant.types.Commandline
 import org.gradle.api.tasks.testing.logging.*
-import org.apache.lucene.gradle.ErrorReportingTestListener
 
 def resources = scriptResources(buildscript)
 def verboseModeHookInstalled = false
@@ -201,7 +200,7 @@ allprojects {
       }
 
       def spillDir = getTemporaryDir().toPath()
-      def listener = new ErrorReportingTestListener(test.testLogging, spillDir, testOutputsDir.toPath(), verboseMode)
+      def listener = buildinfra.newErrorReportingTestListener(test.testLogging, spillDir, testOutputsDir.toPath(), verboseMode)
       addTestOutputListener(listener)
       addTestListener(listener)
 

gradle/testing/failed-tests-at-end.gradle

@@ -15,8 +15,6 @@
  * limitations under the License.
  */
 
-import org.apache.lucene.gradle.ErrorReportingTestListener
-
 // Display all failed tests at the end of the build.
 
 def failedTests = []
@@ -28,7 +26,7 @@ allprojects {
         failedTests << [
             "name": "${desc.className}.${desc.name}",
             "project": "${test.project.path}",
-            "output": file("${task.testOutputsDir}/${ErrorReportingTestListener.getOutputLogName(desc.parent)}"),
+            "output": file("${task.testOutputsDir}/${buildinfra.getOutputLogName(desc.parent)}"),
             "reproduce": "gradlew ${project.path}:test --tests \"${desc.className}.${desc.name}\" ${task.project.testOptionsForReproduceLine}"
         ]
       }
@@ -39,7 +37,7 @@ allprojects {
         failedTests << [
             "name": "${desc.name}",
             "project": "${test.project.path}",
-            "output": file("${task.testOutputsDir}/${ErrorReportingTestListener.getOutputLogName(desc)}"),
+            "output": file("${task.testOutputsDir}/${buildinfra.getOutputLogName(desc)}"),
             "reproduce": "gradlew ${project.path}:test --tests \"${desc.name}\" ${task.project.testOptionsForReproduceLine}"
         ]
       }

gradle/validation/jar-checks.gradle

@@ -20,8 +20,6 @@
 // 2) notice file
 // 3) checksum validation/ generation.
 
-import org.apache.commons.codec.digest.DigestUtils
-
 // This should be false only for debugging.
 def failOnError = true
 
@@ -136,7 +134,7 @@ subprojects {
                   jarName        : file.toPath().getFileName().toString(),
                   path           : file,
                   module         : resolvedArtifact.moduleVersion,
-                  checksum       : provider { new DigestUtils(DigestUtils.sha1Digest).digestAsHex(file).trim() },
+                  checksum       : provider { buildinfra.sha1Digest.digestAsHex(file).trim() },
                   // We keep track of the files referenced by this dependency (sha, license, notice, etc.)
                   // so that we can determine unused dangling files later on.
                   referencedFiles: []

gradle/validation/rat-sources.gradle

@@ -78,10 +78,10 @@ allprojects {
                     // Exclude github stuff (templates, workflows).
                     exclude ".github"
 
-                    // The root project also includes patterns for the boostrap (buildSrc) and composite
+                    // The root project also includes patterns for the include composite
                     // projects. Include their sources in the scan.
-                    include "buildSrc/src/**"
-                    include "dev-tools/missing-doclet/src/**"
+                    include "build-tools/build-infra/src/**"
+                    include "build-tools/missing-doclet/src/**"
 
                     // do not let RAT attempt to scan a python venv, it gets lost and confused...
                     exclude "dev-tools/aws-jmh/build/**"

gradlew

@@ -158,7 +158,7 @@ fi
 
 GRADLE_WRAPPER_JAR="$APP_HOME/gradle/wrapper/gradle-wrapper.jar"
 if [ ! -e "$GRADLE_WRAPPER_JAR" ]; then
-    "$JAVACMD" $JAVA_OPTS "$APP_HOME/buildSrc/src/main/java/org/apache/lucene/gradle/WrapperDownloader.java" "$GRADLE_WRAPPER_JAR"
+    "$JAVACMD" $JAVA_OPTS "$APP_HOME/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/WrapperDownloader.java" "$GRADLE_WRAPPER_JAR"
     WRAPPER_STATUS=$?
     if [ "$WRAPPER_STATUS" -eq 1 ]; then
         echo "ERROR: Something went wrong. Make sure you're using Java version of exactly 21."
@@ -173,7 +173,7 @@ CLASSPATH=$GRADLE_WRAPPER_JAR
 # START OF LUCENE CUSTOMIZATION
 # Generate gradle.properties if they don't exist
 if [ ! -e "$APP_HOME/gradle.properties" ]; then
-    "$JAVACMD" $JAVA_OPTS "$APP_HOME/buildSrc/src/main/java/org/apache/lucene/gradle/GradlePropertiesGenerator.java" "$APP_HOME/gradle/template.gradle.properties" "$APP_HOME/gradle.properties"
+    "$JAVACMD" $JAVA_OPTS "$APP_HOME/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/GradlePropertiesGenerator.java" "$APP_HOME/gradle/template.gradle.properties" "$APP_HOME/gradle.properties"
     GENERATOR_STATUS=$?
     if [ "$GENERATOR_STATUS" -ne 0 ]; then
         exit $GENERATOR_STATUS

gradlew.bat

@@ -76,7 +76,7 @@ goto fail
 @rem LUCENE-9266: verify and download the gradle wrapper jar if we don't have one.
 set GRADLE_WRAPPER_JAR=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar
 IF NOT EXIST "%GRADLE_WRAPPER_JAR%" (
-    "%JAVA_EXE%" %JAVA_OPTS% "%APP_HOME%/buildSrc/src/main/java/org/apache/lucene/gradle/WrapperDownloader.java" "%GRADLE_WRAPPER_JAR%"
+    "%JAVA_EXE%" %JAVA_OPTS% "%APP_HOME%/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/WrapperDownloader.java" "%GRADLE_WRAPPER_JAR%"
     IF %ERRORLEVEL% EQU 1 goto failWithJvmMessage
     IF %ERRORLEVEL% NEQ 0 goto fail
 )
@@ -89,7 +89,7 @@ set CLASSPATH=%GRADLE_WRAPPER_JAR%
 IF NOT EXIST "%APP_HOME%\gradle.properties" (
   @rem local expansion is needed to check ERRORLEVEL inside control blocks.
   setlocal enableDelayedExpansion
-  "%JAVA_EXE%" %JAVA_OPTS% "%APP_HOME%/buildSrc/src/main/java/org/apache/lucene/gradle/GradlePropertiesGenerator.java" "%APP_HOME%\gradle\template.gradle.properties" "%APP_HOME%\gradle.properties"
+  "%JAVA_EXE%" %JAVA_OPTS% "%APP_HOME%/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/GradlePropertiesGenerator.java" "%APP_HOME%\gradle\template.gradle.properties" "%APP_HOME%\gradle.properties"
   IF %ERRORLEVEL% NEQ 0 goto fail
   endlocal
 )

lucene/distribution/build.gradle

@@ -15,8 +15,6 @@
  * limitations under the License.
  */
 
-import org.apache.lucene.gradle.Checksum
-
 import java.nio.charset.StandardCharsets
 import java.nio.file.Files
 
@@ -60,9 +58,7 @@ dependencies {
 
 
 // Compute checksums for release archives.
-task computeChecksums(type: Checksum) {
-  algorithm = Checksum.Algorithm.SHA512
-
+task computeChecksums(type: buildinfra.checksumClass()) {
   files = objects.fileCollection()
   [
       tasks.assembleSourceTgz,

settings.gradle

@@ -20,6 +20,8 @@ pluginManagement {
         mavenCentral()
         gradlePluginPortal()
     }
+
+    includeBuild("build-tools/build-infra")
 }
 
 plugins {
@@ -40,7 +42,7 @@ apply from: file('gradle/ge.gradle')
 
 rootProject.name = "lucene-root"
 
-includeBuild("dev-tools/missing-doclet")
+includeBuild("build-tools/missing-doclet")
 
 include "lucene:analysis:common"
 include "lucene:analysis:icu"