Add individual precommit task plugins (#56926) (#57015)

Each precommit task is currently registered inside the shared
PrecommitTasks class. Having a single class with all precommit tasks
makes individualizing which precommit tasks are needed based on type of
project difficult, where we often just disable somet tasks eg for all qa
projects. This commit creates plugins for each precommit task, and moves
the configuration of the task into each plugin. PrecommitTasks remains
for now, and just delegates to each plugin, but will be removed in a
future change.

Author: rjernst

buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy

@@ -23,7 +23,6 @@ import groovy.transform.CompileStatic
 import org.apache.commons.io.IOUtils
 import org.elasticsearch.gradle.info.BuildParams
 import org.elasticsearch.gradle.info.GlobalBuildInfoPlugin
-import org.elasticsearch.gradle.precommit.DependencyLicensesTask
 import org.elasticsearch.gradle.precommit.PrecommitTasks
 import org.elasticsearch.gradle.test.ErrorReportingTestListener
 import org.elasticsearch.gradle.testclusters.ElasticsearchCluster
@@ -51,10 +50,8 @@ import org.gradle.api.credentials.HttpHeaderCredentials
 import org.gradle.api.execution.TaskActionListener
 import org.gradle.api.plugins.ExtraPropertiesExtension
 import org.gradle.api.plugins.JavaPlugin
-import org.gradle.api.tasks.TaskProvider
 import org.gradle.api.tasks.testing.Test
 import org.gradle.authentication.http.HttpHeaderAuthentication
-import org.gradle.language.base.plugins.LifecycleBasePlugin
 import org.gradle.util.GradleVersion
 
 import java.nio.charset.StandardCharsets
@@ -98,7 +95,7 @@ class BuildPlugin implements Plugin<Project> {
 
         configureRepositories(project)
         project.extensions.getByType(ExtraPropertiesExtension).set('versions', VersionProperties.versions)
-        configurePrecommit(project)
+        PrecommitTasks.create(project, true)
         configureFips140(project)
     }
 
@@ -277,18 +274,6 @@ class BuildPlugin implements Plugin<Project> {
             throw new GradleException(message)
         }
     }
-    
-    private static configurePrecommit(Project project) {
-        TaskProvider precommit = PrecommitTasks.create(project, true)
-        project.tasks.named(LifecycleBasePlugin.CHECK_TASK_NAME).configure { it.dependsOn(precommit) }
-        project.tasks.named(JavaPlugin.TEST_TASK_NAME).configure { it.mustRunAfter(precommit) }
-        // only require dependency licenses for non-elasticsearch deps
-        project.tasks.withType(DependencyLicensesTask).named('dependencyLicenses').configure {
-            it.dependencies = project.configurations.getByName(JavaPlugin.RUNTIME_CLASSPATH_CONFIGURATION_NAME).fileCollection { Dependency dependency ->
-                dependency.group.startsWith('org.elasticsearch') == false
-            } - project.configurations.getByName(JavaPlugin.COMPILE_ONLY_CONFIGURATION_NAME)
-        }
-    }
 
     private static class TestFailureReportingPlugin implements Plugin<Project> {
         @Override

buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/LicenseHeadersPrecommitPlugin.groovy

@@ -0,0 +1,31 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.gradle.precommit
+
+import org.gradle.api.Project
+import org.gradle.api.Task
+import org.gradle.api.tasks.TaskProvider
+
+class LicenseHeadersPrecommitPlugin extends PrecommitPlugin {
+    @Override
+    TaskProvider<? extends Task> createTask(Project project) {
+        return project.getTasks().register("licenseHeaders", LicenseHeadersTask.class);
+    }
+}

buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/PrecommitTasks.groovy

@@ -18,20 +18,8 @@
  */
 package org.elasticsearch.gradle.precommit
 
-import de.thetaphi.forbiddenapis.gradle.CheckForbiddenApis
-import de.thetaphi.forbiddenapis.gradle.ForbiddenApisPlugin
-import org.elasticsearch.gradle.ExportElasticsearchBuildResourcesTask
-import org.elasticsearch.gradle.VersionProperties
-import org.elasticsearch.gradle.info.BuildParams
-import org.elasticsearch.gradle.util.Util
-import org.gradle.api.JavaVersion
+
 import org.gradle.api.Project
-import org.gradle.api.artifacts.Configuration
-import org.gradle.api.file.FileCollection
-import org.gradle.api.plugins.JavaBasePlugin
-import org.gradle.api.plugins.quality.Checkstyle
-import org.gradle.api.tasks.SourceSet
-import org.gradle.api.tasks.TaskProvider
 
 /**
  * Validation tasks which should be run before committing. These run before tests.
@@ -40,42 +28,21 @@ class PrecommitTasks {
 
     /** Adds a precommit task, which depends on non-test verification tasks. */
 
-    public static TaskProvider create(Project project, boolean includeDependencyLicenses) {
-        project.configurations.create("forbiddenApisCliJar")
-        project.dependencies {
-            forbiddenApisCliJar('de.thetaphi:forbiddenapis:2.7')
-        }
-
-        Configuration jarHellConfig = project.configurations.create("jarHell")
-        if (BuildParams.internal && project.path.equals(":libs:elasticsearch-core") == false) {
-            // External plugins will depend on this already via transitive dependencies.
-            // Internal projects are not all plugins, so make sure the check is available
-            // we are not doing this for this project itself to avoid jar hell with itself
-            project.dependencies {
-                jarHell project.project(":libs:elasticsearch-core")
-            }
-        }
+    public static void create(Project project, boolean includeDependencyLicenses) {
 
-        List<TaskProvider> precommitTasks = [
-                configureCheckstyle(project),
-                configureForbiddenApisCli(project),
-                project.tasks.register('forbiddenPatterns', ForbiddenPatternsTask),
-                project.tasks.register('licenseHeaders', LicenseHeadersTask),
-                project.tasks.register('filepermissions', FilePermissionsTask),
-                configureJarHell(project, jarHellConfig),
-                configureThirdPartyAudit(project),
-                configureTestingConventions(project)
-        ]
+        project.pluginManager.apply(CheckstylePrecommitPlugin)
+        project.pluginManager.apply(ForbiddenApisPrecommitPlugin)
+        project.pluginManager.apply(JarHellPrecommitPlugin)
+        project.pluginManager.apply(ForbiddenPatternsPrecommitPlugin)
+        project.pluginManager.apply(LicenseHeadersPrecommitPlugin)
+        project.pluginManager.apply(FilePermissionsPrecommitPlugin)
+        project.pluginManager.apply(ThirdPartyAuditPrecommitPlugin)
+        project.pluginManager.apply(TestingConventionsPrecommitPlugin)
 
         // tasks with just tests don't need dependency licenses, so this flag makes adding
         // the task optional
         if (includeDependencyLicenses) {
-            TaskProvider<DependencyLicensesTask> dependencyLicenses = project.tasks.register('dependencyLicenses', DependencyLicensesTask)
-            precommitTasks.add(dependencyLicenses)
-            // we also create the updateShas helper task that is associated with dependencyLicenses
-            project.tasks.register('updateShas', UpdateShasTask) {
-                it.parentTask = dependencyLicenses
-            }
+            project.pluginManager.apply(DependencyLicensesPrecommitPlugin)
         }
         if (project.path != ':build-tools') {
             /*
@@ -88,195 +55,7 @@ class PrecommitTasks {
              * use the NamingConventionsCheck we break the circular dependency
              * here.
              */
-            precommitTasks.add(configureLoggerUsage(project))
-        }
-
-        // We want to get any compilation error before running the pre-commit checks.
-        project.sourceSets.all { sourceSet ->
-            precommitTasks.each { provider ->
-                provider.configure {
-                    shouldRunAfter(sourceSet.getClassesTaskName())
-                }
-            }
-        }
-
-        TaskProvider precommit = project.tasks.register('precommit') {
-            group = JavaBasePlugin.VERIFICATION_GROUP
-            description = 'Runs all non-test checks.'
-            dependsOn = precommitTasks
-        }
-
-        // not all jar projects produce a pom (we don't ship all jars), so a pom validation
-        // task is only added on some projects, and thus we can't always have a task
-        // here to add to precommit tasks explicitly. Instead, we apply our internal
-        // pom validation plugin after the precommit task is created and let the
-        // plugin add the task if necessary
-        project.plugins.apply(PomValidationPlugin)
-
-        return precommit
-    }
-
-    static TaskProvider configureTestingConventions(Project project) {
-        return project.getTasks().register("testingConventions", TestingConventionsTasks) {
-            naming {
-                Tests {
-                    baseClass "org.apache.lucene.util.LuceneTestCase"
-                }
-                IT {
-                    baseClass "org.elasticsearch.test.ESIntegTestCase"
-                    baseClass 'org.elasticsearch.test.rest.ESRestTestCase'
-                }
-            }
-        }
-    }
-
-    private static TaskProvider configureJarHell(Project project, Configuration jarHellConfig) {
-        return project.tasks.register('jarHell', JarHellTask) { task ->
-            task.classpath = project.sourceSets.test.runtimeClasspath + jarHellConfig
-            task.dependsOn(jarHellConfig)
-        }
-    }
-
-    private static TaskProvider configureThirdPartyAudit(Project project) {
-        ExportElasticsearchBuildResourcesTask buildResources = project.tasks.getByName('buildResources')
-        return project.tasks.register('thirdPartyAudit', ThirdPartyAuditTask) { task ->
-            task.dependsOn(buildResources)
-            task.signatureFile = buildResources.copy("forbidden/third-party-audit.txt")
-            task.javaHome = BuildParams.runtimeJavaHome
-            task.targetCompatibility.set(project.provider({ BuildParams.runtimeJavaVersion }))
-        }
-    }
-
-    private static TaskProvider configureForbiddenApisCli(Project project) {
-        project.pluginManager.apply(ForbiddenApisPlugin)
-        ExportElasticsearchBuildResourcesTask buildResources = project.tasks.getByName('buildResources')
-        project.tasks.withType(CheckForbiddenApis).configureEach {
-            dependsOn(buildResources)
-
-            assert name.startsWith(ForbiddenApisPlugin.FORBIDDEN_APIS_TASK_NAME)
-            String sourceSetName
-            if (ForbiddenApisPlugin.FORBIDDEN_APIS_TASK_NAME.equals(name)) {
-                sourceSetName = "main"
-            } else {
-                //parse out the sourceSetName
-                char[] chars = name.substring(ForbiddenApisPlugin.FORBIDDEN_APIS_TASK_NAME.length()).toCharArray()
-                chars[0] = Character.toLowerCase(chars[0])
-                sourceSetName = new String(chars)
-            }
-
-            SourceSet sourceSet = project.sourceSets.getByName(sourceSetName)
-            classpath = project.files { sourceSet.runtimeClasspath.plus(sourceSet.compileClasspath) }
-
-            targetCompatibility = BuildParams.runtimeJavaVersion.majorVersion
-            if (BuildParams.runtimeJavaVersion > JavaVersion.VERSION_14) {
-                // TODO: forbidden apis does not yet support java 15, rethink using runtime version
-                targetCompatibility = JavaVersion.VERSION_14.majorVersion
-            }
-            bundledSignatures = [
-                    "jdk-unsafe", "jdk-deprecated", "jdk-non-portable", "jdk-system-out"
-            ]
-            signaturesFiles = project.files(
-                    buildResources.copy("forbidden/jdk-signatures.txt"),
-                    buildResources.copy("forbidden/es-all-signatures.txt")
-            )
-            suppressAnnotations = ['**.SuppressForbidden']
-            if (name.endsWith('Test')) {
-                signaturesFiles += project.files(
-                        buildResources.copy("forbidden/es-test-signatures.txt"),
-                        buildResources.copy("forbidden/http-signatures.txt")
-                )
-            } else {
-                signaturesFiles += project.files(buildResources.copy("forbidden/es-server-signatures.txt"))
-            }
-            ext.replaceSignatureFiles = { String... names ->
-                signaturesFiles = project.files(
-                        names.collect { buildResources.copy("forbidden/${it}.txt") }
-                )
-            }
-            ext.addSignatureFiles = { String... names ->
-                signaturesFiles += project.files(
-                        names.collect { buildResources.copy("forbidden/${it}.txt") }
-                )
-            }
-        }
-        TaskProvider forbiddenApis = project.tasks.named("forbiddenApis")
-        forbiddenApis.configure {
-            group = ""
-        }
-        return forbiddenApis
-    }
-
-    private static TaskProvider configureCheckstyle(Project project) {
-        // Always copy the checkstyle configuration files to 'buildDir/checkstyle' since the resources could be located in a jar
-        // file. If the resources are located in a jar, Gradle will fail when it tries to turn the URL into a file
-        URL checkstyleConfUrl = PrecommitTasks.getResource("/checkstyle.xml")
-        URL checkstyleSuppressionsUrl = PrecommitTasks.getResource("/checkstyle_suppressions.xml")
-        File checkstyleDir = new File(project.buildDir, "checkstyle")
-        File checkstyleSuppressions = new File(checkstyleDir, "checkstyle_suppressions.xml")
-        File checkstyleConf = new File(checkstyleDir, "checkstyle.xml");
-        TaskProvider copyCheckstyleConf = project.tasks.register("copyCheckstyleConf")
-
-        // configure inputs and outputs so up to date works properly
-        copyCheckstyleConf.configure {
-            outputs.files(checkstyleSuppressions, checkstyleConf)
-        }
-        if ("jar".equals(checkstyleConfUrl.getProtocol())) {
-            JarURLConnection jarURLConnection = (JarURLConnection) checkstyleConfUrl.openConnection()
-            copyCheckstyleConf.configure {
-                inputs.file(jarURLConnection.getJarFileURL())
-            }
-        } else if ("file".equals(checkstyleConfUrl.getProtocol())) {
-            copyCheckstyleConf.configure {
-                inputs.files(checkstyleConfUrl.getFile(), checkstyleSuppressionsUrl.getFile())
-            }
-        }
-
-        copyCheckstyleConf.configure {
-            doLast {
-                checkstyleDir.mkdirs()
-                // withStream will close the output stream and IOGroovyMethods#getBytes reads the InputStream fully and closes it
-                new FileOutputStream(checkstyleConf).withStream {
-                    it.write(checkstyleConfUrl.openStream().getBytes())
-                }
-                new FileOutputStream(checkstyleSuppressions).withStream {
-                    it.write(checkstyleSuppressionsUrl.openStream().getBytes())
-                }
-            }
-        }
-
-        TaskProvider checkstyleTask = project.tasks.register('checkstyle') {
-            dependsOn project.tasks.withType(Checkstyle)
-        }
-        // Apply the checkstyle plugin to create `checkstyleMain` and `checkstyleTest`. It only
-        // creates them if there is main or test code to check and it makes `check` depend
-        // on them. We also want `precommit` to depend on `checkstyle`.
-        project.pluginManager.apply('checkstyle')
-        project.checkstyle {
-            configDir = checkstyleDir
-        }
-        project.dependencies {
-            checkstyle "com.puppycrawl.tools:checkstyle:${VersionProperties.versions.checkstyle}"
-            checkstyle project.files(Util.buildSrcCodeSource)
-        }
-
-        project.tasks.withType(Checkstyle).configureEach { task ->
-            task.dependsOn(copyCheckstyleConf)
-            task.reports {
-                html.enabled false
-            }
-        }
-
-        return checkstyleTask
-    }
-
-    private static TaskProvider configureLoggerUsage(Project project) {
-        Object dependency = BuildParams.internal ? project.project(':test:logger-usage') :
-                "org.elasticsearch.test:logger-usage:${VersionProperties.elasticsearch}"
-
-        project.configurations.create('loggerUsagePlugin')
-        project.dependencies.add('loggerUsagePlugin', dependency)
-        return project.tasks.register('loggerUsageCheck', LoggerUsageTask) {
-            classpath = project.configurations.loggerUsagePlugin
+            project.pluginManager.apply(LoggerUsagePrecommitPlugin)
         }
     }
 }

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/StandaloneRestTestPlugin.groovy

@@ -95,6 +95,5 @@ class StandaloneRestTestPlugin implements Plugin<Project> {
         idea.module.scopes.put('TEST', [plus: [project.configurations.getByName(JavaPlugin.TEST_RUNTIME_CLASSPATH_CONFIGURATION_NAME)]] as Map<String, Collection<Configuration>>)
 
         PrecommitTasks.create(project, false)
-        project.tasks.named('check').configure { it.dependsOn(project.tasks.named('precommit')) }
     }
 }

buildSrc/src/main/java/org/elasticsearch/gradle/PublishPlugin.java

@@ -24,6 +24,7 @@
 import groovy.util.Node;
 import groovy.util.NodeList;
 import org.elasticsearch.gradle.info.BuildParams;
+import org.elasticsearch.gradle.precommit.PomValidationPrecommitPlugin;
 import org.elasticsearch.gradle.util.Util;
 import org.gradle.api.Plugin;
 import org.gradle.api.Project;
@@ -50,6 +51,7 @@ public class PublishPlugin implements Plugin<Project> {
     @Override
     public void apply(Project project) {
         project.getPluginManager().apply("nebula.maven-base-publish");
+        project.getPluginManager().apply(PomValidationPrecommitPlugin.class);
 
         configureJavadocJar(project);
         configureSourcesJar(project);