From e4cad2447917ace36407fa2092bec92c9b967c7d Mon Sep 17 00:00:00 2001 From: Alpar Torok Date: Fri, 17 Aug 2018 17:38:42 +0300 Subject: [PATCH 01/10] Initial skeleton --- .../gradle/precommit/ThirdPartyAuditNew.java | 53 +++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 buildSrc/src/test/java/org/elasticsearch/gradle/precommit/ThirdPartyAuditNew.java diff --git a/buildSrc/src/test/java/org/elasticsearch/gradle/precommit/ThirdPartyAuditNew.java b/buildSrc/src/test/java/org/elasticsearch/gradle/precommit/ThirdPartyAuditNew.java new file mode 100644 index 0000000000000..a1134fd03ddf8 --- /dev/null +++ b/buildSrc/src/test/java/org/elasticsearch/gradle/precommit/ThirdPartyAuditNew.java @@ -0,0 +1,53 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.gradle.precommit; + +import org.gradle.api.file.FileCollection; + +import java.util.ArrayList; +import java.util.List; +import java.util.regex.Pattern; + +public class ThirdPartyAuditNew extends ForbiddenApisCliTask { + + static final Pattern MISSING_CLASS_PATTERN = Pattern.compile( + "WARNING: The referenced class '(.*)' cannot be loaded\\. Please fix the classpath\\!" + ); + + static final Pattern VIOLATION_PATTERN = Pattern.compile( + "\\s\\sin ([a-zA-Z0-9\\$\\.]+) \\(.*\\)" + ); + + // patterns for classes to exclude, because we understand their issues + private List excludes = new ArrayList<>(); + + /** + * Input for the task. Set javadoc for {#link getJars} for more. Protected + * so the afterEvaluate closure in the constructor can write it. + */ + protected FileCollection jars; + + /** + * Classpath against which to run the third patty audit. Protected so the + * afterEvaluate closure in the constructor can write it. + */ + protected FileCollection classpath; + + +} From 92267d317fda0598ecc2e4df289ff9940ec66ac2 Mon Sep 17 00:00:00 2001 From: Alpar Torok Date: Tue, 21 Aug 2018 18:14:27 +0300 Subject: [PATCH 02/10] use cli based forbiddenapis fir third party audit --- .../gradle/precommit/PrecommitTasks.groovy | 10 +- .../precommit/ThirdPartyAuditTask.groovy | 297 ------------------ .../test/StandaloneRestTestPlugin.groovy | 2 + .../gradle/precommit/ThirdPartyAuditTask.java | 233 ++++++++++++++ .../gradle/precommit/ThirdPartyAuditNew.java | 53 ---- x-pack/plugin/security/build.gradle | 2 +- 6 files changed, 245 insertions(+), 352 deletions(-) delete mode 100644 buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/ThirdPartyAuditTask.groovy create mode 100644 buildSrc/src/main/java/org/elasticsearch/gradle/precommit/ThirdPartyAuditTask.java delete mode 100644 buildSrc/src/test/java/org/elasticsearch/gradle/precommit/ThirdPartyAuditNew.java diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/PrecommitTasks.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/PrecommitTasks.groovy index b63b1f40d8049..73ac591ea6955 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/PrecommitTasks.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/PrecommitTasks.groovy @@ -39,7 +39,7 @@ class PrecommitTasks { project.tasks.create('licenseHeaders', LicenseHeadersTask.class), project.tasks.create('filepermissions', FilePermissionsTask.class), project.tasks.create('jarHell', JarHellTask.class), - project.tasks.create('thirdPartyAudit', ThirdPartyAuditTask.class) + configureThirdPartyAudit(project) ] // tasks with just tests don't need dependency licenses, so this flag makes adding @@ -75,6 +75,14 @@ class PrecommitTasks { return project.tasks.create(precommitOptions) } + private static Task configureThirdPartyAudit(Project project) { + ThirdPartyAuditTask thirdPartyAuditTask = project.tasks.create('thirdPartyAudit', ThirdPartyAuditTask.class) + ExportElasticsearchBuildResourcesTask buildResources = project.tasks.getByName('buildResources') + thirdPartyAuditTask.dependsOn(buildResources) + thirdPartyAuditTask.signatureFile = buildResources.copy("forbidden/third-party-audit.txt") + return thirdPartyAuditTask + } + private static Task configureForbiddenApisCli(Project project) { Configuration forbiddenApisConfiguration = project.configurations.create("forbiddenApisCliJar") project.dependencies { diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/ThirdPartyAuditTask.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/ThirdPartyAuditTask.groovy deleted file mode 100644 index 52b13a5664427..0000000000000 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/ThirdPartyAuditTask.groovy +++ /dev/null @@ -1,297 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.elasticsearch.gradle.precommit; - -import com.github.jengelman.gradle.plugins.shadow.ShadowPlugin -import org.apache.tools.ant.BuildEvent; -import org.apache.tools.ant.BuildException; -import org.apache.tools.ant.BuildListener; -import org.apache.tools.ant.BuildLogger; -import org.apache.tools.ant.DefaultLogger; -import org.apache.tools.ant.Project; -import org.elasticsearch.gradle.AntTask; -import org.gradle.api.artifacts.Configuration; -import org.gradle.api.file.FileCollection; -import org.gradle.api.tasks.Input -import org.gradle.api.tasks.InputFiles -import org.gradle.api.tasks.OutputFile - -import java.nio.file.FileVisitResult; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.SimpleFileVisitor; -import java.nio.file.attribute.BasicFileAttributes; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -/** - * Basic static checking to keep tabs on third party JARs - */ -public class ThirdPartyAuditTask extends AntTask { - - // patterns for classes to exclude, because we understand their issues - private List excludes = []; - - /** - * Input for the task. Set javadoc for {#link getJars} for more. Protected - * so the afterEvaluate closure in the constructor can write it. - */ - protected FileCollection jars; - - /** - * Classpath against which to run the third patty audit. Protected so the - * afterEvaluate closure in the constructor can write it. - */ - protected FileCollection classpath; - - /** - * We use a simple "marker" file that we touch when the task succeeds - * as the task output. This is compared against the modified time of the - * inputs (ie the jars/class files). - */ - @OutputFile - File successMarker = new File(project.buildDir, 'markers/thirdPartyAudit') - - ThirdPartyAuditTask() { - // we depend on this because its the only reliable configuration - // this probably makes the build slower: gradle you suck here when it comes to configurations, you pay the price. - dependsOn(project.configurations.testCompile); - description = "Checks third party JAR bytecode for missing classes, use of internal APIs, and other horrors'"; - - project.afterEvaluate { - Configuration configuration = project.configurations.findByName('runtime') - Configuration compileOnly = project.configurations.findByName('compileOnly') - if (configuration == null) { - // some projects apparently do not have 'runtime'? what a nice inconsistency, - // basically only serves to waste time in build logic! - configuration = project.configurations.findByName('testCompile') - } - assert configuration != null - if (project.plugins.hasPlugin(ShadowPlugin)) { - Configuration original = configuration - configuration = project.configurations.create('thirdPartyAudit') - configuration.extendsFrom(original, project.configurations.bundle) - } - if (compileOnly == null) { - classpath = configuration - } else { - classpath = project.files(configuration, compileOnly) - } - - // we only want third party dependencies. - jars = configuration.fileCollection({ dependency -> - dependency.group.startsWith("org.elasticsearch") == false - }); - - // we don't want provided dependencies, which we have already scanned. e.g. don't - // scan ES core's dependencies for every single plugin - if (compileOnly != null) { - jars -= compileOnly - } - inputs.files(jars) - onlyIf { jars.isEmpty() == false } - } - } - - /** - * classes that should be excluded from the scan, - * e.g. because we know what sheisty stuff those particular classes are up to. - */ - public void setExcludes(String[] classes) { - for (String s : classes) { - if (s.indexOf('*') != -1) { - throw new IllegalArgumentException("illegal third party audit exclusion: '" + s + "', wildcards are not permitted!"); - } - } - excludes = classes.sort(); - } - - /** - * Returns current list of exclusions. - */ - @Input - public List getExcludes() { - return excludes; - } - - // yes, we parse Uwe Schindler's errors to find missing classes, and to keep a continuous audit. Just don't let him know! - static final Pattern MISSING_CLASS_PATTERN = - Pattern.compile(/WARNING: The referenced class '(.*)' cannot be loaded\. Please fix the classpath\!/); - - static final Pattern VIOLATION_PATTERN = - Pattern.compile(/\s\sin ([a-zA-Z0-9\$\.]+) \(.*\)/); - - // we log everything and capture errors and handle them with our whitelist - // this is important, as we detect stale whitelist entries, workaround forbidden apis bugs, - // and it also allows whitelisting missing classes! - static class EvilLogger extends DefaultLogger { - final Set missingClasses = new TreeSet<>(); - final Map> violations = new TreeMap<>(); - String previousLine = null; - - @Override - public void messageLogged(BuildEvent event) { - if (event.getTask().getClass() == de.thetaphi.forbiddenapis.ant.AntTask.class) { - if (event.getPriority() == Project.MSG_WARN) { - Matcher m = MISSING_CLASS_PATTERN.matcher(event.getMessage()); - if (m.matches()) { - missingClasses.add(m.group(1).replace('.', '/') + ".class"); - } - - // Reset the priority of the event to DEBUG, so it doesn't - // pollute the build output - event.setMessage(event.getMessage(), Project.MSG_DEBUG); - } else if (event.getPriority() == Project.MSG_ERR) { - Matcher m = VIOLATION_PATTERN.matcher(event.getMessage()); - if (m.matches()) { - String violation = previousLine + '\n' + event.getMessage(); - String clazz = m.group(1).replace('.', '/') + ".class"; - List current = violations.get(clazz); - if (current == null) { - current = new ArrayList<>(); - violations.put(clazz, current); - } - current.add(violation); - } - previousLine = event.getMessage(); - } - } - super.messageLogged(event); - } - } - - @Override - protected BuildLogger makeLogger(PrintStream stream, int outputLevel) { - DefaultLogger log = new EvilLogger(); - log.errorPrintStream = stream; - log.outputPrintStream = stream; - log.messageOutputLevel = outputLevel; - return log; - } - - @Override - protected void runAnt(AntBuilder ant) { - ant.project.addTaskDefinition('thirdPartyAudit', de.thetaphi.forbiddenapis.ant.AntTask); - - // print which jars we are going to scan, always - // this is not the time to try to be succinct! Forbidden will print plenty on its own! - Set names = new TreeSet<>(); - for (File jar : jars) { - names.add(jar.getName()); - } - - // TODO: forbidden-apis + zipfileset gives O(n^2) behavior unless we dump to a tmpdir first, - // and then remove our temp dir afterwards. don't complain: try it yourself. - // we don't use gradle temp dir handling, just google it, or try it yourself. - - File tmpDir = new File(project.buildDir, 'tmp/thirdPartyAudit'); - - // clean up any previous mess (if we failed), then unzip everything to one directory - ant.delete(dir: tmpDir.getAbsolutePath()); - tmpDir.mkdirs(); - for (File jar : jars) { - ant.unzip(src: jar.getAbsolutePath(), dest: tmpDir.getAbsolutePath()); - } - - // convert exclusion class names to binary file names - List excludedFiles = excludes.collect {it.replace('.', '/') + ".class"} - Set excludedSet = new TreeSet<>(excludedFiles); - - // jarHellReprise - Set sheistySet = getSheistyClasses(tmpDir.toPath()); - - try { - ant.thirdPartyAudit(failOnUnsupportedJava: false, - failOnMissingClasses: false, - classpath: classpath.asPath) { - fileset(dir: tmpDir) - signatures { - string(value: getClass().getResourceAsStream('/forbidden/third-party-audit.txt').getText('UTF-8')) - } - } - } catch (BuildException ignore) {} - - EvilLogger evilLogger = null; - for (BuildListener listener : ant.project.getBuildListeners()) { - if (listener instanceof EvilLogger) { - evilLogger = (EvilLogger) listener; - break; - } - } - assert evilLogger != null; - - // keep our whitelist up to date - Set bogusExclusions = new TreeSet<>(excludedSet); - bogusExclusions.removeAll(sheistySet); - bogusExclusions.removeAll(evilLogger.missingClasses); - bogusExclusions.removeAll(evilLogger.violations.keySet()); - if (!bogusExclusions.isEmpty()) { - throw new IllegalStateException("Invalid exclusions, nothing is wrong with these classes: " + bogusExclusions); - } - - // don't duplicate classes with the JDK - sheistySet.removeAll(excludedSet); - if (!sheistySet.isEmpty()) { - throw new IllegalStateException("JAR HELL WITH JDK! " + sheistySet); - } - - // don't allow a broken classpath - evilLogger.missingClasses.removeAll(excludedSet); - if (!evilLogger.missingClasses.isEmpty()) { - throw new IllegalStateException("CLASSES ARE MISSING! " + evilLogger.missingClasses); - } - - // don't use internal classes - evilLogger.violations.keySet().removeAll(excludedSet); - if (!evilLogger.violations.isEmpty()) { - throw new IllegalStateException("VIOLATIONS WERE FOUND! " + evilLogger.violations); - } - - // clean up our mess (if we succeed) - ant.delete(dir: tmpDir.getAbsolutePath()); - - successMarker.setText("", 'UTF-8') - } - - /** - * check for sheisty classes: if they also exist in the extensions classloader, its jar hell with the jdk! - */ - private Set getSheistyClasses(Path root) { - // system.parent = extensions loader. - // note: for jigsaw, this evilness will need modifications (e.g. use jrt filesystem!). - // but groovy/gradle needs to work at all first! - ClassLoader ext = ClassLoader.getSystemClassLoader().getParent(); - assert ext != null; - - Set sheistySet = new TreeSet<>(); - Files.walkFileTree(root, new SimpleFileVisitor() { - @Override - public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { - String entry = root.relativize(file).toString().replace('\\', '/'); - if (entry.endsWith(".class")) { - if (ext.getResource(entry) != null) { - sheistySet.add(entry); - } - } - return FileVisitResult.CONTINUE; - } - }); - return sheistySet; - } -} diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/StandaloneRestTestPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/StandaloneRestTestPlugin.groovy index a2484e9c5fce0..a5d3b41339db6 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/StandaloneRestTestPlugin.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/StandaloneRestTestPlugin.groovy @@ -53,6 +53,8 @@ public class StandaloneRestTestPlugin implements Plugin { // only setup tests to build project.sourceSets.create('test') + // create a compileOnly configuration as others might expect it + project.configurations.create("compileOnly") project.dependencies.add('testCompile', "org.elasticsearch.test:framework:${VersionProperties.elasticsearch}") project.eclipse.classpath.sourceSets = [project.sourceSets.test] diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/precommit/ThirdPartyAuditTask.java b/buildSrc/src/main/java/org/elasticsearch/gradle/precommit/ThirdPartyAuditTask.java new file mode 100644 index 0000000000000..65e32f33b44ce --- /dev/null +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/precommit/ThirdPartyAuditTask.java @@ -0,0 +1,233 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.gradle.precommit; + +import de.thetaphi.forbiddenapis.cli.CliMain; +import org.apache.commons.io.output.NullOutputStream; +import org.gradle.api.DefaultTask; +import org.gradle.api.artifacts.Configuration; +import org.gradle.api.file.FileCollection; +import org.gradle.api.tasks.Input; +import org.gradle.api.tasks.InputFile; +import org.gradle.api.tasks.InputFiles; +import org.gradle.api.tasks.OutputDirectory; +import org.gradle.api.tasks.StopExecutionException; +import org.gradle.api.tasks.TaskAction; + +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.FileVisitResult; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.attribute.BasicFileAttributes; +import java.util.Collections; +import java.util.Set; +import java.util.TreeSet; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.Collectors; + +public class ThirdPartyAuditTask extends DefaultTask { + + private static final Pattern MISSING_CLASS_PATTERN = Pattern.compile( + "WARNING: The referenced class '(.*)' cannot be loaded\\. Please fix the classpath!" + ); + + private static final Pattern VIOLATION_PATTERN = Pattern.compile( + "\\s\\sin ([a-zA-Z0-9$.]+) \\(.*\\)" + ); + + /** + * patterns for classes to exclude, because we understand their issues + */ + private Set excludes = new TreeSet<>(); + + private File signatureFile; + + @InputFile + public File getSignatureFile() { + return signatureFile; + } + + public void setSignatureFile(File signatureFile) { + this.signatureFile = signatureFile; + } + + @InputFiles + public Configuration getRuntimeConfiguration() { + Configuration runtime = getProject().getConfigurations().findByName("runtime"); + if (runtime == null) { + return getProject().getConfigurations().getByName("testCompile"); + } + return runtime; + } + + @InputFiles + public Configuration getCompileOnlyConfiguration() { + return getProject().getConfigurations().getByName("compileOnly"); + } + + @OutputDirectory + public File getJarExpandDir() { + return new File( + new File(getProject().getBuildDir(), "precommit/thirdPartyAudit"), + getName() + ); + } + + public void setExcludes(String... classes) { + excludes.clear(); + for (String each : classes) { + if (each.indexOf('*') != -1) { + throw new IllegalArgumentException("illegal third party audit exclusion: '" + each + "', wildcards are not permitted!"); + } + excludes.add(each); + } + } + + @Input + public Set getExcludes() { + return Collections.unmodifiableSet(excludes); + } + + @TaskAction + public void runThirdPartyAudit() throws IOException { + FileCollection jars = getRuntimeConfiguration() + .fileCollection(dep -> dep.getGroup().startsWith("org.elasticsearch") == false); + Configuration compileOnlyConfiguration = getCompileOnlyConfiguration(); + // don't scan provided dependencies that we already scanned, e.x. don't scan cores dependencies for every plugin + if (compileOnlyConfiguration != null) { + jars.minus(compileOnlyConfiguration); + } + if (jars.isEmpty()) { + throw new StopExecutionException("No jars to scan"); + } + + // Stage all the jars + File jarExpandDir = getJarExpandDir(); + jars.forEach(jar -> + getProject().copy(spec -> { + spec.from(getProject().zipTree(jar)); + spec.into(jarExpandDir); + }) + ); + + ByteArrayOutputStream errorOut = new ByteArrayOutputStream(); + getProject().javaexec(spec -> { + spec.classpath( + getProject().getConfigurations().getByName("forbiddenApisCliJar"), + getRuntimeConfiguration(), + getCompileOnlyConfiguration() + ); + spec.setMain(CliMain.class.getName()); + spec.args( + "-f", getSignatureFile().getAbsolutePath(), + "-d", getJarExpandDir(), + "--allowmissingclasses" + ); + spec.setErrorOutput(errorOut); + spec.setStandardOutput(new NullOutputStream()); + spec.setIgnoreExitValue(true); + }); + final Set missingClasses = new TreeSet<>(); + final Set violationsClasses = new TreeSet<>(); + final String forbiddenApisOutput; + try (ByteArrayOutputStream outputStream = errorOut) { + forbiddenApisOutput = outputStream.toString(StandardCharsets.UTF_8.name()); + } + Matcher missingMatcher = MISSING_CLASS_PATTERN.matcher(forbiddenApisOutput); + while (missingMatcher.find()) { + missingClasses.add(missingMatcher.group(1)); + } + Matcher violationMatcher = VIOLATION_PATTERN.matcher(forbiddenApisOutput); + while (violationMatcher.find()) { + violationsClasses.add(violationMatcher.group(1)); + } + + Set sheistyClasses = getSheistyClasses(); + + // keep our whitelist up to date + Set bogusExclusions = new TreeSet<>(excludes); + bogusExclusions.removeAll(missingClasses); + bogusExclusions.removeAll(sheistyClasses); + bogusExclusions.removeAll(violationsClasses); + if (bogusExclusions.isEmpty() == false) { + throw new IllegalStateException( + "Invalid exclusions, nothing is wrong with these classes: " + formatClassList(bogusExclusions) + ); + } + + missingClasses.removeAll(excludes); + violationsClasses.removeAll(excludes); + String missingText = formatClassList(missingClasses); + String violationsText = formatClassList(violationsClasses); + if (missingText.isEmpty() && violationsText.isEmpty()) { + getLogger().info("Third party audit passed successfully"); + } else { + throw new IllegalStateException( + "Audit of third party dependencies failed:\n" + + (missingText.isEmpty() ? "" : "Missing classes:\n" + missingText) + + (violationsText.isEmpty() ? "" : "Classes with violations:\n" + violationsText) + ); + } + + sheistyClasses.removeAll(excludes); + if (sheistyClasses.isEmpty() == false) { + throw new IllegalStateException("Jar Hell with the JDK:" + formatClassList(sheistyClasses)); + } + } + + private String formatClassList(Set classList) { + return classList.stream() + .map(name -> " * " + name) + .collect(Collectors.joining("\n")); + } + + private Set getSheistyClasses() throws IOException { + // system.parent = extensions loader. + // note: for jigsaw, this evilness will need modifications (e.g. use jrt filesystem!) + ClassLoader ext = ClassLoader.getSystemClassLoader().getParent(); + assert ext != null; + + Set sheistySet = new TreeSet<>(); + Path root = getJarExpandDir().toPath(); + Files.walkFileTree(root, new SimpleFileVisitor() { + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) { + String entry = root.relativize(file).toString().replace('\\', '/'); + if (entry.endsWith(".class")) { + if (ext.getResource(entry) != null) { + sheistySet.add( + entry + .replace("/", ".") + .replace(".class","") + ); + } + } + return FileVisitResult.CONTINUE; + } + }); + return sheistySet; + } + + +} diff --git a/buildSrc/src/test/java/org/elasticsearch/gradle/precommit/ThirdPartyAuditNew.java b/buildSrc/src/test/java/org/elasticsearch/gradle/precommit/ThirdPartyAuditNew.java deleted file mode 100644 index a1134fd03ddf8..0000000000000 --- a/buildSrc/src/test/java/org/elasticsearch/gradle/precommit/ThirdPartyAuditNew.java +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.elasticsearch.gradle.precommit; - -import org.gradle.api.file.FileCollection; - -import java.util.ArrayList; -import java.util.List; -import java.util.regex.Pattern; - -public class ThirdPartyAuditNew extends ForbiddenApisCliTask { - - static final Pattern MISSING_CLASS_PATTERN = Pattern.compile( - "WARNING: The referenced class '(.*)' cannot be loaded\\. Please fix the classpath\\!" - ); - - static final Pattern VIOLATION_PATTERN = Pattern.compile( - "\\s\\sin ([a-zA-Z0-9\\$\\.]+) \\(.*\\)" - ); - - // patterns for classes to exclude, because we understand their issues - private List excludes = new ArrayList<>(); - - /** - * Input for the task. Set javadoc for {#link getJars} for more. Protected - * so the afterEvaluate closure in the constructor can write it. - */ - protected FileCollection jars; - - /** - * Classpath against which to run the third patty audit. Protected so the - * afterEvaluate closure in the constructor can write it. - */ - protected FileCollection classpath; - - -} diff --git a/x-pack/plugin/security/build.gradle b/x-pack/plugin/security/build.gradle index 5198c3da66983..caec0a7ec41e8 100644 --- a/x-pack/plugin/security/build.gradle +++ b/x-pack/plugin/security/build.gradle @@ -241,7 +241,7 @@ thirdPartyAudit.excludes = [ 'javax.persistence.EntityManagerFactory', 'javax.persistence.EntityTransaction', 'javax.persistence.LockModeType', - 'javax/persistence/Query', + 'javax.persistence.Query', // [missing classes] OpenSAML storage and HttpClient cache have optional memcache support 'net.spy.memcached.CASResponse', 'net.spy.memcached.CASValue', From 257a72a3e37ec93298af52ef60299148c9278dff Mon Sep 17 00:00:00 2001 From: Alpar Torok Date: Tue, 21 Aug 2018 18:20:35 +0300 Subject: [PATCH 03/10] add todo --- .../org/elasticsearch/gradle/precommit/ThirdPartyAuditTask.java | 1 + 1 file changed, 1 insertion(+) diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/precommit/ThirdPartyAuditTask.java b/buildSrc/src/main/java/org/elasticsearch/gradle/precommit/ThirdPartyAuditTask.java index 65e32f33b44ce..1df3038b28bf7 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/precommit/ThirdPartyAuditTask.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/precommit/ThirdPartyAuditTask.java @@ -190,6 +190,7 @@ public void runThirdPartyAudit() throws IOException { ); } + // TODO: This either no longer works, or we have no such classes sheistyClasses.removeAll(excludes); if (sheistyClasses.isEmpty() == false) { throw new IllegalStateException("Jar Hell with the JDK:" + formatClassList(sheistyClasses)); From 586f082b648a838dfd508e1dee160f35e3866411 Mon Sep 17 00:00:00 2001 From: Alpar Torok Date: Wed, 22 Aug 2018 09:55:27 +0300 Subject: [PATCH 04/10] Run third party audit with runtime java --- buildSrc/build.gradle | 1 - .../gradle/precommit/PrecommitTasks.groovy | 10 ++++++++-- .../gradle/precommit/ForbiddenApisCliTask.java | 3 +-- .../gradle/precommit/ThirdPartyAuditTask.java | 18 ++++++++++++++---- 4 files changed, 23 insertions(+), 9 deletions(-) diff --git a/buildSrc/build.gradle b/buildSrc/build.gradle index 967c2e27ee8df..9918d54d70737 100644 --- a/buildSrc/build.gradle +++ b/buildSrc/build.gradle @@ -102,7 +102,6 @@ dependencies { compile 'com.netflix.nebula:gradle-info-plugin:3.0.3' compile 'org.eclipse.jgit:org.eclipse.jgit:3.2.0.201312181205-r' compile 'com.perforce:p4java:2012.3.551082' // THIS IS SUPPOSED TO BE OPTIONAL IN THE FUTURE.... - compile 'de.thetaphi:forbiddenapis:2.5' compile 'org.apache.rat:apache-rat:0.11' compile "org.elasticsearch:jna:4.5.1" compile 'com.github.jengelman.gradle.plugins:shadow:2.0.4' diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/PrecommitTasks.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/PrecommitTasks.groovy index 73ac591ea6955..f405095fdf0d9 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/PrecommitTasks.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/PrecommitTasks.groovy @@ -78,8 +78,14 @@ class PrecommitTasks { private static Task configureThirdPartyAudit(Project project) { ThirdPartyAuditTask thirdPartyAuditTask = project.tasks.create('thirdPartyAudit', ThirdPartyAuditTask.class) ExportElasticsearchBuildResourcesTask buildResources = project.tasks.getByName('buildResources') - thirdPartyAuditTask.dependsOn(buildResources) - thirdPartyAuditTask.signatureFile = buildResources.copy("forbidden/third-party-audit.txt") + thirdPartyAuditTask.configure { + dependsOn(buildResources) + signatureFile = buildResources.copy("forbidden/third-party-audit.txt") + execAction = { spec -> + spec.classpath(project.configurations.forbiddenApisCliJar) + spec.executable = "${project.runtimeJavaHome}/bin/java" + } + } return thirdPartyAuditTask } diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/precommit/ForbiddenApisCliTask.java b/buildSrc/src/main/java/org/elasticsearch/gradle/precommit/ForbiddenApisCliTask.java index e33f167096414..e3f6aadab52af 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/precommit/ForbiddenApisCliTask.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/precommit/ForbiddenApisCliTask.java @@ -18,7 +18,6 @@ */ package org.elasticsearch.gradle.precommit; -import de.thetaphi.forbiddenapis.cli.CliMain; import org.gradle.api.Action; import org.gradle.api.DefaultTask; import org.gradle.api.JavaVersion; @@ -123,7 +122,7 @@ public void setSuppressAnnotations(Set suppressAnnotations) { public void runForbiddenApisAndWriteMarker() throws IOException { getProject().javaexec((JavaExecSpec spec) -> { execAction.execute(spec); - spec.setMain(CliMain.class.getName()); + spec.setMain("de.thetaphi.forbiddenapis.cli.CliMain"); // build the command line getSignaturesFiles().forEach(file -> spec.args("-f", file.getAbsolutePath())); getSuppressAnnotations().forEach(annotation -> spec.args("--suppressannotation", annotation)); diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/precommit/ThirdPartyAuditTask.java b/buildSrc/src/main/java/org/elasticsearch/gradle/precommit/ThirdPartyAuditTask.java index 1df3038b28bf7..76de95af2b74b 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/precommit/ThirdPartyAuditTask.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/precommit/ThirdPartyAuditTask.java @@ -18,8 +18,8 @@ */ package org.elasticsearch.gradle.precommit; -import de.thetaphi.forbiddenapis.cli.CliMain; import org.apache.commons.io.output.NullOutputStream; +import org.gradle.api.Action; import org.gradle.api.DefaultTask; import org.gradle.api.artifacts.Configuration; import org.gradle.api.file.FileCollection; @@ -29,6 +29,7 @@ import org.gradle.api.tasks.OutputDirectory; import org.gradle.api.tasks.StopExecutionException; import org.gradle.api.tasks.TaskAction; +import org.gradle.process.JavaExecSpec; import java.io.ByteArrayOutputStream; import java.io.File; @@ -63,6 +64,8 @@ public class ThirdPartyAuditTask extends DefaultTask { private File signatureFile; + private Action execAction; + @InputFile public File getSignatureFile() { return signatureFile; @@ -81,6 +84,14 @@ public Configuration getRuntimeConfiguration() { return runtime; } + public Action getExecAction() { + return execAction; + } + + public void setExecAction(Action execAction) { + this.execAction = execAction; + } + @InputFiles public Configuration getCompileOnlyConfiguration() { return getProject().getConfigurations().getByName("compileOnly"); @@ -133,12 +144,12 @@ public void runThirdPartyAudit() throws IOException { ByteArrayOutputStream errorOut = new ByteArrayOutputStream(); getProject().javaexec(spec -> { + execAction.execute(spec); spec.classpath( - getProject().getConfigurations().getByName("forbiddenApisCliJar"), getRuntimeConfiguration(), getCompileOnlyConfiguration() ); - spec.setMain(CliMain.class.getName()); + spec.setMain("de.thetaphi.forbiddenapis.cli.CliMain"); spec.args( "-f", getSignatureFile().getAbsolutePath(), "-d", getJarExpandDir(), @@ -190,7 +201,6 @@ public void runThirdPartyAudit() throws IOException { ); } - // TODO: This either no longer works, or we have no such classes sheistyClasses.removeAll(excludes); if (sheistyClasses.isEmpty() == false) { throw new IllegalStateException("Jar Hell with the JDK:" + formatClassList(sheistyClasses)); From f7b4c9ec5082dc79498ac2e35b9f0ef4cc63ae50 Mon Sep 17 00:00:00 2001 From: Alpar Torok Date: Wed, 22 Aug 2018 15:35:45 +0300 Subject: [PATCH 05/10] Fix third party audit up-to-date checks on java version change --- .../gradle/precommit/PrecommitTasks.groovy | 7 ++-- .../gradle/precommit/ThirdPartyAuditTask.java | 36 ++++++++++++------- 2 files changed, 26 insertions(+), 17 deletions(-) diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/PrecommitTasks.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/PrecommitTasks.groovy index f405095fdf0d9..6fb7e2b6972a5 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/PrecommitTasks.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/PrecommitTasks.groovy @@ -81,10 +81,7 @@ class PrecommitTasks { thirdPartyAuditTask.configure { dependsOn(buildResources) signatureFile = buildResources.copy("forbidden/third-party-audit.txt") - execAction = { spec -> - spec.classpath(project.configurations.forbiddenApisCliJar) - spec.executable = "${project.runtimeJavaHome}/bin/java" - } + javaHome = project.runtimeJavaHome } return thirdPartyAuditTask } @@ -103,7 +100,7 @@ class PrecommitTasks { dependsOn(buildResources) execAction = { spec -> spec.classpath = project.files( - project.configurations.forbiddenApisCliJar, + forbiddenApisConfiguration, sourceSet.compileClasspath, sourceSet.runtimeClasspath ) diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/precommit/ThirdPartyAuditTask.java b/buildSrc/src/main/java/org/elasticsearch/gradle/precommit/ThirdPartyAuditTask.java index 76de95af2b74b..f191caf76ea99 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/precommit/ThirdPartyAuditTask.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/precommit/ThirdPartyAuditTask.java @@ -19,7 +19,6 @@ package org.elasticsearch.gradle.precommit; import org.apache.commons.io.output.NullOutputStream; -import org.gradle.api.Action; import org.gradle.api.DefaultTask; import org.gradle.api.artifacts.Configuration; import org.gradle.api.file.FileCollection; @@ -29,7 +28,7 @@ import org.gradle.api.tasks.OutputDirectory; import org.gradle.api.tasks.StopExecutionException; import org.gradle.api.tasks.TaskAction; -import org.gradle.process.JavaExecSpec; +import org.gradle.process.ExecResult; import java.io.ByteArrayOutputStream; import java.io.File; @@ -64,7 +63,12 @@ public class ThirdPartyAuditTask extends DefaultTask { private File signatureFile; - private Action execAction; + private String javaHome; + + @Input + public Configuration getForbiddenAPIsConfiguration() { + return getProject().getConfigurations().getByName("forbiddenApisCliJar"); + } @InputFile public File getSignatureFile() { @@ -84,12 +88,13 @@ public Configuration getRuntimeConfiguration() { return runtime; } - public Action getExecAction() { - return execAction; + @Input + public String getJavaHome() { + return javaHome; } - public void setExecAction(Action execAction) { - this.execAction = execAction; + public void setJavaHome(String javaHome) { + this.javaHome = javaHome; } @InputFiles @@ -143,9 +148,11 @@ public void runThirdPartyAudit() throws IOException { ); ByteArrayOutputStream errorOut = new ByteArrayOutputStream(); - getProject().javaexec(spec -> { - execAction.execute(spec); + ExecResult execResult = getProject().javaexec(spec -> { + + spec.setExecutable(javaHome + "/bin/java"); spec.classpath( + getForbiddenAPIsConfiguration(), getRuntimeConfiguration(), getCompileOnlyConfiguration() ); @@ -156,15 +163,20 @@ public void runThirdPartyAudit() throws IOException { "--allowmissingclasses" ); spec.setErrorOutput(errorOut); - spec.setStandardOutput(new NullOutputStream()); + if (getLogger().isInfoEnabled() == false) { + spec.setStandardOutput(new NullOutputStream()); + } spec.setIgnoreExitValue(true); }); - final Set missingClasses = new TreeSet<>(); - final Set violationsClasses = new TreeSet<>(); final String forbiddenApisOutput; try (ByteArrayOutputStream outputStream = errorOut) { forbiddenApisOutput = outputStream.toString(StandardCharsets.UTF_8.name()); } + if (getLogger().isInfoEnabled()) { + getLogger().info(forbiddenApisOutput); + } + final Set missingClasses = new TreeSet<>(); + final Set violationsClasses = new TreeSet<>(); Matcher missingMatcher = MISSING_CLASS_PATTERN.matcher(forbiddenApisOutput); while (missingMatcher.find()) { missingClasses.add(missingMatcher.group(1)); From 9a90ce39654216a9d3a3c1f00a4a1b7c173e9c0a Mon Sep 17 00:00:00 2001 From: Alpar Torok Date: Wed, 22 Aug 2018 15:54:46 +0300 Subject: [PATCH 06/10] Make the forbidden apis task sensitive to java evrsion changes --- .../gradle/precommit/PrecommitTasks.groovy | 26 +++------- .../precommit/ForbiddenApisCliTask.java | 47 ++++++++++++++----- .../gradle/precommit/ThirdPartyAuditTask.java | 2 +- 3 files changed, 44 insertions(+), 31 deletions(-) diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/PrecommitTasks.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/PrecommitTasks.groovy index 6fb7e2b6972a5..d82302c84742e 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/PrecommitTasks.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/PrecommitTasks.groovy @@ -31,6 +31,11 @@ class PrecommitTasks { /** Adds a precommit task, which depends on non-test verification tasks. */ public static Task create(Project project, boolean includeDependencyLicenses) { + Configuration forbiddenApisConfiguration = project.configurations.create("forbiddenApisCliJar") + project.dependencies { + forbiddenApisCliJar ('de.thetaphi:forbiddenapis:2.5') + } + List precommitTasks = [ configureCheckstyle(project), configureForbiddenApisCli(project), @@ -87,31 +92,14 @@ class PrecommitTasks { } private static Task configureForbiddenApisCli(Project project) { - Configuration forbiddenApisConfiguration = project.configurations.create("forbiddenApisCliJar") - project.dependencies { - forbiddenApisCliJar ('de.thetaphi:forbiddenapis:2.5') - } Task forbiddenApisCli = project.tasks.create('forbiddenApis') - project.sourceSets.forEach { sourceSet -> forbiddenApisCli.dependsOn( project.tasks.create(sourceSet.getTaskName('forbiddenApis', null), ForbiddenApisCliTask) { ExportElasticsearchBuildResourcesTask buildResources = project.tasks.getByName('buildResources') dependsOn(buildResources) - execAction = { spec -> - spec.classpath = project.files( - forbiddenApisConfiguration, - sourceSet.compileClasspath, - sourceSet.runtimeClasspath - ) - spec.executable = "${project.runtimeJavaHome}/bin/java" - } - inputs.files( - forbiddenApisConfiguration, - sourceSet.compileClasspath, - sourceSet.runtimeClasspath - ) - + it.sourceSet = sourceSet + javaHome = project.runtimeJavaHome targetCompatibility = project.compilerJavaVersion bundledSignatures = [ "jdk-unsafe", "jdk-deprecated", "jdk-non-portable", "jdk-system-out" diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/precommit/ForbiddenApisCliTask.java b/buildSrc/src/main/java/org/elasticsearch/gradle/precommit/ForbiddenApisCliTask.java index e3f6aadab52af..d585b5d9cd41f 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/precommit/ForbiddenApisCliTask.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/precommit/ForbiddenApisCliTask.java @@ -18,14 +18,15 @@ */ package org.elasticsearch.gradle.precommit; -import org.gradle.api.Action; import org.gradle.api.DefaultTask; import org.gradle.api.JavaVersion; +import org.gradle.api.artifacts.Configuration; import org.gradle.api.file.FileCollection; import org.gradle.api.tasks.Input; import org.gradle.api.tasks.InputFiles; import org.gradle.api.tasks.OutputFile; import org.gradle.api.tasks.SkipWhenEmpty; +import org.gradle.api.tasks.SourceSet; import org.gradle.api.tasks.TaskAction; import org.gradle.process.JavaExecSpec; @@ -46,8 +47,10 @@ public class ForbiddenApisCliTask extends DefaultTask { private Set suppressAnnotations = new LinkedHashSet<>(); private JavaVersion targetCompatibility; private FileCollection classesDirs; - private Action execAction; + private SourceSet sourceSet; + private String javaHome; + @Input public JavaVersion getTargetCompatibility() { return targetCompatibility; } @@ -56,14 +59,6 @@ public void setTargetCompatibility(JavaVersion targetCompatibility) { this.targetCompatibility = targetCompatibility; } - public Action getExecAction() { - return execAction; - } - - public void setExecAction(Action execAction) { - this.execAction = execAction; - } - @OutputFile public File getMarkerFile() { return new File( @@ -118,10 +113,40 @@ public void setSuppressAnnotations(Set suppressAnnotations) { this.suppressAnnotations = suppressAnnotations; } + @InputFiles + public FileCollection getClassPathFromSourceSet() { + return getProject().files( + sourceSet.getCompileClasspath(), + sourceSet.getRuntimeClasspath() + ); + } + + public void setSourceSet(SourceSet sourceSet) { + this.sourceSet = sourceSet; + } + + @InputFiles + public Configuration getForbiddenAPIsConfiguration() { + return getProject().getConfigurations().getByName("forbiddenApisCliJar"); + } + + @Input + public String getJavaHome() { + return javaHome; + } + + public void setJavaHome(String javaHome) { + this.javaHome = javaHome; + } + @TaskAction public void runForbiddenApisAndWriteMarker() throws IOException { getProject().javaexec((JavaExecSpec spec) -> { - execAction.execute(spec); + spec.classpath( + getForbiddenAPIsConfiguration(), + getClassPathFromSourceSet() + ); + spec.setExecutable(getJavaHome() + "/bin/java"); spec.setMain("de.thetaphi.forbiddenapis.cli.CliMain"); // build the command line getSignaturesFiles().forEach(file -> spec.args("-f", file.getAbsolutePath())); diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/precommit/ThirdPartyAuditTask.java b/buildSrc/src/main/java/org/elasticsearch/gradle/precommit/ThirdPartyAuditTask.java index f191caf76ea99..606d5efbc9466 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/precommit/ThirdPartyAuditTask.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/precommit/ThirdPartyAuditTask.java @@ -65,7 +65,7 @@ public class ThirdPartyAuditTask extends DefaultTask { private String javaHome; - @Input + @InputFiles public Configuration getForbiddenAPIsConfiguration() { return getProject().getConfigurations().getByName("forbiddenApisCliJar"); } From 85c17354610e463702699e1d095c06cd67efbb1f Mon Sep 17 00:00:00 2001 From: Alpar Torok Date: Wed, 22 Aug 2018 17:34:05 +0300 Subject: [PATCH 07/10] Conditional exclusions should be based on runtime java --- plugins/discovery-azure-classic/build.gradle | 2 +- plugins/discovery-ec2/build.gradle | 2 +- plugins/ingest-attachment/build.gradle | 2 +- plugins/repository-hdfs/build.gradle | 2 +- plugins/repository-s3/build.gradle | 2 +- server/build.gradle | 19 ++++++++++++------- test/logger-usage/build.gradle | 2 +- x-pack/plugin/security/build.gradle | 2 +- x-pack/plugin/sql/sql-action/build.gradle | 2 +- x-pack/plugin/watcher/build.gradle | 2 +- 10 files changed, 21 insertions(+), 16 deletions(-) diff --git a/plugins/discovery-azure-classic/build.gradle b/plugins/discovery-azure-classic/build.gradle index 6f177f7b7f5b2..3dae3d3642c54 100644 --- a/plugins/discovery-azure-classic/build.gradle +++ b/plugins/discovery-azure-classic/build.gradle @@ -128,7 +128,7 @@ thirdPartyAudit.excludes = [ ] // jarhell with jdk (intentionally, because jaxb was removed from default modules in java 9) -if (JavaVersion.current() <= JavaVersion.VERSION_1_8) { +if (project.runtimeJavaVersion <= JavaVersion.VERSION_1_8) { thirdPartyAudit.excludes += [ 'javax.xml.bind.Binder', 'javax.xml.bind.ContextFinder$1', diff --git a/plugins/discovery-ec2/build.gradle b/plugins/discovery-ec2/build.gradle index b1c3b62fd6edf..e32ba6948d62d 100644 --- a/plugins/discovery-ec2/build.gradle +++ b/plugins/discovery-ec2/build.gradle @@ -87,7 +87,7 @@ thirdPartyAudit.excludes = [ 'org.apache.log.Logger', ] -if (JavaVersion.current() > JavaVersion.VERSION_1_8) { +if (project.runtimeJavaVersion > JavaVersion.VERSION_1_8) { thirdPartyAudit.excludes += [ 'javax.xml.bind.DatatypeConverter', 'javax.xml.bind.JAXBContext' diff --git a/plugins/ingest-attachment/build.gradle b/plugins/ingest-attachment/build.gradle index 1a6aa809de040..fb7314e931787 100644 --- a/plugins/ingest-attachment/build.gradle +++ b/plugins/ingest-attachment/build.gradle @@ -2106,7 +2106,7 @@ thirdPartyAudit.excludes = [ 'ucar.nc2.dataset.NetcdfDataset' ] -if (JavaVersion.current() > JavaVersion.VERSION_1_8) { +if (project.runtimeJavaVersion > JavaVersion.VERSION_1_8) { thirdPartyAudit.excludes += [ 'javax.activation.ActivationDataFlavor', 'javax.activation.CommandMap', diff --git a/plugins/repository-hdfs/build.gradle b/plugins/repository-hdfs/build.gradle index 6debaf5282fc1..26bdc03feddab 100644 --- a/plugins/repository-hdfs/build.gradle +++ b/plugins/repository-hdfs/build.gradle @@ -582,6 +582,6 @@ thirdPartyAudit.excludes = [ 'com.squareup.okhttp.ResponseBody' ] -if (JavaVersion.current() > JavaVersion.VERSION_1_8) { +if (project.runtimeJavaVersion > JavaVersion.VERSION_1_8) { thirdPartyAudit.excludes += ['javax.xml.bind.annotation.adapters.HexBinaryAdapter'] } diff --git a/plugins/repository-s3/build.gradle b/plugins/repository-s3/build.gradle index 7f0ca209db797..5d248b22caf16 100644 --- a/plugins/repository-s3/build.gradle +++ b/plugins/repository-s3/build.gradle @@ -447,7 +447,7 @@ thirdPartyAudit.excludes = [ ] // jarhell with jdk (intentionally, because jaxb was removed from default modules in java 9) -if (JavaVersion.current() <= JavaVersion.VERSION_1_8) { +if (project.runtimeJavaVersion <= JavaVersion.VERSION_1_8) { thirdPartyAudit.excludes += [ 'javax.xml.bind.Binder', 'javax.xml.bind.ContextFinder$1', diff --git a/server/build.gradle b/server/build.gradle index b22a93a702c2b..f8a604941f7e0 100644 --- a/server/build.gradle +++ b/server/build.gradle @@ -304,17 +304,22 @@ thirdPartyAudit.excludes = [ 'com.google.common.geometry.S2LatLng', ] -if (JavaVersion.current() <= JavaVersion.VERSION_1_8) { - // Used by Log4J 2.11.1 +if (project.runtimeJavaVersion <= JavaVersion.VERSION_1_8) { thirdPartyAudit.excludes += [ - 'java.io.ObjectInputFilter', - 'java.io.ObjectInputFilter$Config', - 'java.io.ObjectInputFilter$FilterInfo', - 'java.io.ObjectInputFilter$Status' + // Used by Log4J 2.11.1 + 'java.io.ObjectInputFilter', + 'java.io.ObjectInputFilter$Config', + 'java.io.ObjectInputFilter$FilterInfo', + 'java.io.ObjectInputFilter$Status', + // added in 9 + 'java.lang.ProcessHandle', + 'java.lang.StackWalker', + 'java.lang.StackWalker$Option', + 'java.lang.StackWalker$StackFrame' ] } -if (JavaVersion.current() > JavaVersion.VERSION_1_8) { +if (project.runtimeJavaVersion > JavaVersion.VERSION_1_8) { thirdPartyAudit.excludes += ['javax.xml.bind.DatatypeConverter'] } diff --git a/test/logger-usage/build.gradle b/test/logger-usage/build.gradle index 0f02283e53738..88c6b9e38b48b 100644 --- a/test/logger-usage/build.gradle +++ b/test/logger-usage/build.gradle @@ -44,7 +44,7 @@ thirdPartyAudit.excludes = [ 'org.osgi.framework.wiring.BundleWiring' ] -if (JavaVersion.current() <= JavaVersion.VERSION_1_8) { +if (project.runtimeJavaVersion <= JavaVersion.VERSION_1_8) { // Used by Log4J 2.11.1 thirdPartyAudit.excludes += [ 'java.io.ObjectInputFilter', diff --git a/x-pack/plugin/security/build.gradle b/x-pack/plugin/security/build.gradle index caec0a7ec41e8..1a26d1d03d963 100644 --- a/x-pack/plugin/security/build.gradle +++ b/x-pack/plugin/security/build.gradle @@ -265,7 +265,7 @@ thirdPartyAudit.excludes = [ 'com.google.common.util.concurrent.AbstractFuture$UnsafeAtomicHelper$1', ] -if (JavaVersion.current() > JavaVersion.VERSION_1_8) { +if (project.runtimeJavaVersion > JavaVersion.VERSION_1_8) { thirdPartyAudit.excludes += [ 'javax.xml.bind.JAXBContext', 'javax.xml.bind.JAXBElement', diff --git a/x-pack/plugin/sql/sql-action/build.gradle b/x-pack/plugin/sql/sql-action/build.gradle index 345318d20b803..f850b24ab46be 100644 --- a/x-pack/plugin/sql/sql-action/build.gradle +++ b/x-pack/plugin/sql/sql-action/build.gradle @@ -140,7 +140,7 @@ thirdPartyAudit.excludes = [ 'org.zeromq.ZMQ' ] -if (JavaVersion.current() <= JavaVersion.VERSION_1_8) { +if (project.runtimeJavaVersion <= JavaVersion.VERSION_1_8) { // Used by Log4J 2.11.1 thirdPartyAudit.excludes += [ 'java.io.ObjectInputFilter', diff --git a/x-pack/plugin/watcher/build.gradle b/x-pack/plugin/watcher/build.gradle index 3a9d759c46d12..3412cafc4f4ce 100644 --- a/x-pack/plugin/watcher/build.gradle +++ b/x-pack/plugin/watcher/build.gradle @@ -68,7 +68,7 @@ thirdPartyAudit.excludes = [ ] // pulled in as external dependency to work on java 9 -if (JavaVersion.current() <= JavaVersion.VERSION_1_8) { +if (project.runtimeJavaVersion <= JavaVersion.VERSION_1_8) { thirdPartyAudit.excludes += [ 'com.sun.activation.registries.MailcapParseException', 'javax.activation.ActivationDataFlavor', From 60727aeeb643a48347370bfd6ce4edb3690deeed Mon Sep 17 00:00:00 2001 From: Alpar Torok Date: Wed, 22 Aug 2018 17:34:30 +0300 Subject: [PATCH 08/10] run jdk jar hell check against runtime java --- .../elasticsearch/gradle/JdkJarHellCheck.java | 81 +++++++++++++++++++ .../gradle/precommit/ThirdPartyAuditTask.java | 63 ++++++++------- 2 files changed, 116 insertions(+), 28 deletions(-) create mode 100644 buildSrc/src/main/java/org/elasticsearch/gradle/JdkJarHellCheck.java diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/JdkJarHellCheck.java b/buildSrc/src/main/java/org/elasticsearch/gradle/JdkJarHellCheck.java new file mode 100644 index 0000000000000..60de1981f9827 --- /dev/null +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/JdkJarHellCheck.java @@ -0,0 +1,81 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.gradle; + +import java.io.IOException; +import java.nio.file.FileVisitResult; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.attribute.BasicFileAttributes; +import java.util.Collections; +import java.util.HashSet; +import java.util.Set; + +public class JdkJarHellCheck { + + private Set detected = new HashSet<>(); + + private void scanForJDKJarHell(Path root) throws IOException { + // system.parent = extensions loader. + // note: for jigsaw, this evilness will need modifications (e.g. use jrt filesystem!) + ClassLoader ext = ClassLoader.getSystemClassLoader().getParent(); + assert ext != null; + + Files.walkFileTree(root, new SimpleFileVisitor() { + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) { + String entry = root.relativize(file).toString().replace('\\', '/'); + if (entry.endsWith(".class")) { + if (ext.getResource(entry) != null) { + detected.add( + entry + .replace("/", ".") + .replace(".class","") + ); + } + } + return FileVisitResult.CONTINUE; + } + }); + } + + public Set getDetected() { + return Collections.unmodifiableSet(detected); + } + + public static void main(String[] argv) throws IOException { + JdkJarHellCheck checker = new JdkJarHellCheck(); + for (String location : argv) { + Path path = Paths.get(location); + if (Files.exists(path) == false) { + throw new IllegalArgumentException("Path does not exist: " + path); + } + checker.scanForJDKJarHell(path); + } + if (checker.getDetected().isEmpty()) { + System.exit(0); + } else { + checker.getDetected().forEach(System.out::println); + System.exit(1); + } + } + +} diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/precommit/ThirdPartyAuditTask.java b/buildSrc/src/main/java/org/elasticsearch/gradle/precommit/ThirdPartyAuditTask.java index 606d5efbc9466..d6c66a51792be 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/precommit/ThirdPartyAuditTask.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/precommit/ThirdPartyAuditTask.java @@ -19,7 +19,10 @@ package org.elasticsearch.gradle.precommit; import org.apache.commons.io.output.NullOutputStream; +import org.elasticsearch.gradle.JdkJarHellCheck; +import org.elasticsearch.test.NamingConventionsCheck; import org.gradle.api.DefaultTask; +import org.gradle.api.GradleException; import org.gradle.api.artifacts.Configuration; import org.gradle.api.file.FileCollection; import org.gradle.api.tasks.Input; @@ -33,12 +36,10 @@ import java.io.ByteArrayOutputStream; import java.io.File; import java.io.IOException; +import java.net.URISyntaxException; +import java.net.URL; import java.nio.charset.StandardCharsets; -import java.nio.file.FileVisitResult; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.SimpleFileVisitor; -import java.nio.file.attribute.BasicFileAttributes; +import java.util.Arrays; import java.util.Collections; import java.util.Set; import java.util.TreeSet; @@ -149,7 +150,6 @@ public void runThirdPartyAudit() throws IOException { ByteArrayOutputStream errorOut = new ByteArrayOutputStream(); ExecResult execResult = getProject().javaexec(spec -> { - spec.setExecutable(javaHome + "/bin/java"); spec.classpath( getForbiddenAPIsConfiguration(), @@ -226,30 +226,37 @@ private String formatClassList(Set classList) { } private Set getSheistyClasses() throws IOException { - // system.parent = extensions loader. - // note: for jigsaw, this evilness will need modifications (e.g. use jrt filesystem!) - ClassLoader ext = ClassLoader.getSystemClassLoader().getParent(); - assert ext != null; - - Set sheistySet = new TreeSet<>(); - Path root = getJarExpandDir().toPath(); - Files.walkFileTree(root, new SimpleFileVisitor() { - @Override - public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) { - String entry = root.relativize(file).toString().replace('\\', '/'); - if (entry.endsWith(".class")) { - if (ext.getResource(entry) != null) { - sheistySet.add( - entry - .replace("/", ".") - .replace(".class","") - ); - } - } - return FileVisitResult.CONTINUE; + ByteArrayOutputStream standardOut = new ByteArrayOutputStream(); + ExecResult execResult = getProject().javaexec(spec -> { + URL location = NamingConventionsCheck.class.getProtectionDomain().getCodeSource().getLocation(); + if (location.getProtocol().equals("file") == false) { + throw new GradleException("Unexpected location for NamingConventionCheck class: " + location); } + try { + spec.classpath( + location.toURI().getPath(), + getRuntimeConfiguration(), + getCompileOnlyConfiguration() + ); + } catch (URISyntaxException e) { + throw new AssertionError(e); + } + spec.setMain(JdkJarHellCheck.class.getName()); + spec.args(getJarExpandDir()); + spec.setIgnoreExitValue(true); + spec.setExecutable(javaHome + "/bin/java"); + spec.setStandardOutput(standardOut); }); - return sheistySet; + if (execResult.getExitValue() == 0) { + return Collections.emptySet(); + } + final String jdkJarHellCheckList; + try (ByteArrayOutputStream outputStream = standardOut) { + jdkJarHellCheckList = outputStream.toString(StandardCharsets.UTF_8.name()); + } + return Collections.unmodifiableSet( + new TreeSet<>(Arrays.asList(jdkJarHellCheckList.split("\\r?\\n"))) + ); } From fe624ae6c7290eea3a7ec7a6918085ad95de32be Mon Sep 17 00:00:00 2001 From: Alpar Torok Date: Wed, 22 Aug 2018 18:35:49 +0300 Subject: [PATCH 09/10] Add additional exclusions for java 8 --- .../gradle/precommit/ThirdPartyAuditTask.java | 4 +--- plugins/ingest-attachment/build.gradle | 20 +++++++++++++++++++ plugins/repository-hdfs/build.gradle | 19 ++++++++++++++++++ test/logger-usage/build.gradle | 9 +++++++++ x-pack/plugin/sql/sql-action/build.gradle | 9 +++++++++ 5 files changed, 58 insertions(+), 3 deletions(-) diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/precommit/ThirdPartyAuditTask.java b/buildSrc/src/main/java/org/elasticsearch/gradle/precommit/ThirdPartyAuditTask.java index d6c66a51792be..b87761f04307b 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/precommit/ThirdPartyAuditTask.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/precommit/ThirdPartyAuditTask.java @@ -254,9 +254,7 @@ private Set getSheistyClasses() throws IOException { try (ByteArrayOutputStream outputStream = standardOut) { jdkJarHellCheckList = outputStream.toString(StandardCharsets.UTF_8.name()); } - return Collections.unmodifiableSet( - new TreeSet<>(Arrays.asList(jdkJarHellCheckList.split("\\r?\\n"))) - ); + return new TreeSet<>(Arrays.asList(jdkJarHellCheckList.split("\\r?\\n"))); } diff --git a/plugins/ingest-attachment/build.gradle b/plugins/ingest-attachment/build.gradle index fb7314e931787..6cd55f682c8b4 100644 --- a/plugins/ingest-attachment/build.gradle +++ b/plugins/ingest-attachment/build.gradle @@ -2106,6 +2106,26 @@ thirdPartyAudit.excludes = [ 'ucar.nc2.dataset.NetcdfDataset' ] +if (project.runtimeJavaVersion == JavaVersion.VERSION_1_8) { + thirdPartyAudit.excludes += [ + // TODO: Why is this needed ? + 'com.sun.javadoc.ClassDoc', + 'com.sun.javadoc.Doc', + 'com.sun.javadoc.Doclet', + 'com.sun.javadoc.ExecutableMemberDoc', + 'com.sun.javadoc.FieldDoc', + 'com.sun.javadoc.MethodDoc', + 'com.sun.javadoc.PackageDoc', + 'com.sun.javadoc.Parameter', + 'com.sun.javadoc.ProgramElementDoc', + 'com.sun.javadoc.RootDoc', + 'com.sun.javadoc.SourcePosition', + 'com.sun.javadoc.Tag', + 'com.sun.javadoc.Type', + 'com.sun.tools.javadoc.Main' + ] +} + if (project.runtimeJavaVersion > JavaVersion.VERSION_1_8) { thirdPartyAudit.excludes += [ 'javax.activation.ActivationDataFlavor', diff --git a/plugins/repository-hdfs/build.gradle b/plugins/repository-hdfs/build.gradle index 26bdc03feddab..557dcaa5faedc 100644 --- a/plugins/repository-hdfs/build.gradle +++ b/plugins/repository-hdfs/build.gradle @@ -585,3 +585,22 @@ thirdPartyAudit.excludes = [ if (project.runtimeJavaVersion > JavaVersion.VERSION_1_8) { thirdPartyAudit.excludes += ['javax.xml.bind.annotation.adapters.HexBinaryAdapter'] } + +if (project.runtimeJavaVersion == JavaVersion.VERSION_1_8) { + thirdPartyAudit.excludes += [ + // TODO: Why is this needed ? + 'com.sun.javadoc.AnnotationDesc', + 'com.sun.javadoc.AnnotationTypeDoc', + 'com.sun.javadoc.ClassDoc', + 'com.sun.javadoc.ConstructorDoc', + 'com.sun.javadoc.Doc', + 'com.sun.javadoc.DocErrorReporter', + 'com.sun.javadoc.FieldDoc', + 'com.sun.javadoc.LanguageVersion', + 'com.sun.javadoc.MethodDoc', + 'com.sun.javadoc.PackageDoc', + 'com.sun.javadoc.ProgramElementDoc', + 'com.sun.javadoc.RootDoc', + 'com.sun.tools.doclets.standard.Standard' + ] +} diff --git a/test/logger-usage/build.gradle b/test/logger-usage/build.gradle index 88c6b9e38b48b..2da9065641436 100644 --- a/test/logger-usage/build.gradle +++ b/test/logger-usage/build.gradle @@ -52,4 +52,13 @@ if (project.runtimeJavaVersion <= JavaVersion.VERSION_1_8) { 'java.io.ObjectInputFilter$FilterInfo', 'java.io.ObjectInputFilter$Status' ] +} + +if (project.runtimeJavaVersion == JavaVersion.VERSION_1_8) { + thirdPartyAudit.excludes += [ + 'java.lang.ProcessHandle', + 'java.lang.StackWalker', + 'java.lang.StackWalker$Option', + 'java.lang.StackWalker$StackFrame' + ] } \ No newline at end of file diff --git a/x-pack/plugin/sql/sql-action/build.gradle b/x-pack/plugin/sql/sql-action/build.gradle index f850b24ab46be..ee99c36b90627 100644 --- a/x-pack/plugin/sql/sql-action/build.gradle +++ b/x-pack/plugin/sql/sql-action/build.gradle @@ -148,4 +148,13 @@ if (project.runtimeJavaVersion <= JavaVersion.VERSION_1_8) { 'java.io.ObjectInputFilter$FilterInfo', 'java.io.ObjectInputFilter$Status' ] +} + +if (project.runtimeJavaVersion == JavaVersion.VERSION_1_8) { + thirdPartyAudit.excludes += [ + 'java.lang.ProcessHandle', + 'java.lang.StackWalker', + 'java.lang.StackWalker$Option', + 'java.lang.StackWalker$StackFrame' + ] } \ No newline at end of file From 8aff435de7fec50ffddb702242441827575b7e63 Mon Sep 17 00:00:00 2001 From: Alpar Torok Date: Thu, 23 Aug 2018 08:17:15 +0300 Subject: [PATCH 10/10] Refactor for readability --- .../gradle/precommit/ThirdPartyAuditTask.java | 127 +++++++++++------- 1 file changed, 77 insertions(+), 50 deletions(-) diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/precommit/ThirdPartyAuditTask.java b/buildSrc/src/main/java/org/elasticsearch/gradle/precommit/ThirdPartyAuditTask.java index b87761f04307b..d1939d5c65261 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/precommit/ThirdPartyAuditTask.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/precommit/ThirdPartyAuditTask.java @@ -128,18 +128,34 @@ public Set getExcludes() { @TaskAction public void runThirdPartyAudit() throws IOException { - FileCollection jars = getRuntimeConfiguration() - .fileCollection(dep -> dep.getGroup().startsWith("org.elasticsearch") == false); - Configuration compileOnlyConfiguration = getCompileOnlyConfiguration(); - // don't scan provided dependencies that we already scanned, e.x. don't scan cores dependencies for every plugin - if (compileOnlyConfiguration != null) { - jars.minus(compileOnlyConfiguration); + FileCollection jars = getJarsToScan(); + + extractJars(jars); + + final String forbiddenApisOutput = runForbiddenAPIsCli(); + + final Set missingClasses = new TreeSet<>(); + Matcher missingMatcher = MISSING_CLASS_PATTERN.matcher(forbiddenApisOutput); + while (missingMatcher.find()) { + missingClasses.add(missingMatcher.group(1)); } - if (jars.isEmpty()) { - throw new StopExecutionException("No jars to scan"); + + final Set violationsClasses = new TreeSet<>(); + Matcher violationMatcher = VIOLATION_PATTERN.matcher(forbiddenApisOutput); + while (violationMatcher.find()) { + violationsClasses.add(violationMatcher.group(1)); } - // Stage all the jars + Set jdkJarHellClasses = runJdkJarHellCheck(); + + assertNoPointlessExclusions(missingClasses, violationsClasses, jdkJarHellClasses); + + assertNoMissingAndViolations(missingClasses, violationsClasses); + + assertNoJarHell(jdkJarHellClasses); + } + + private void extractJars(FileCollection jars) { File jarExpandDir = getJarExpandDir(); jars.forEach(jar -> getProject().copy(spec -> { @@ -147,9 +163,47 @@ public void runThirdPartyAudit() throws IOException { spec.into(jarExpandDir); }) ); + } + + private void assertNoJarHell(Set jdkJarHellClasses) { + jdkJarHellClasses.removeAll(excludes); + if (jdkJarHellClasses.isEmpty() == false) { + throw new IllegalStateException("Jar Hell with the JDK:" + formatClassList(jdkJarHellClasses)); + } + } + private void assertNoMissingAndViolations(Set missingClasses, Set violationsClasses) { + missingClasses.removeAll(excludes); + violationsClasses.removeAll(excludes); + String missingText = formatClassList(missingClasses); + String violationsText = formatClassList(violationsClasses); + if (missingText.isEmpty() && violationsText.isEmpty()) { + getLogger().info("Third party audit passed successfully"); + } else { + throw new IllegalStateException( + "Audit of third party dependencies failed:\n" + + (missingText.isEmpty() ? "" : "Missing classes:\n" + missingText) + + (violationsText.isEmpty() ? "" : "Classes with violations:\n" + violationsText) + ); + } + } + + private void assertNoPointlessExclusions(Set missingClasses, Set violationsClasses, Set jdkJarHellClasses) { + // keep our whitelist up to date + Set bogusExclusions = new TreeSet<>(excludes); + bogusExclusions.removeAll(missingClasses); + bogusExclusions.removeAll(jdkJarHellClasses); + bogusExclusions.removeAll(violationsClasses); + if (bogusExclusions.isEmpty() == false) { + throw new IllegalStateException( + "Invalid exclusions, nothing is wrong with these classes: " + formatClassList(bogusExclusions) + ); + } + } + + private String runForbiddenAPIsCli() throws IOException { ByteArrayOutputStream errorOut = new ByteArrayOutputStream(); - ExecResult execResult = getProject().javaexec(spec -> { + getProject().javaexec(spec -> { spec.setExecutable(javaHome + "/bin/java"); spec.classpath( getForbiddenAPIsConfiguration(), @@ -175,48 +229,21 @@ public void runThirdPartyAudit() throws IOException { if (getLogger().isInfoEnabled()) { getLogger().info(forbiddenApisOutput); } - final Set missingClasses = new TreeSet<>(); - final Set violationsClasses = new TreeSet<>(); - Matcher missingMatcher = MISSING_CLASS_PATTERN.matcher(forbiddenApisOutput); - while (missingMatcher.find()) { - missingClasses.add(missingMatcher.group(1)); - } - Matcher violationMatcher = VIOLATION_PATTERN.matcher(forbiddenApisOutput); - while (violationMatcher.find()) { - violationsClasses.add(violationMatcher.group(1)); - } - - Set sheistyClasses = getSheistyClasses(); - - // keep our whitelist up to date - Set bogusExclusions = new TreeSet<>(excludes); - bogusExclusions.removeAll(missingClasses); - bogusExclusions.removeAll(sheistyClasses); - bogusExclusions.removeAll(violationsClasses); - if (bogusExclusions.isEmpty() == false) { - throw new IllegalStateException( - "Invalid exclusions, nothing is wrong with these classes: " + formatClassList(bogusExclusions) - ); - } + return forbiddenApisOutput; + } - missingClasses.removeAll(excludes); - violationsClasses.removeAll(excludes); - String missingText = formatClassList(missingClasses); - String violationsText = formatClassList(violationsClasses); - if (missingText.isEmpty() && violationsText.isEmpty()) { - getLogger().info("Third party audit passed successfully"); - } else { - throw new IllegalStateException( - "Audit of third party dependencies failed:\n" + - (missingText.isEmpty() ? "" : "Missing classes:\n" + missingText) + - (violationsText.isEmpty() ? "" : "Classes with violations:\n" + violationsText) - ); + private FileCollection getJarsToScan() { + FileCollection jars = getRuntimeConfiguration() + .fileCollection(dep -> dep.getGroup().startsWith("org.elasticsearch") == false); + Configuration compileOnlyConfiguration = getCompileOnlyConfiguration(); + // don't scan provided dependencies that we already scanned, e.x. don't scan cores dependencies for every plugin + if (compileOnlyConfiguration != null) { + jars.minus(compileOnlyConfiguration); } - - sheistyClasses.removeAll(excludes); - if (sheistyClasses.isEmpty() == false) { - throw new IllegalStateException("Jar Hell with the JDK:" + formatClassList(sheistyClasses)); + if (jars.isEmpty()) { + throw new StopExecutionException("No jars to scan"); } + return jars; } private String formatClassList(Set classList) { @@ -225,7 +252,7 @@ private String formatClassList(Set classList) { .collect(Collectors.joining("\n")); } - private Set getSheistyClasses() throws IOException { + private Set runJdkJarHellCheck() throws IOException { ByteArrayOutputStream standardOut = new ByteArrayOutputStream(); ExecResult execResult = getProject().javaexec(spec -> { URL location = NamingConventionsCheck.class.getProtectionDomain().getCodeSource().getLocation();