Skip to content

Commit

Permalink
ensure exclusions win over inclusions for scanning
Browse files Browse the repository at this point in the history 
using maven shared utils and adding a flag to have a fast scan
  • Loading branch information
@rmannibucau
rmannibucau committed Jun 9, 2020
1 parent 91b730a commit 20faa4d
Show file tree
Hide file tree
Showing 5 changed files with 162 additions and 12 deletions.
12 changes: 10 additions & 2 deletions .travis.yml
View file
Original file line number Diff line number Diff line change
@@ -1,8 +1,16 @@
dist: trusty

sudo: false
language: java

script: mvn clean install --quiet -B
before_cache:
- find $HOME/.m2/repository/ -name *SNAPSHOT | xargs rm -Rf

cache:
timeout: 1000
directories:
- "$HOME/.m2"

script: mvn clean install --quiet -B -Dmaven.artifact.threads=64 -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn

install: /bin/true

Expand Down
7 changes: 6 additions & 1 deletion license-maven-plugin/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -122,7 +122,12 @@
<dependency>
<groupId>org.apache.maven</groupId>
<artifactId>maven-settings-builder</artifactId>
</dependency>
</dependency>
<dependency>
<groupId>org.apache.maven.shared</groupId>
<artifactId>maven-shared-utils</artifactId>
<version>3.2.1</version>
</dependency>

<dependency>
<groupId>org.springframework</groupId>
Expand Down
4 changes: 3 additions & 1 deletion license-maven-plugin/src/main/java/com/mycila/maven/plugin/license/AbstractLicenseMojo.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -601,7 +601,9 @@ private Map<String, String> mergeProperties(final LicenseSet licenseSet, final D

private String[] listSelectedFiles(final LicenseSet licenseSet) {
final boolean useDefaultExcludes = (licenseSet.useDefaultExcludes != null ? licenseSet.useDefaultExcludes : defaultUseDefaultExcludes);
final Selection selection = new Selection(firstNonNull(licenseSet.basedir, defaultBasedir), licenseSet.includes, buildExcludes(licenseSet), useDefaultExcludes);
final Selection selection = new Selection(
firstNonNull(licenseSet.basedir, defaultBasedir), licenseSet.includes, buildExcludes(licenseSet), useDefaultExcludes,
getLog());
debug("From: %s", firstNonNull(licenseSet.basedir, defaultBasedir));
debug("Including: %s", deepToString(selection.getIncluded()));
debug("Excluding: %s", deepToString(selection.getExcluded()));
Expand Down
58 changes: 53 additions & 5 deletions license-maven-plugin/src/main/java/com/mycila/maven/plugin/license/util/Selection.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,14 +16,18 @@
package com.mycila.maven.plugin.license.util;

import com.mycila.maven.plugin.license.Default;
import org.codehaus.plexus.util.DirectoryScanner;
import org.apache.maven.plugin.logging.Log;
import org.apache.maven.shared.utils.io.DirectoryScanner;
import org.apache.maven.shared.utils.io.MatchPatterns;
import org.apache.maven.shared.utils.io.ScanConductor;

import java.io.File;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.concurrent.TimeUnit;

import static java.util.Arrays.*;
import static java.util.Arrays.asList;

/**
Expand All @@ -35,13 +39,18 @@ public final class Selection {
private final File basedir;
private final String[] included;
private final String[] excluded;
private final Log log;
private final String[] userExcluded;

private DirectoryScanner scanner;

public Selection(File basedir, String[] included, String[] excluded, boolean useDefaultExcludes) {
public Selection(File basedir, String[] included, String[] excluded, boolean useDefaultExcludes,
final Log log) {
this.basedir = basedir;
this.log = log;
String[] overrides = buildOverrideInclusions(useDefaultExcludes, included);
this.included = buildInclusions(included, overrides);
this.userExcluded = excluded;
this.excluded = buildExclusions(useDefaultExcludes, excluded, overrides);
}

Expand All @@ -50,6 +59,11 @@ public String[] getSelectedFiles() {
return scanner.getIncludedFiles();
}

// for tests
DirectoryScanner getScanner() {
return scanner;
}

public File getBasedir() {
return basedir;
}
Expand All @@ -64,14 +78,49 @@ public String[] getExcluded() {

private void scanIfneeded() {
if (scanner == null) {
final boolean debugEnabled = log.isDebugEnabled();
final String[] folderExcludes = findFolderExcludes();
final MatchPatterns excludePatterns = MatchPatterns.from(folderExcludes);
if (debugEnabled) {
log.debug("Starting to visit '" + basedir + "' with excludes: " + asList(folderExcludes));
}
scanner = new DirectoryScanner();
scanner.setScanConductor(new ScanConductor() {
@Override
public ScanAction visitDirectory(final String name, final File directory) {
if (excludePatterns.matches(name, true)) {
return ScanAction.NO_RECURSE;
}
return ScanAction.CONTINUE;
}

@Override
public ScanAction visitFile(final String name, final File file) {
return ScanAction.CONTINUE;
}
});
scanner.setBasedir(basedir);
scanner.setIncludes(included);
scanner.setExcludes(excluded);
scanner.scan();
}
}

private String[] findFolderExcludes() { // less we keep, less overhead we get so we only use user excludes there
final List<String> excludes = new ArrayList<String>(excluded.length / 2 /*estimate*/);
for (final String exclude : (userExcluded != null ? userExcluded : excluded)) {
if (isFolderExclusion(exclude)) {
excludes.add(exclude);
}
}
Collections.reverse(excludes); // assume user ones are more important than the set of defaults we appended
return excludes.toArray(new String[0]);
}

private boolean isFolderExclusion(final String exclude) {
return exclude.endsWith(File.separator + "**");
}

private static String[] buildExclusions(boolean useDefaultExcludes, String[] excludes, String[] overrides) {
List<String> exclusions = new ArrayList<String>();
if (useDefaultExcludes) {
Expand Down Expand Up @@ -106,5 +155,4 @@ private static String[] buildOverrideInclusions(boolean useDefaultExcludes, Stri
overrides.retainAll(asList(includes));
return overrides.toArray(new String[0]);
}

}
93 changes: 90 additions & 3 deletions license-maven-plugin/src/test/java/com/mycila/maven/plugin/license/util/SelectionTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,40 +16,127 @@
package com.mycila.maven.plugin.license.util;

import com.mycila.maven.plugin.license.Default;
import org.apache.maven.plugin.logging.Log;
import org.apache.maven.plugin.logging.SystemStreamLog;
import org.apache.maven.shared.utils.io.DirectoryScanner;
import org.junit.Test;

import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.UUID;

import static java.util.Arrays.asList;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;

/**
* @author Mathieu Carbou (mathieu.carbou@gmail.com)
*/
public final class SelectionTest {
private final Log log = new SystemStreamLog();

@Test
public void test_default_select_all() {
Selection selection = new Selection(new File("."), new String[0], new String[0], false);
Selection selection = new Selection(new File("."), new String[0], new String[0], false, log);
assertEquals(selection.getExcluded().length, 0);
assertEquals(selection.getIncluded().length, 1);
assertTrue(selection.getSelectedFiles().length > 0);
}

@Test
public void test_limit_inclusion() {
Selection selection = new Selection(new File("."), new String[]{"toto"}, new String[]{"tata"}, false);
Selection selection = new Selection(new File("."), new String[]{"toto"}, new String[]{"tata"}, false, log);
assertEquals(selection.getExcluded().length, 1);
assertEquals(selection.getIncluded().length, 1);
assertEquals(selection.getSelectedFiles().length, 0);
}

@Test
public void test_limit_inclusion_and_check_default_excludes() {
Selection selection = new Selection(new File("."), new String[]{"toto"}, new String[0], true);
Selection selection = new Selection(new File("."), new String[]{"toto"}, new String[0], true, log);
assertEquals(selection.getExcluded().length, Default.EXCLUDES.length); // default exludes from Scanner and Selection + toto
assertEquals(selection.getIncluded().length, 1);
assertEquals(selection.getSelectedFiles().length, 0);
assertTrue(Arrays.asList(selection.getExcluded()).containsAll(Arrays.asList(Default.EXCLUDES)));
}

@Test
public void test_exclusions_respect_with_fastScan() throws IOException {
SystemStreamLog log = new SystemStreamLog() {
@Override
public boolean isDebugEnabled() {
return true;
}
};
File root = createAFakeProject(log);
Selection selection = new Selection(root,
new String[] { "**" + File.separator + "*.txt" },
new String[] {"target" + File.separator + "**", "module/**/target" + File.separator + "**"}, false,
log);

selection.getSelectedFiles(); // triggers scan and scanner build
String debugMessage = buildDebugMessage(selection.getScanner());
assertIncludedFilesInFakeProject(selection, debugMessage);
assertEquals(debugMessage, 0, selection.getScanner().getExcludedFiles().length);
}

private String buildDebugMessage(DirectoryScanner scanner) {
return "excludedDirs=" + asList(scanner.getExcludedDirectories()) + ",\n" +
"excludedFiles=" + asList(scanner.getExcludedFiles()) + ",\n" +
"includedDirsFiles=" + asList(scanner.getIncludedDirectories()) + ",\n" +
"includedFiles=" + asList(scanner.getIncludedFiles()) + ",\n" +
"notIncludedDirs=" + asList(scanner.getNotIncludedDirectories()) + ",\n" +
"notIncludedFiles=" + asList(scanner.getNotIncludedFiles()) + ",\n" +
"diskFiles=" + listFiles(scanner.getBasedir(), new ArrayList<File>());
}

private Collection<File> listFiles(File basedir, Collection<File> files) {
files.add(basedir);
for (File f : basedir.listFiles()) {
if (f.isDirectory()) {
listFiles(f, files);
} else {
files.add(f);
}
}
return files;
}

private void assertIncludedFilesInFakeProject(Selection selection, String debugMessage) {
List<String> selected = new ArrayList<String>(asList(selection.getSelectedFiles()));
Collections.sort(selected);
assertEquals(debugMessage, asList("included.txt", "module/src/main/java/not-ignored.txt", "module/sub/subsub/src/main/java/not-ignored.txt"), selected);
}

private File createAFakeProject(Log log) throws IOException {
File temp = new File("target/workdir_" + UUID.randomUUID().toString());
touch(new File(temp, "included.txt"), log);
touch(new File(temp, "target/ignored.txt"), log);
touch(new File(temp, "module/src/main/java/not-ignored.txt"), log);
touch(new File(temp, "module/target/ignored.txt"), log);
touch(new File(temp, "module/sub/subsub/src/main/java/not-ignored.txt"), log);
touch(new File(temp, "module/sub/subsub/target/foo/not-ignored.txt"), log);
return temp;
}

private void touch(File newFile, Log log) throws IOException {
final File parentFile = newFile.getParentFile();
if (parentFile != null && !parentFile.isDirectory() && !parentFile.mkdirs()) {
fail("Can't create '" + parentFile + "'");
}
final FileWriter w = new FileWriter(newFile);
w.write("touched");
w.close();
if (!newFile.exists()) {
fail("Can't create " + newFile);
}
log.debug("Created '" + newFile.getAbsolutePath() + "'");
}
}

0 comments on commit 20faa4d

Please sign in to comment.