diff --git a/substratevm/Resources.md b/substratevm/Resources.md index b102dba9a734..d66497ec5fa5 100644 --- a/substratevm/Resources.md +++ b/substratevm/Resources.md @@ -5,21 +5,28 @@ To make calls such as `Class.getResource()` or `Class.getResourceAsStream()` (or ```json { - "resources": [ - {"pattern": ""}, - {"pattern": ""}, - ... - ] + "resources": { + "includes": [ + {"pattern": ""}, + {"pattern": ""}, + ... + ], + "excludes": [ + {"pattern": ""}, + {"pattern": ""}, + ... + ] + } } ``` -The configuration file's path must be provided to `native-image` with `-H:ResourceConfigurationFiles=/path/to/resource-config.json`. Alternatively, individual resource paths can also be specified directly to `native-image`: -```shell +The configuration file's path must be provided to the native image builder with `-H:ResourceConfigurationFiles=/path/to/resource-config.json`. Alternatively, individual resource paths can also be specified directly to `native-image`: +``` native-image -H:IncludeResources= ... ``` -The `-H:IncludeResources` option can be passed several times to define more than one regexp to match resources. +The `-H:IncludeResources` and `-H:ExcludeResources` options can be passed several times to define more than one regexp to match or exclude resources, respectively. -To see which resources get included into the image, you can enable the related logging info with `-H:Log=registerResource:`. +To see which resources get ultimately included into the image, you can enable the related logging info with `-H:Log=registerResource:`. ### Example Usage @@ -41,6 +48,7 @@ Then: * `Resource0.txt` can be loaded with `.*/Resource0.txt$`. * `Resource0.txt` and `Resource1.txt` can be loaded with `.*/Resource0.txt$` and `.*/Resource1.txt$` (or alternatively with a single `.*/(Resource0|Resource1).txt$`). +* Also, if we want to include everything except the `Resource2.txt` file, we can simply exclude it with `-H:IncludeResources='.*/Resource.*txt$'` followed by `-H:ExcludeResources='.*/Resource2.txt$'`. See also the [guide on assisted configuration of Java resources and other dynamic features](Configuration.md#assisted-configuration-of-native-image-builds). diff --git a/substratevm/mx.substratevm/suite.py b/substratevm/mx.substratevm/suite.py index a06037a91ff8..73f2ec218c49 100644 --- a/substratevm/mx.substratevm/suite.py +++ b/substratevm/mx.substratevm/suite.py @@ -588,6 +588,24 @@ "testProject": True, }, + "com.oracle.svm.configure.test": { + "subDir": "src", + "sourceDirs": ["src"], + "dependencies": [ + "mx:JUNIT_TOOL", + "sdk:GRAAL_SDK", + "com.oracle.svm.configure", + ], + "checkstyle": "com.oracle.svm.core", + "workingSets": "SVM", + "annotationProcessors": [ + "compiler:GRAAL_PROCESSOR", + ], + "javaCompliance": "8+", + "spotbugs": "false", + "testProject": True, + }, + "com.oracle.svm.reflect": { "subDir": "src", "sourceDirs": ["src"], @@ -1126,10 +1144,12 @@ "dependencies" : [ "com.oracle.svm.test", "com.oracle.svm.test.jdk11", + "com.oracle.svm.configure.test", ], "distDependencies": [ "mx:JUNIT_TOOL", "sdk:GRAAL_SDK", + "SVM_CONFIGURE", ], "testDistribution" : True, }, diff --git a/substratevm/src/com.oracle.svm.configure.test/src/com/oracle/svm/configure/test/config/ResourceConfigurationTest.java b/substratevm/src/com.oracle.svm.configure.test/src/com/oracle/svm/configure/test/config/ResourceConfigurationTest.java new file mode 100644 index 000000000000..b6eb11674069 --- /dev/null +++ b/substratevm/src/com.oracle.svm.configure.test/src/com/oracle/svm/configure/test/config/ResourceConfigurationTest.java @@ -0,0 +1,123 @@ +/* + * Copyright (c) 2015, 2020, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +package com.oracle.svm.configure.test.config; + +import java.io.IOException; +import java.io.PipedReader; +import java.io.PipedWriter; +import java.util.LinkedList; +import java.util.List; + +import org.junit.Assert; +import org.junit.Test; + +import com.oracle.svm.configure.config.ResourceConfiguration; +import com.oracle.svm.configure.json.JsonWriter; +import com.oracle.svm.core.configure.ResourceConfigurationParser; +import com.oracle.svm.core.configure.ResourcesRegistry; + +public class ResourceConfigurationTest { + + @Test + public void anyResourceMatches() { + ResourceConfiguration rc = new ResourceConfiguration(); + rc.addResourcePattern(".*/Resource.*txt$"); + + Assert.assertTrue(rc.anyResourceMatches("com/my/app/Resource0.txt")); + Assert.assertTrue(rc.anyResourceMatches("com/my/app/Resource1.txt")); + Assert.assertTrue(rc.anyResourceMatches("/Resource2.txt")); + Assert.assertTrue(rc.anyResourceMatches("/Resource3.txt")); + + rc.ignoreResourcePattern(".*/Resource2.txt$"); + + Assert.assertTrue(rc.anyResourceMatches("com/my/app/Resource0.txt")); + Assert.assertTrue(rc.anyResourceMatches("com/my/app/Resource1.txt")); + Assert.assertFalse(rc.anyResourceMatches("/Resource2.txt")); + Assert.assertTrue(rc.anyResourceMatches("/Resource3.txt")); + } + + @Test + public void printJson() { + ResourceConfiguration rc = new ResourceConfiguration(); + rc.addResourcePattern(".*/Resource.*txt$"); + rc.ignoreResourcePattern(".*/Resource2.txt$"); + PipedWriter pw = new PipedWriter(); + JsonWriter jw = new JsonWriter(pw); + + try (PipedReader pr = new PipedReader()) { + pr.connect(pw); + + Thread writerThread = new Thread(new Runnable() { + + @Override + public void run() { + try { + rc.printJson(jw); + } catch (IOException e) { + Assert.fail(e.getMessage()); + } finally { + try { + jw.close(); + } catch (IOException e) { + } + } + } + }); + + List addedResources = new LinkedList<>(); + List ignoredResources = new LinkedList<>(); + + ResourcesRegistry registry = new ResourcesRegistry() { + + @Override + public void addResources(String pattern) { + addedResources.add(pattern); + } + + @Override + public void ignoreResources(String pattern) { + ignoredResources.add(pattern); + } + + @Override + public void addResourceBundles(String name) { + } + }; + + ResourceConfigurationParser rcp = new ResourceConfigurationParser(registry); + writerThread.start(); + rcp.parseAndRegister(pr); + + writerThread.join(); + + Assert.assertTrue(addedResources.contains(".*/Resource.*txt$")); + Assert.assertTrue(ignoredResources.contains(".*/Resource2.txt$")); + } catch (IOException | InterruptedException e) { + e.printStackTrace(); + Assert.fail(e.getMessage()); + } + } +} diff --git a/substratevm/src/com.oracle.svm.configure/src/com/oracle/svm/configure/config/ResourceConfiguration.java b/substratevm/src/com.oracle.svm.configure/src/com/oracle/svm/configure/config/ResourceConfiguration.java index 075018133c88..d37fd77e30cc 100644 --- a/substratevm/src/com.oracle.svm.configure/src/com/oracle/svm/configure/config/ResourceConfiguration.java +++ b/substratevm/src/com.oracle.svm.configure/src/com/oracle/svm/configure/config/ResourceConfiguration.java @@ -49,17 +49,27 @@ public void addResources(String pattern) { configuration.addResourcePattern(pattern); } + @Override + public void ignoreResources(String pattern) { + configuration.ignoreResourcePattern(pattern); + } + @Override public void addResourceBundles(String name) { configuration.addBundle(name); } } - private final ConcurrentMap resources = new ConcurrentHashMap<>(); + private final ConcurrentMap addedResources = new ConcurrentHashMap<>(); + private final ConcurrentMap ignoredResources = new ConcurrentHashMap<>(); private final ConcurrentHashMap.KeySetView bundles = ConcurrentHashMap.newKeySet(); public void addResourcePattern(String pattern) { - resources.computeIfAbsent(pattern, Pattern::compile); + addedResources.computeIfAbsent(pattern, Pattern::compile); + } + + public void ignoreResourcePattern(String pattern) { + ignoredResources.computeIfAbsent(pattern, Pattern::compile); } public void addBundle(String bundle) { @@ -71,7 +81,12 @@ public boolean anyResourceMatches(String s) { * Naive -- if the need arises, we could match in the order of most frequently matched * patterns, or somehow merge the patterns into a single big pattern. */ - for (Pattern pattern : resources.values()) { + for (Pattern pattern : ignoredResources.values()) { + if (pattern.matcher(s).matches()) { + return false; + } + } + for (Pattern pattern : addedResources.values()) { if (pattern.matcher(s).matches()) { return true; } @@ -86,9 +101,15 @@ public boolean anyBundleMatches(String s) { @Override public void printJson(JsonWriter writer) throws IOException { writer.append('{').indent().newline(); - writer.quote("resources").append(':'); - JsonPrinter.printCollection(writer, resources.keySet(), Comparator.naturalOrder(), (String p, JsonWriter w) -> w.append('{').quote("pattern").append(':').quote(p).append('}')); - writer.append(',').newline(); + writer.quote("resources").append(':').append('{').newline(); + writer.quote("includes").append(':'); + JsonPrinter.printCollection(writer, addedResources.keySet(), Comparator.naturalOrder(), (String p, JsonWriter w) -> w.append('{').quote("pattern").append(':').quote(p).append('}')); + if (!ignoredResources.isEmpty()) { + writer.append(',').newline(); + writer.quote("excludes").append(':'); + JsonPrinter.printCollection(writer, ignoredResources.keySet(), Comparator.naturalOrder(), (String p, JsonWriter w) -> w.append('{').quote("pattern").append(':').quote(p).append('}')); + } + writer.append('}').append(',').newline(); writer.quote("bundles").append(':'); JsonPrinter.printCollection(writer, bundles, Comparator.naturalOrder(), (String p, JsonWriter w) -> w.append('{').quote("name").append(':').quote(p).append('}')); writer.unindent().newline().append('}').newline(); diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/configure/ResourceConfigurationParser.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/configure/ResourceConfigurationParser.java index 012888356358..bf688d45ef34 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/configure/ResourceConfigurationParser.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/configure/ResourceConfigurationParser.java @@ -60,9 +60,35 @@ private void parseTopLevelObject(Map obj) { } } if (resourcesObject != null) { - List resources = asList(resourcesObject, "Attribute 'resources' must be a list of resources"); - for (Object object : resources) { - parseEntry(object, "pattern", registry::addResources, "resource descriptor object", "'resources' list"); + if (resourcesObject instanceof Map) { // New format + Object includesObject = null; + Object excludesObject = null; + for (Map.Entry pair : ((Map) resourcesObject).entrySet()) { + if ("includes".equals(pair.getKey())) { + includesObject = pair.getValue(); + } else if ("excludes".equals(pair.getKey())) { + excludesObject = pair.getValue(); + } else { + throw new JSONParserException("Unknown attribute '" + pair.getKey() + "' (supported attributes: name) in resource definition"); + } + } + + List includes = asList(includesObject, "Attribute 'includes' must be a list of resources"); + for (Object object : includes) { + parseEntry(object, "pattern", registry::addResources, "resource descriptor object", "'includes' list"); + } + + if (excludesObject != null) { + List excludes = asList(excludesObject, "Attribute 'excludes' must be a list of resources"); + for (Object object : excludes) { + parseEntry(object, "pattern", registry::ignoreResources, "resource descriptor object", "'excludes' list"); + } + } + } else { // Old format: may be deprecated in future versions + List resources = asList(resourcesObject, "Attribute 'resources' must be a list of resources"); + for (Object object : resources) { + parseEntry(object, "pattern", registry::addResources, "resource descriptor object", "'resources' list"); + } } } if (bundlesObject != null) { diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/configure/ResourcesRegistry.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/configure/ResourcesRegistry.java index a249a144bc7c..afd44513c0e2 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/configure/ResourcesRegistry.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/configure/ResourcesRegistry.java @@ -27,5 +27,7 @@ public interface ResourcesRegistry { void addResources(String pattern); + void ignoreResources(String pattern); + void addResourceBundles(String name); } diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java index b6896208093b..14efeb319312 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java @@ -74,10 +74,14 @@ public final class ResourcesFeature implements Feature { public static class Options { @Option(help = "Regexp to match names of resources to be included in the image.", type = OptionType.User)// public static final HostedOptionKey IncludeResources = new HostedOptionKey<>(new String[0]); + + @Option(help = "Regexp to match names of resources to be excluded from the image.", type = OptionType.User)// + public static final HostedOptionKey ExcludeResources = new HostedOptionKey<>(new String[0]); } private boolean sealed = false; private Set newResources = Collections.newSetFromMap(new ConcurrentHashMap<>()); + private Set ignoredResources = Collections.newSetFromMap(new ConcurrentHashMap<>()); private int loadedConfigurations; private class ResourcesRegistryImpl implements ResourcesRegistry { @@ -87,6 +91,12 @@ public void addResources(String pattern) { newResources.add(pattern); } + @Override + public void ignoreResources(String pattern) { + UserError.guarantee(!sealed, "Resources ignored too late: %s", pattern); + ignoredResources.add(pattern); + } + @Override public void addResourceBundles(String name) { ImageSingletons.lookup(LocalizationFeature.class).addBundleToCache(name); @@ -107,6 +117,7 @@ public void beforeAnalysis(BeforeAnalysisAccess access) { ConfigurationFiles.RESOURCES_NAME); newResources.addAll(Arrays.asList(Options.IncludeResources.getValue())); + ignoredResources.addAll(Arrays.asList(Options.ExcludeResources.getValue())); } @Override @@ -117,15 +128,12 @@ public void duringAnalysis(DuringAnalysisAccess access) { access.requireAnalysisIteration(); DebugContext debugContext = ((DuringAnalysisAccessImpl) access).getDebugContext(); - final Pattern[] patterns = newResources.stream() - .filter(s -> s.length() > 0) - .map(Pattern::compile) - .collect(Collectors.toList()) - .toArray(new Pattern[]{}); + final Pattern[] includePatterns = compilePatterns(newResources); + final Pattern[] excludePatterns = compilePatterns(ignoredResources); if (JavaVersionUtil.JAVA_SPEC > 8) { try { - ModuleSupport.findResourcesInModules(name -> matches(patterns, name), + ModuleSupport.findResourcesInModules(name -> matches(includePatterns, excludePatterns, name), (resName, content) -> registerResource(debugContext, resName, content)); } catch (IOException ex) { throw UserError.abort(ex, "Can not read resources from modules. This is possible due to incorrect module path or missing module visibility directives"); @@ -159,9 +167,9 @@ public void duringAnalysis(DuringAnalysisAccess access) { for (File element : todo) { try { if (element.isDirectory()) { - scanDirectory(debugContext, element, "", patterns); + scanDirectory(debugContext, element, "", includePatterns, excludePatterns); } else { - scanJar(debugContext, element, patterns); + scanJar(debugContext, element, includePatterns, excludePatterns); } } catch (IOException ex) { throw UserError.abort("Unable to handle classpath element '%s'. Make sure that all classpath entries are either directories or valid jar files.", element); @@ -170,6 +178,14 @@ public void duringAnalysis(DuringAnalysisAccess access) { newResources.clear(); } + private static Pattern[] compilePatterns(Set patterns) { + return patterns.stream() + .filter(s -> s.length() > 0) + .map(Pattern::compile) + .collect(Collectors.toList()) + .toArray(new Pattern[]{}); + } + @Override public void afterAnalysis(AfterAnalysisAccess access) { sealed = true; @@ -186,18 +202,18 @@ public void beforeCompilation(BeforeCompilationAccess access) { } } - private void scanDirectory(DebugContext debugContext, File f, String relativePath, Pattern... patterns) throws IOException { + private void scanDirectory(DebugContext debugContext, File f, String relativePath, Pattern[] includePatterns, Pattern[] excludePatterns) throws IOException { if (f.isDirectory()) { File[] files = f.listFiles(); if (files == null) { throw UserError.abort("Cannot scan directory %s", f); } else { for (File ch : files) { - scanDirectory(debugContext, ch, relativePath.isEmpty() ? ch.getName() : relativePath + "/" + ch.getName(), patterns); + scanDirectory(debugContext, ch, relativePath.isEmpty() ? ch.getName() : relativePath + "/" + ch.getName(), includePatterns, excludePatterns); } } } else { - if (matches(patterns, relativePath)) { + if (matches(includePatterns, excludePatterns, relativePath)) { try (FileInputStream is = new FileInputStream(f)) { registerResource(debugContext, relativePath, is); } @@ -205,7 +221,7 @@ private void scanDirectory(DebugContext debugContext, File f, String relativePat } } - private static void scanJar(DebugContext debugContext, File element, Pattern... patterns) throws IOException { + private static void scanJar(DebugContext debugContext, File element, Pattern[] includePatterns, Pattern[] excludePatterns) throws IOException { JarFile jf = new JarFile(element); Enumeration en = jf.entries(); @@ -216,13 +232,13 @@ private static void scanJar(DebugContext debugContext, File element, Pattern... if (e.isDirectory()) { String dirName = e.getName().substring(0, e.getName().length() - 1); allEntries.add(dirName); - if (matches(patterns, dirName)) { + if (matches(includePatterns, excludePatterns, dirName)) { matchedDirectoryResources.put(dirName, new ArrayList<>()); } continue; } allEntries.add(e.getName()); - if (matches(patterns, e.getName())) { + if (matches(includePatterns, excludePatterns, e.getName())) { try (InputStream is = jf.getInputStream(e)) { registerResource(debugContext, e.getName(), is); } @@ -244,12 +260,19 @@ private static void scanJar(DebugContext debugContext, File element, Pattern... }); } - private static boolean matches(Pattern[] patterns, String relativePath) { - for (Pattern p : patterns) { + private static boolean matches(Pattern[] includePatterns, Pattern[] excludePatterns, String relativePath) { + for (Pattern p : excludePatterns) { + if (p.matcher(relativePath).matches()) { + return false; + } + } + + for (Pattern p : includePatterns) { if (p.matcher(relativePath).matches()) { return true; } } + return false; }