From a2125e1a9894dab60456d031270b1c472baab4a8 Mon Sep 17 00:00:00 2001 From: David Nestorovic Date: Thu, 7 Mar 2024 17:09:07 +0100 Subject: [PATCH 01/19] Print registered resources and sizes in terminal --- .../com/oracle/svm/core/jdk/Resources.java | 36 +++++++++++-------- .../NativeImageResourceFileSystem.java | 5 ++- .../oracle/svm/hosted/ResourceReporter.java | 33 +++++++++++++++++ .../oracle/svm/hosted/ResourcesFeature.java | 17 ++++++++- 4 files changed, 72 insertions(+), 19 deletions(-) create mode 100644 substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourceReporter.java diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/Resources.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/Resources.java index 949f1a7660ab..49f3e4092690 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/Resources.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/Resources.java @@ -33,6 +33,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.Comparator; import java.util.Date; import java.util.Enumeration; import java.util.List; @@ -43,7 +44,6 @@ import org.graalvm.collections.EconomicMap; import org.graalvm.collections.MapCursor; -import org.graalvm.collections.Pair; import org.graalvm.nativeimage.ImageSingletons; import org.graalvm.nativeimage.Platform; import org.graalvm.nativeimage.Platforms; @@ -81,18 +81,24 @@ public static Resources singleton() { } /** - * The hosted map used to collect registered resources. Using a {@link Pair} of (module, - * resourceName) provides implementations for {@code hashCode()} and {@code equals()} needed for - * the map keys. Hosted module instances differ to runtime instances, so the map that ends up in - * the image heap is computed after the runtime module instances have been computed {see - * com.oracle.svm.hosted.ModuleLayerFeature}. + * The hosted map used to collect registered resources. Using a {@link ModuleResourceRecord} of + * (module, resourceName) provides implementations for {@code hashCode()} and {@code equals()} + * needed for the map keys. Hosted module instances differ to runtime instances, so the map that + * ends up in the image heap is computed after the runtime module instances have been computed + * {see com.oracle.svm.hosted.ModuleLayerFeature}. */ - private final EconomicMap, ResourceStorageEntryBase> resources = ImageHeapMap.create(); + private final EconomicMap resources = ImageHeapMap.create(); private final EconomicMap requestedPatterns = ImageHeapMap.create(); public record RequestedPattern(String module, String resource) { } + public record ModuleResourceRecord(Module module, String resource) { + public static Comparator comparator() { + return Comparator.comparing(ModuleResourceRecord::resource); + } + } + /** * The object used to mark a resource as reachable according to the metadata. It can be obtained * when accessing the {@link Resources#resources} map, and it means that even though the @@ -106,7 +112,7 @@ public record RequestedPattern(String module, String resource) { * specified in the configuration, but we do not want to throw directly (for example when we try * to check all the modules for a resource). */ - private static final ResourceStorageEntryBase MISSING_METADATA_MARKER = new ResourceStorageEntryBase(); + public static final ResourceStorageEntryBase MISSING_METADATA_MARKER = new ResourceStorageEntryBase(); /** * Embedding a resource into an image is counted as a modification. Since all resources are @@ -118,7 +124,7 @@ public record RequestedPattern(String module, String resource) { Resources() { } - public EconomicMap, ResourceStorageEntryBase> getResourceStorage() { + public EconomicMap getResourceStorage() { return resources; } @@ -138,14 +144,14 @@ public static String moduleName(Module module) { return module == null ? null : module.getName(); } - private static Pair createStorageKey(Module module, String resourceName) { + private static ModuleResourceRecord createStorageKey(Module module, String resourceName) { Module m = module != null && module.isNamed() ? module : null; - return Pair.create(m, resourceName); + return new ModuleResourceRecord(m, resourceName); } public static Set getIncludedResourcesModules() { return StreamSupport.stream(singleton().resources.getKeys().spliterator(), false) - .map(Pair::getLeft) + .map(ModuleResourceRecord::module) .filter(Objects::nonNull) .map(Module::getName) .collect(Collectors.toSet()); @@ -173,7 +179,7 @@ private void addEntry(Module module, String resourceName, boolean isDirectory, b m = RuntimeModuleSupport.instance().getRuntimeModuleForHostedModule(m); } synchronized (resources) { - Pair key = createStorageKey(m, resourceName); + ModuleResourceRecord key = createStorageKey(m, resourceName); ResourceStorageEntryBase entry = resources.get(key); if (isNegativeQuery) { if (entry == null) { @@ -187,7 +193,7 @@ private void addEntry(Module module, String resourceName, boolean isDirectory, b entry = new ResourceStorageEntry(isDirectory, fromJar); resources.put(key, entry); } else { - if (key.getLeft() != null) { + if (key.module() != null) { // if the entry already exists, and it comes from a module, it is the same entry // that we registered at some point before return; @@ -232,7 +238,7 @@ public void registerIOException(Module module, String resourceName, IOException LogUtils.warning("Resource " + resourceName + " from module " + moduleName(module) + " produced the following IOException: " + e.getClass().getTypeName() + ": " + e.getMessage()); } } - Pair key = createStorageKey(module, resourceName); + ModuleResourceRecord key = createStorageKey(module, resourceName); synchronized (resources) { updateTimeStamp(); resources.put(key, new ResourceExceptionEntry(e)); diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/resources/NativeImageResourceFileSystem.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/resources/NativeImageResourceFileSystem.java index 7e8d6bceca69..1f1ba248ecf3 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/resources/NativeImageResourceFileSystem.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/resources/NativeImageResourceFileSystem.java @@ -85,7 +85,6 @@ import java.util.regex.Pattern; import org.graalvm.collections.MapCursor; -import org.graalvm.collections.Pair; import com.oracle.svm.core.MissingRegistrationUtils; import com.oracle.svm.core.jdk.Resources; @@ -657,9 +656,9 @@ private void update(Entry e) { } private void readAllEntries() { - MapCursor, ResourceStorageEntryBase> entries = Resources.singleton().getResourceStorage().getEntries(); + MapCursor entries = Resources.singleton().getResourceStorage().getEntries(); while (entries.advance()) { - byte[] name = getBytes(entries.getKey().getRight()); + byte[] name = getBytes(entries.getKey().resource()); ResourceStorageEntryBase entry = entries.getValue(); if (entry.hasData()) { IndexNode newIndexNode = new IndexNode(name, entry.isDirectory(), true); diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourceReporter.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourceReporter.java new file mode 100644 index 000000000000..2ea9abb37642 --- /dev/null +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourceReporter.java @@ -0,0 +1,33 @@ +package com.oracle.svm.hosted; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.Collection; + +import com.oracle.svm.core.jdk.Resources; +import com.oracle.svm.core.option.HostedOptionValues; +import com.oracle.svm.core.util.VMError; +import com.oracle.svm.core.util.json.JsonPrinter; +import com.oracle.svm.core.util.json.JsonWriter; + +public class ResourceReporter { + + public static void printReport(Collection registeredResources) { + Path reportLocation = NativeImageGenerator.generatedFiles(HostedOptionValues.singleton()).resolve("registered_resources.json"); + try (JsonWriter writer = new JsonWriter(reportLocation)) { + JsonPrinter.printCollection(writer, registeredResources, Resources.ModuleResourceRecord.comparator(), ResourceReporter::resourceReportElement); + } catch (IOException e) { + throw VMError.shouldNotReachHere("Json writer cannot write to: " + reportLocation + "\n Reason: " + e.getMessage()); + } + } + + private static void resourceReportElement(Resources.ModuleResourceRecord p, JsonWriter w) throws IOException { + w.indent().newline(); + w.append('{').indent().newline(); + w.quote("resource").append(':').quote(p.resource()); + w.newline(); + w.quote("module").append(':').quote(p.module() == null ? "ALL_UNNAMED" : p.module().getName()); + w.unindent().newline().append('}'); + w.unindent(); + } +} diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java index f8642070a2db..716b873fea1d 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java @@ -147,6 +147,10 @@ public static class Options { @Option(help = "Regexp to match names of resources to be excluded from the image.", type = OptionType.User)// public static final HostedOptionKey ExcludeResources = new HostedOptionKey<>(LocatableMultiOptionValue.Strings.build()); + + @Option(help = "Create a Native Image registered resources json", type = OptionType.Debug) // + public static final HostedOptionKey DumpRegisteredResources = new HostedOptionKey<>(false); + } private boolean sealed = false; @@ -442,7 +446,7 @@ public void beforeAnalysis(BeforeAnalysisAccess access) { includePatterns.stream() .map(pattern -> pattern.compiledPattern) .forEach(resourcePattern -> { - Resources.singleton().registerIncludePattern(resourcePattern.moduleName, resourcePattern.pattern.pattern()); + Resources.singleton().registerIncludePattern(resourcePattern.moduleName(), resourcePattern.pattern.pattern()); }); } ResourcePattern[] excludePatterns = compilePatterns(excludedResourcePatterns); @@ -596,6 +600,17 @@ boolean moduleNameMatches(String resourceContainerModuleName) { @Override public void afterAnalysis(AfterAnalysisAccess access) { sealed = true; +// if (Options.DumpRegisteredResources.getValue()) { + Iterable registeredResources = Resources.singleton().getResourceStorage().getKeys(); + List resourceInfoList = new ArrayList<>(); + registeredResources.forEach(resource -> { + String resourceName = resource.resource(); + Module module = resource.module(); + resourceInfoList.add(new Resources.ModuleResourceRecord(module, resourceName)); + }); + + ResourceReporter.printReport(resourceInfoList); +// } } @Override From eb342e584deddd5ca0eb0b98e7e10b9ba9e6ccf2 Mon Sep 17 00:00:00 2001 From: David Nestorovic Date: Fri, 8 Mar 2024 14:55:12 +0100 Subject: [PATCH 02/19] Enable option check --- .../com/oracle/svm/core/jdk/Resources.java | 2 +- .../oracle/svm/hosted/ResourceReporter.java | 2 +- .../oracle/svm/hosted/ResourcesFeature.java | 20 +++++++++---------- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/Resources.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/Resources.java index 49f3e4092690..48c7ee1d8be2 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/Resources.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/Resources.java @@ -112,7 +112,7 @@ public static Comparator comparator() { * specified in the configuration, but we do not want to throw directly (for example when we try * to check all the modules for a resource). */ - public static final ResourceStorageEntryBase MISSING_METADATA_MARKER = new ResourceStorageEntryBase(); + private static final ResourceStorageEntryBase MISSING_METADATA_MARKER = new ResourceStorageEntryBase(); /** * Embedding a resource into an image is counted as a modification. Since all resources are diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourceReporter.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourceReporter.java index 2ea9abb37642..e7c411e57a70 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourceReporter.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourceReporter.java @@ -26,7 +26,7 @@ private static void resourceReportElement(Resources.ModuleResourceRecord p, Json w.append('{').indent().newline(); w.quote("resource").append(':').quote(p.resource()); w.newline(); - w.quote("module").append(':').quote(p.module() == null ? "ALL_UNNAMED" : p.module().getName()); + w.quote("module").append(':').quote(p.module() == null ? "ALL-UNNAMED" : p.module().getName()); w.unindent().newline().append('}'); w.unindent(); } diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java index 716b873fea1d..e897095278d2 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java @@ -600,17 +600,17 @@ boolean moduleNameMatches(String resourceContainerModuleName) { @Override public void afterAnalysis(AfterAnalysisAccess access) { sealed = true; -// if (Options.DumpRegisteredResources.getValue()) { - Iterable registeredResources = Resources.singleton().getResourceStorage().getKeys(); - List resourceInfoList = new ArrayList<>(); - registeredResources.forEach(resource -> { - String resourceName = resource.resource(); - Module module = resource.module(); - resourceInfoList.add(new Resources.ModuleResourceRecord(module, resourceName)); - }); + if (Options.DumpRegisteredResources.getValue()) { + Iterable registeredResources = Resources.singleton().getResourceStorage().getKeys(); + List resourceInfoList = new ArrayList<>(); + registeredResources.forEach(resource -> { + String resourceName = resource.resource(); + Module module = resource.module(); + resourceInfoList.add(new Resources.ModuleResourceRecord(module, resourceName)); + }); - ResourceReporter.printReport(resourceInfoList); -// } + ResourceReporter.printReport(resourceInfoList); + } } @Override From e8e3f9f73ca79c97a3d6136c0f9d778c3865e49f Mon Sep 17 00:00:00 2001 From: David Nestorovic Date: Wed, 13 Mar 2024 17:37:07 +0100 Subject: [PATCH 03/19] Remove redudant module transformation and print entries in new format --- .../com/oracle/svm/core/jdk/Resources.java | 10 +---- .../oracle/svm/hosted/ResourceReporter.java | 44 +++++++++++++++---- .../oracle/svm/hosted/ResourcesFeature.java | 38 +++++++++++++--- 3 files changed, 70 insertions(+), 22 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/Resources.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/Resources.java index 48c7ee1d8be2..2c15a457fb8b 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/Resources.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/Resources.java @@ -33,7 +33,6 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; -import java.util.Comparator; import java.util.Date; import java.util.Enumeration; import java.util.List; @@ -94,9 +93,6 @@ public record RequestedPattern(String module, String resource) { } public record ModuleResourceRecord(Module module, String resource) { - public static Comparator comparator() { - return Comparator.comparing(ModuleResourceRecord::resource); - } } /** @@ -144,7 +140,7 @@ public static String moduleName(Module module) { return module == null ? null : module.getName(); } - private static ModuleResourceRecord createStorageKey(Module module, String resourceName) { + public static ModuleResourceRecord createStorageKey(Module module, String resourceName) { Module m = module != null && module.isNamed() ? module : null; return new ModuleResourceRecord(m, resourceName); } @@ -175,9 +171,7 @@ private void updateTimeStamp() { private void addEntry(Module module, String resourceName, boolean isDirectory, byte[] data, boolean fromJar, boolean isNegativeQuery) { VMError.guarantee(!BuildPhaseProvider.isAnalysisFinished(), "Trying to add a resource entry after analysis."); Module m = module != null && module.isNamed() ? module : null; - if (m != null) { - m = RuntimeModuleSupport.instance().getRuntimeModuleForHostedModule(m); - } + synchronized (resources) { ModuleResourceRecord key = createStorageKey(m, resourceName); ResourceStorageEntryBase entry = resources.get(key); diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourceReporter.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourceReporter.java index e7c411e57a70..15a229b00446 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourceReporter.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourceReporter.java @@ -3,8 +3,9 @@ import java.io.IOException; import java.nio.file.Path; import java.util.Collection; +import java.util.Comparator; +import java.util.List; -import com.oracle.svm.core.jdk.Resources; import com.oracle.svm.core.option.HostedOptionValues; import com.oracle.svm.core.util.VMError; import com.oracle.svm.core.util.json.JsonPrinter; @@ -12,22 +13,49 @@ public class ResourceReporter { - public static void printReport(Collection registeredResources) { + public record SourceSizePair(String source, String size) { + // NOTE: size is string because it could be NEGATIVE QUERY + public static Comparator comparator() { + return Comparator.comparing(SourceSizePair::source); + } + } + + public record ResourceReportEntry(Module module, String resourceName, List entries) { + public static Comparator comparator() { + return Comparator.comparing(ResourceReportEntry::resourceName); + } + } + + public static void printReport(Collection registeredResources) { Path reportLocation = NativeImageGenerator.generatedFiles(HostedOptionValues.singleton()).resolve("registered_resources.json"); try (JsonWriter writer = new JsonWriter(reportLocation)) { - JsonPrinter.printCollection(writer, registeredResources, Resources.ModuleResourceRecord.comparator(), ResourceReporter::resourceReportElement); + JsonPrinter.printCollection(writer, registeredResources, ResourceReportEntry.comparator(), ResourceReporter::resourceReportElement); } catch (IOException e) { throw VMError.shouldNotReachHere("Json writer cannot write to: " + reportLocation + "\n Reason: " + e.getMessage()); } } - private static void resourceReportElement(Resources.ModuleResourceRecord p, JsonWriter w) throws IOException { + private static void resourceReportElement(ResourceReportEntry p, JsonWriter w) throws IOException { + w.indent().newline(); + w.appendObjectStart().newline(); + w.appendKeyValue("name", p.resourceName()).appendSeparator(); + w.newline(); + if (p.module() != null) { + w.appendKeyValue("module", p.module().getName()).appendSeparator(); + w.newline(); + } + w.quote("entries").append(":"); + JsonPrinter.printCollection(w, p.entries(), SourceSizePair.comparator(), ResourceReporter::sourceElement); + w.unindent().newline().appendObjectEnd(); + } + + private static void sourceElement(SourceSizePair p, JsonWriter w) throws IOException { w.indent().newline(); - w.append('{').indent().newline(); - w.quote("resource").append(':').quote(p.resource()); + w.appendObjectStart().newline(); + w.appendKeyValue("origin", p.source()).appendSeparator(); w.newline(); - w.quote("module").append(':').quote(p.module() == null ? "ALL-UNNAMED" : p.module().getName()); - w.unindent().newline().append('}'); + w.appendKeyValue("size", p.size()); + w.newline().appendObjectEnd(); w.unindent(); } } diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java index e897095278d2..230bd1243b8b 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java @@ -25,7 +25,9 @@ package com.oracle.svm.hosted; +import static com.oracle.svm.core.jdk.Resources.NEGATIVE_QUERY_MARKER; import static com.oracle.svm.core.jdk.Resources.RESOURCES_INTERNAL_PATH_SEPARATOR; +import static com.oracle.svm.core.jdk.Resources.createStorageKey; import java.io.IOException; import java.io.InputStream; @@ -83,6 +85,7 @@ import com.oracle.svm.core.jdk.resources.NativeImageResourceFileAttributesView; import com.oracle.svm.core.jdk.resources.NativeImageResourceFileSystem; import com.oracle.svm.core.jdk.resources.NativeImageResourceFileSystemProvider; +import com.oracle.svm.core.jdk.resources.ResourceStorageEntryBase; import com.oracle.svm.core.option.HostedOptionKey; import com.oracle.svm.core.option.LocatableMultiOptionValue; import com.oracle.svm.core.option.OptionMigrationMessage; @@ -163,6 +166,7 @@ private record CompiledConditionalPattern(ConfigurationCondition condition, Reso private Set resourcePatternWorkSet = Collections.newSetFromMap(new ConcurrentHashMap<>()); private final Set excludedResourcePatterns = Collections.newSetFromMap(new ConcurrentHashMap<>()); + private final ConcurrentHashMap> registeredResources = new ConcurrentHashMap<>(); private int loadedConfigurations; private ImageClassLoader imageClassLoader; @@ -197,6 +201,8 @@ public void addResource(Module module, String resourcePath) { @Override public void injectResource(Module module, String resourcePath, byte[] resourceContent) { + // we don't have source (only module and resourcePath) + declareResourceAsRegistered(module, resourcePath, "Injected resource"); Resources.singleton().registerResource(module, resourcePath, resourceContent); } @@ -224,6 +230,16 @@ public void addResourceBundles(ConfigurationCondition condition, String basename registerConditionalConfiguration(condition, (cnd) -> ImageSingletons.lookup(LocalizationFeature.class).prepareBundle(basename, locales)); } + private void declareResourceAsRegistered(Module module, String resource, String source) { + Resources.ModuleResourceRecord key = createStorageKey(module, resource); + synchronized (registeredResources) { + registeredResources.computeIfAbsent(key, k -> new ArrayList<>()); + if (!registeredResources.get(key).contains(source)) { + registeredResources.get(key).add(source); + } + } + } + /* * It is possible that one resource can be registered under different conditions * (typeReachable). In some cases, few conditions will be satisfied, and we will try to @@ -278,6 +294,7 @@ private void processResourceFromModule(Module module, String resourcePath) { InputStream is = module.getResourceAsStream(resourcePath); registerResource(module, resourcePath, false, is); } + declareResourceAsRegistered(module, resourcePath, resourcePath); } catch (IOException e) { Resources.singleton().registerIOException(module, resourcePath, e, LinkAtBuildTimeSupport.singleton().packageOrClassAtBuildTime(resourcePath)); } @@ -315,6 +332,7 @@ private void processResourceFromClasspath(String resourcePath) { InputStream is = url.openStream(); registerResource(null, resourcePath, fromJar, is); } + declareResourceAsRegistered(null, resourcePath, url.toString()); } catch (IOException e) { Resources.singleton().registerIOException(null, resourcePath, e, LinkAtBuildTimeSupport.singleton().packageOrClassAtBuildTime(resourcePath)); return; @@ -326,6 +344,7 @@ private void processResourceFromClasspath(String resourcePath) { private void registerResource(Module module, String resourcePath, boolean fromJar, InputStream is) { if (is == null) { + Resources.singleton().registerNegativeQuery(module, resourcePath); return; } @@ -601,12 +620,19 @@ boolean moduleNameMatches(String resourceContainerModuleName) { public void afterAnalysis(AfterAnalysisAccess access) { sealed = true; if (Options.DumpRegisteredResources.getValue()) { - Iterable registeredResources = Resources.singleton().getResourceStorage().getKeys(); - List resourceInfoList = new ArrayList<>(); - registeredResources.forEach(resource -> { - String resourceName = resource.resource(); - Module module = resource.module(); - resourceInfoList.add(new Resources.ModuleResourceRecord(module, resourceName)); + List resourceInfoList = new ArrayList<>(); + registeredResources.forEach((key, value) -> { + Module module = key.module(); + String resourceName = key.resource(); + ResourceStorageEntryBase storageEntry = Resources.singleton().getResourceStorage().get(key); + List sources = new ArrayList<>(); + for (int i = 0; i < value.size(); i++) { + String source = value.get(i); + String size = storageEntry == NEGATIVE_QUERY_MARKER ? "NEGATIVE QUERY" : String.valueOf(storageEntry.getData().get(i).length); + sources.add(new ResourceReporter.SourceSizePair(source, size)); + } + + resourceInfoList.add(new ResourceReporter.ResourceReportEntry(module, resourceName, sources)); }); ResourceReporter.printReport(resourceInfoList); From ad6a5b4546d6229afa26a8ef24019c3c182c575c Mon Sep 17 00:00:00 2001 From: David Nestorovic Date: Thu, 14 Mar 2024 16:16:27 +0100 Subject: [PATCH 04/19] Redefine collecting information and add missing negative query reigistrations --- .../oracle/svm/hosted/ResourcesFeature.java | 62 +++++++++++++------ 1 file changed, 43 insertions(+), 19 deletions(-) diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java index 230bd1243b8b..f91ffd421d2c 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java @@ -63,7 +63,10 @@ import java.util.stream.Collectors; import java.util.stream.Stream; +import org.graalvm.collections.EconomicMap; import org.graalvm.nativeimage.ImageSingletons; +import org.graalvm.nativeimage.Platform; +import org.graalvm.nativeimage.Platforms; import org.graalvm.nativeimage.hosted.RuntimeResourceAccess; import org.graalvm.nativeimage.impl.ConfigurationCondition; import org.graalvm.nativeimage.impl.RuntimeResourceSupport; @@ -166,7 +169,20 @@ private record CompiledConditionalPattern(ConfigurationCondition condition, Reso private Set resourcePatternWorkSet = Collections.newSetFromMap(new ConcurrentHashMap<>()); private final Set excludedResourcePatterns = Collections.newSetFromMap(new ConcurrentHashMap<>()); - private final ConcurrentHashMap> registeredResources = new ConcurrentHashMap<>(); + + @Platforms(Platform.HOSTED_ONLY.class) private static final ConcurrentHashMap> registeredResources = new ConcurrentHashMap<>(); + + @Platforms(Platform.HOSTED_ONLY.class) + public static void declareResourceAsRegistered(Module module, String resource, String source) { + Resources.ModuleResourceRecord key = createStorageKey(module, resource); + synchronized (registeredResources) { + registeredResources.computeIfAbsent(key, k -> new ArrayList<>()); + if (!registeredResources.get(key).contains(source)) { + registeredResources.get(key).add(source); + } + } + } + private int loadedConfigurations; private ImageClassLoader imageClassLoader; @@ -202,7 +218,7 @@ public void addResource(Module module, String resourcePath) { @Override public void injectResource(Module module, String resourcePath, byte[] resourceContent) { // we don't have source (only module and resourcePath) - declareResourceAsRegistered(module, resourcePath, "Injected resource"); + declareResourceAsRegistered(module, resourcePath, "INJECTED RESOURCE"); Resources.singleton().registerResource(module, resourcePath, resourceContent); } @@ -230,16 +246,6 @@ public void addResourceBundles(ConfigurationCondition condition, String basename registerConditionalConfiguration(condition, (cnd) -> ImageSingletons.lookup(LocalizationFeature.class).prepareBundle(basename, locales)); } - private void declareResourceAsRegistered(Module module, String resource, String source) { - Resources.ModuleResourceRecord key = createStorageKey(module, resource); - synchronized (registeredResources) { - registeredResources.computeIfAbsent(key, k -> new ArrayList<>()); - if (!registeredResources.get(key).contains(source)) { - registeredResources.get(key).add(source); - } - } - } - /* * It is possible that one resource can be registered under different conditions * (typeReachable). In some cases, few conditions will be satisfied, and we will try to @@ -294,6 +300,7 @@ private void processResourceFromModule(Module module, String resourcePath) { InputStream is = module.getResourceAsStream(resourcePath); registerResource(module, resourcePath, false, is); } + // TODO maybe under if as well? declareResourceAsRegistered(module, resourcePath, resourcePath); } catch (IOException e) { Resources.singleton().registerIOException(module, resourcePath, e, LinkAtBuildTimeSupport.singleton().packageOrClassAtBuildTime(resourcePath)); @@ -332,7 +339,9 @@ private void processResourceFromClasspath(String resourcePath) { InputStream is = url.openStream(); registerResource(null, resourcePath, fromJar, is); } - declareResourceAsRegistered(null, resourcePath, url.toString()); + // TODO maybe under if as well? + String source = fromJar ? url.toString().split("!")[0] : Paths.get(url.toURI()).toString(); + declareResourceAsRegistered(null, resourcePath, source); } catch (IOException e) { Resources.singleton().registerIOException(null, resourcePath, e, LinkAtBuildTimeSupport.singleton().packageOrClassAtBuildTime(resourcePath)); return; @@ -579,6 +588,7 @@ public void registerIOException(Module module, String resourceName, IOException @Override public void registerNegativeQuery(Module module, String resourceName) { + declareResourceAsRegistered(module, resourceName, resourceName); Resources.singleton().registerNegativeQuery(module, resourceName); } } @@ -621,15 +631,29 @@ public void afterAnalysis(AfterAnalysisAccess access) { sealed = true; if (Options.DumpRegisteredResources.getValue()) { List resourceInfoList = new ArrayList<>(); - registeredResources.forEach((key, value) -> { + EconomicMap resourceStorage = Resources.singleton().getResourceStorage(); + resourceStorage.getKeys().forEach(key -> { Module module = key.module(); String resourceName = key.resource(); - ResourceStorageEntryBase storageEntry = Resources.singleton().getResourceStorage().get(key); + + ResourceStorageEntryBase storageEntry = resourceStorage.get(key); + List registeredEntrySources = registeredResources.get(key); List sources = new ArrayList<>(); - for (int i = 0; i < value.size(); i++) { - String source = value.get(i); - String size = storageEntry == NEGATIVE_QUERY_MARKER ? "NEGATIVE QUERY" : String.valueOf(storageEntry.getData().get(i).length); - sources.add(new ResourceReporter.SourceSizePair(source, size)); + + // if registeredEntrySource is null we are processing + if (registeredEntrySources == null) { + if (storageEntry != NEGATIVE_QUERY_MARKER) { + VMError.shouldNotReachHere("Resource: " + resourceName + + " from module: " + module + + " wasn't register from ResourcesFeature. It should never happen except for NEGATIVE_QUERIES in some cases"); + } + sources.add(new ResourceReporter.SourceSizePair("NO SOURCE", "NEGATIVE QUERY")); + } else { + for (int i = 0; i < registeredEntrySources.size(); i++) { + String source = registeredEntrySources.get(i); + String size = storageEntry == NEGATIVE_QUERY_MARKER ? "NEGATIVE QUERY" : String.valueOf(storageEntry.getData().get(i).length); + sources.add(new ResourceReporter.SourceSizePair(source, size)); + } } resourceInfoList.add(new ResourceReporter.ResourceReportEntry(module, resourceName, sources)); From 1eb0b4d05f75b37387e99118bc8f5855acb7d1fa Mon Sep 17 00:00:00 2001 From: David Nestorovic Date: Thu, 14 Mar 2024 16:21:44 +0100 Subject: [PATCH 05/19] Execute declareResourceAsRegister function only when DumpRegisterResource is passed --- .../oracle/svm/hosted/ResourceReporter.java | 12 +++--------- .../oracle/svm/hosted/ResourcesFeature.java | 19 +++++++++---------- 2 files changed, 12 insertions(+), 19 deletions(-) diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourceReporter.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourceReporter.java index 15a229b00446..1aa8c1b7d410 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourceReporter.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourceReporter.java @@ -15,23 +15,17 @@ public class ResourceReporter { public record SourceSizePair(String source, String size) { // NOTE: size is string because it could be NEGATIVE QUERY - public static Comparator comparator() { - return Comparator.comparing(SourceSizePair::source); - } } public record ResourceReportEntry(Module module, String resourceName, List entries) { - public static Comparator comparator() { - return Comparator.comparing(ResourceReportEntry::resourceName); - } } public static void printReport(Collection registeredResources) { Path reportLocation = NativeImageGenerator.generatedFiles(HostedOptionValues.singleton()).resolve("registered_resources.json"); try (JsonWriter writer = new JsonWriter(reportLocation)) { - JsonPrinter.printCollection(writer, registeredResources, ResourceReportEntry.comparator(), ResourceReporter::resourceReportElement); + JsonPrinter.printCollection(writer, registeredResources, Comparator.comparing(ResourceReportEntry::resourceName), ResourceReporter::resourceReportElement); } catch (IOException e) { - throw VMError.shouldNotReachHere("Json writer cannot write to: " + reportLocation + "\n Reason: " + e.getMessage()); + throw VMError.shouldNotReachHere("Json writer cannot write to: " + reportLocation, e); } } @@ -45,7 +39,7 @@ private static void resourceReportElement(ResourceReportEntry p, JsonWriter w) t w.newline(); } w.quote("entries").append(":"); - JsonPrinter.printCollection(w, p.entries(), SourceSizePair.comparator(), ResourceReporter::sourceElement); + JsonPrinter.printCollection(w, p.entries(), Comparator.comparing(SourceSizePair::source), ResourceReporter::sourceElement); w.unindent().newline().appendObjectEnd(); } diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java index f91ffd421d2c..04462ff01b09 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java @@ -174,11 +174,13 @@ private record CompiledConditionalPattern(ConfigurationCondition condition, Reso @Platforms(Platform.HOSTED_ONLY.class) public static void declareResourceAsRegistered(Module module, String resource, String source) { - Resources.ModuleResourceRecord key = createStorageKey(module, resource); - synchronized (registeredResources) { - registeredResources.computeIfAbsent(key, k -> new ArrayList<>()); - if (!registeredResources.get(key).contains(source)) { - registeredResources.get(key).add(source); + if (Options.DumpRegisteredResources.getValue()) { + Resources.ModuleResourceRecord key = createStorageKey(module, resource); + synchronized (registeredResources) { + registeredResources.computeIfAbsent(key, k -> new ArrayList<>()); + if (!registeredResources.get(key).contains(source)) { + registeredResources.get(key).add(source); + } } } } @@ -217,7 +219,6 @@ public void addResource(Module module, String resourcePath) { @Override public void injectResource(Module module, String resourcePath, byte[] resourceContent) { - // we don't have source (only module and resourcePath) declareResourceAsRegistered(module, resourcePath, "INJECTED RESOURCE"); Resources.singleton().registerResource(module, resourcePath, resourceContent); } @@ -300,7 +301,6 @@ private void processResourceFromModule(Module module, String resourcePath) { InputStream is = module.getResourceAsStream(resourcePath); registerResource(module, resourcePath, false, is); } - // TODO maybe under if as well? declareResourceAsRegistered(module, resourcePath, resourcePath); } catch (IOException e) { Resources.singleton().registerIOException(module, resourcePath, e, LinkAtBuildTimeSupport.singleton().packageOrClassAtBuildTime(resourcePath)); @@ -339,8 +339,7 @@ private void processResourceFromClasspath(String resourcePath) { InputStream is = url.openStream(); registerResource(null, resourcePath, fromJar, is); } - // TODO maybe under if as well? - String source = fromJar ? url.toString().split("!")[0] : Paths.get(url.toURI()).toString(); + String source = fromJar ? urlToJarPath(url) : Paths.get(url.toURI()).toString(); declareResourceAsRegistered(null, resourcePath, source); } catch (IOException e) { Resources.singleton().registerIOException(null, resourcePath, e, LinkAtBuildTimeSupport.singleton().packageOrClassAtBuildTime(resourcePath)); @@ -647,7 +646,7 @@ public void afterAnalysis(AfterAnalysisAccess access) { " from module: " + module + " wasn't register from ResourcesFeature. It should never happen except for NEGATIVE_QUERIES in some cases"); } - sources.add(new ResourceReporter.SourceSizePair("NO SOURCE", "NEGATIVE QUERY")); + sources.add(new ResourceReporter.SourceSizePair("-", "NEGATIVE QUERY")); } else { for (int i = 0; i < registeredEntrySources.size(); i++) { String source = registeredEntrySources.get(i); From 749ad6095a2bfe4a9e058f0c0fdf85489de9ee73 Mon Sep 17 00:00:00 2001 From: David Nestorovic Date: Tue, 19 Mar 2024 15:27:40 +0100 Subject: [PATCH 06/19] Make resource origins URI --- .../src/com/oracle/svm/hosted/ResourcesFeature.java | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java index 04462ff01b09..7cfbd1e68b45 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java @@ -50,6 +50,7 @@ import java.util.HashSet; import java.util.List; import java.util.Locale; +import java.util.Optional; import java.util.Set; import java.util.TreeSet; import java.util.concurrent.ConcurrentHashMap; @@ -301,7 +302,12 @@ private void processResourceFromModule(Module module, String resourcePath) { InputStream is = module.getResourceAsStream(resourcePath); registerResource(module, resourcePath, false, is); } - declareResourceAsRegistered(module, resourcePath, resourcePath); + + var resolvedModule = module.getLayer().configuration().findModule(module.getName()); + if (resolvedModule.isPresent()) { + Optional location = resolvedModule.get().reference().location(); + location.ifPresent(uri -> declareResourceAsRegistered(module, resourcePath, uri.toString())); + } } catch (IOException e) { Resources.singleton().registerIOException(module, resourcePath, e, LinkAtBuildTimeSupport.singleton().packageOrClassAtBuildTime(resourcePath)); } @@ -339,7 +345,7 @@ private void processResourceFromClasspath(String resourcePath) { InputStream is = url.openStream(); registerResource(null, resourcePath, fromJar, is); } - String source = fromJar ? urlToJarPath(url) : Paths.get(url.toURI()).toString(); + String source = fromJar ? url.toURI().toString() : Path.of(url.getPath()).toUri().toString(); declareResourceAsRegistered(null, resourcePath, source); } catch (IOException e) { Resources.singleton().registerIOException(null, resourcePath, e, LinkAtBuildTimeSupport.singleton().packageOrClassAtBuildTime(resourcePath)); @@ -587,7 +593,7 @@ public void registerIOException(Module module, String resourceName, IOException @Override public void registerNegativeQuery(Module module, String resourceName) { - declareResourceAsRegistered(module, resourceName, resourceName); + declareResourceAsRegistered(module, resourceName, "-"); Resources.singleton().registerNegativeQuery(module, resourceName); } } From cfb3f84e55a507d3a1f0e79c2f0db033199e0e26 Mon Sep 17 00:00:00 2001 From: David Nestorovic Date: Fri, 22 Mar 2024 11:24:10 +0100 Subject: [PATCH 07/19] Collect information whether resource is directory or not --- .../src/com/oracle/svm/hosted/ResourceReporter.java | 6 +++++- .../src/com/oracle/svm/hosted/ResourcesFeature.java | 9 +++++---- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourceReporter.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourceReporter.java index 1aa8c1b7d410..933e3d3230f3 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourceReporter.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourceReporter.java @@ -17,7 +17,7 @@ public record SourceSizePair(String source, String size) { // NOTE: size is string because it could be NEGATIVE QUERY } - public record ResourceReportEntry(Module module, String resourceName, List entries) { + public record ResourceReportEntry(Module module, String resourceName, List entries, Boolean isDirectory) { } public static void printReport(Collection registeredResources) { @@ -38,6 +38,10 @@ private static void resourceReportElement(ResourceReportEntry p, JsonWriter w) t w.appendKeyValue("module", p.module().getName()).appendSeparator(); w.newline(); } + if (p.isDirectory() != null) { + w.appendKeyValue("directory", p.isDirectory()).appendSeparator(); + w.newline(); + } w.quote("entries").append(":"); JsonPrinter.printCollection(w, p.entries(), Comparator.comparing(SourceSizePair::source), ResourceReporter::sourceElement); w.unindent().newline().appendObjectEnd(); diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java index 7cfbd1e68b45..4cb15b03f53f 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java @@ -220,7 +220,7 @@ public void addResource(Module module, String resourcePath) { @Override public void injectResource(Module module, String resourcePath, byte[] resourceContent) { - declareResourceAsRegistered(module, resourcePath, "INJECTED RESOURCE"); + declareResourceAsRegistered(module, resourcePath, "INJECTED"); Resources.singleton().registerResource(module, resourcePath, resourceContent); } @@ -593,7 +593,7 @@ public void registerIOException(Module module, String resourceName, IOException @Override public void registerNegativeQuery(Module module, String resourceName) { - declareResourceAsRegistered(module, resourceName, "-"); + declareResourceAsRegistered(module, resourceName, "n/a"); Resources.singleton().registerNegativeQuery(module, resourceName); } } @@ -652,7 +652,7 @@ public void afterAnalysis(AfterAnalysisAccess access) { " from module: " + module + " wasn't register from ResourcesFeature. It should never happen except for NEGATIVE_QUERIES in some cases"); } - sources.add(new ResourceReporter.SourceSizePair("-", "NEGATIVE QUERY")); + sources.add(new ResourceReporter.SourceSizePair("n/a", "NEGATIVE QUERY")); } else { for (int i = 0; i < registeredEntrySources.size(); i++) { String source = registeredEntrySources.get(i); @@ -661,7 +661,8 @@ public void afterAnalysis(AfterAnalysisAccess access) { } } - resourceInfoList.add(new ResourceReporter.ResourceReportEntry(module, resourceName, sources)); + Boolean isDirectory = storageEntry == NEGATIVE_QUERY_MARKER ? null : storageEntry.isDirectory(); + resourceInfoList.add(new ResourceReporter.ResourceReportEntry(module, resourceName, sources, isDirectory)); }); ResourceReporter.printReport(resourceInfoList); From 18318190faa651c8c8dc7c365e21edbb4db6d072 Mon Sep 17 00:00:00 2001 From: David Nestorovic Date: Mon, 25 Mar 2024 13:52:25 +0100 Subject: [PATCH 08/19] Extract resources info collecting into EmbeddedResourceExporter --- .../svm/hosted/EmbeddedResourceExporter.java | 107 ++++++++++++++++++ .../oracle/svm/hosted/ResourceReporter.java | 59 ---------- .../oracle/svm/hosted/ResourcesFeature.java | 54 +++------ 3 files changed, 123 insertions(+), 97 deletions(-) create mode 100644 substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/EmbeddedResourceExporter.java delete mode 100644 substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourceReporter.java diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/EmbeddedResourceExporter.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/EmbeddedResourceExporter.java new file mode 100644 index 000000000000..cc8842947b7f --- /dev/null +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/EmbeddedResourceExporter.java @@ -0,0 +1,107 @@ +package com.oracle.svm.hosted; + +import static com.oracle.svm.core.jdk.Resources.NEGATIVE_QUERY_MARKER; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; +import java.util.concurrent.ConcurrentHashMap; + +import org.graalvm.collections.EconomicMap; +import org.graalvm.nativeimage.ImageSingletons; +import org.graalvm.nativeimage.Platform; +import org.graalvm.nativeimage.Platforms; + +import com.oracle.svm.core.jdk.Resources; +import com.oracle.svm.core.jdk.resources.ResourceStorageEntryBase; +import com.oracle.svm.core.util.VMError; +import com.oracle.svm.core.util.json.JsonPrinter; +import com.oracle.svm.core.util.json.JsonWriter; + +@Platforms(Platform.HOSTED_ONLY.class) +public class EmbeddedResourceExporter { + + public record SourceSizePair(String source, String size) { + // NOTE: size is string because it could be NEGATIVE QUERY + } + + public record ResourceReportEntry(Module module, String resourceName, List entries, Boolean isDirectory) { + } + + private final List resources; + + public static EmbeddedResourceExporter singleton() { + return ImageSingletons.lookup(EmbeddedResourceExporter.class); + } + + public EmbeddedResourceExporter(ConcurrentHashMap> registeredResources) { + this.resources = getResourceReportEntryList(registeredResources); + } + + public void printReport(JsonWriter writer) throws IOException { + JsonPrinter.printCollection(writer, this.resources, Comparator.comparing(EmbeddedResourceExporter.ResourceReportEntry::resourceName), EmbeddedResourceExporter::resourceReportElement); + } + + private static void resourceReportElement(ResourceReportEntry p, JsonWriter w) throws IOException { + w.indent().newline(); + w.appendObjectStart().newline(); + w.appendKeyValue("name", p.resourceName()).appendSeparator(); + w.newline(); + if (p.module() != null) { + w.appendKeyValue("module", p.module().getName()).appendSeparator(); + w.newline(); + } + if (p.isDirectory() != null) { + w.appendKeyValue("directory", p.isDirectory()).appendSeparator(); + w.newline(); + } + w.quote("entries").append(":"); + JsonPrinter.printCollection(w, p.entries(), Comparator.comparing(SourceSizePair::source), EmbeddedResourceExporter::sourceElement); + w.unindent().newline().appendObjectEnd(); + } + + private static void sourceElement(SourceSizePair p, JsonWriter w) throws IOException { + w.indent().newline(); + w.appendObjectStart().newline(); + w.appendKeyValue("origin", p.source()).appendSeparator(); + w.newline(); + w.appendKeyValue("size", p.size()); + w.newline().appendObjectEnd(); + w.unindent(); + } + + private List getResourceReportEntryList(ConcurrentHashMap> collection) { + List resourceInfoList = new ArrayList<>(); + EconomicMap resourceStorage = Resources.singleton().getResourceStorage(); + resourceStorage.getKeys().forEach(key -> { + Module module = key.module(); + String resourceName = key.resource(); + + ResourceStorageEntryBase storageEntry = resourceStorage.get(key); + List registeredEntrySources = collection.get(key); + List sources = new ArrayList<>(); + + // if registeredEntrySource is null we are processing + if (registeredEntrySources == null) { + if (storageEntry != NEGATIVE_QUERY_MARKER) { + VMError.shouldNotReachHere("Resource: " + resourceName + + " from module: " + module + + " wasn't register from ResourcesFeature. It should never happen except for NEGATIVE_QUERIES in some cases"); + } + sources.add(new EmbeddedResourceExporter.SourceSizePair("n/a", "NEGATIVE QUERY")); + } else { + for (int i = 0; i < registeredEntrySources.size(); i++) { + String source = registeredEntrySources.get(i); + String size = storageEntry == NEGATIVE_QUERY_MARKER ? "NEGATIVE QUERY" : String.valueOf(storageEntry.getData().get(i).length); + sources.add(new EmbeddedResourceExporter.SourceSizePair(source, size)); + } + } + + Boolean isDirectory = storageEntry == NEGATIVE_QUERY_MARKER ? null : storageEntry.isDirectory(); + resourceInfoList.add(new EmbeddedResourceExporter.ResourceReportEntry(module, resourceName, sources, isDirectory)); + }); + + return resourceInfoList; + } +} diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourceReporter.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourceReporter.java deleted file mode 100644 index 933e3d3230f3..000000000000 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourceReporter.java +++ /dev/null @@ -1,59 +0,0 @@ -package com.oracle.svm.hosted; - -import java.io.IOException; -import java.nio.file.Path; -import java.util.Collection; -import java.util.Comparator; -import java.util.List; - -import com.oracle.svm.core.option.HostedOptionValues; -import com.oracle.svm.core.util.VMError; -import com.oracle.svm.core.util.json.JsonPrinter; -import com.oracle.svm.core.util.json.JsonWriter; - -public class ResourceReporter { - - public record SourceSizePair(String source, String size) { - // NOTE: size is string because it could be NEGATIVE QUERY - } - - public record ResourceReportEntry(Module module, String resourceName, List entries, Boolean isDirectory) { - } - - public static void printReport(Collection registeredResources) { - Path reportLocation = NativeImageGenerator.generatedFiles(HostedOptionValues.singleton()).resolve("registered_resources.json"); - try (JsonWriter writer = new JsonWriter(reportLocation)) { - JsonPrinter.printCollection(writer, registeredResources, Comparator.comparing(ResourceReportEntry::resourceName), ResourceReporter::resourceReportElement); - } catch (IOException e) { - throw VMError.shouldNotReachHere("Json writer cannot write to: " + reportLocation, e); - } - } - - private static void resourceReportElement(ResourceReportEntry p, JsonWriter w) throws IOException { - w.indent().newline(); - w.appendObjectStart().newline(); - w.appendKeyValue("name", p.resourceName()).appendSeparator(); - w.newline(); - if (p.module() != null) { - w.appendKeyValue("module", p.module().getName()).appendSeparator(); - w.newline(); - } - if (p.isDirectory() != null) { - w.appendKeyValue("directory", p.isDirectory()).appendSeparator(); - w.newline(); - } - w.quote("entries").append(":"); - JsonPrinter.printCollection(w, p.entries(), Comparator.comparing(SourceSizePair::source), ResourceReporter::sourceElement); - w.unindent().newline().appendObjectEnd(); - } - - private static void sourceElement(SourceSizePair p, JsonWriter w) throws IOException { - w.indent().newline(); - w.appendObjectStart().newline(); - w.appendKeyValue("origin", p.source()).appendSeparator(); - w.newline(); - w.appendKeyValue("size", p.size()); - w.newline().appendObjectEnd(); - w.unindent(); - } -} diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java index 4cb15b03f53f..c5e23d1b7d74 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java @@ -25,7 +25,6 @@ package com.oracle.svm.hosted; -import static com.oracle.svm.core.jdk.Resources.NEGATIVE_QUERY_MARKER; import static com.oracle.svm.core.jdk.Resources.RESOURCES_INTERNAL_PATH_SEPARATOR; import static com.oracle.svm.core.jdk.Resources.createStorageKey; @@ -64,7 +63,6 @@ import java.util.stream.Collectors; import java.util.stream.Stream; -import org.graalvm.collections.EconomicMap; import org.graalvm.nativeimage.ImageSingletons; import org.graalvm.nativeimage.Platform; import org.graalvm.nativeimage.Platforms; @@ -89,12 +87,13 @@ import com.oracle.svm.core.jdk.resources.NativeImageResourceFileAttributesView; import com.oracle.svm.core.jdk.resources.NativeImageResourceFileSystem; import com.oracle.svm.core.jdk.resources.NativeImageResourceFileSystemProvider; -import com.oracle.svm.core.jdk.resources.ResourceStorageEntryBase; import com.oracle.svm.core.option.HostedOptionKey; +import com.oracle.svm.core.option.HostedOptionValues; import com.oracle.svm.core.option.LocatableMultiOptionValue; import com.oracle.svm.core.option.OptionMigrationMessage; import com.oracle.svm.core.util.UserError; import com.oracle.svm.core.util.VMError; +import com.oracle.svm.core.util.json.JsonWriter; import com.oracle.svm.hosted.classinitialization.ClassInitializationSupport; import com.oracle.svm.hosted.config.ConfigurationParserUtils; import com.oracle.svm.hosted.jdk.localization.LocalizationFeature; @@ -155,9 +154,10 @@ public static class Options { @Option(help = "Regexp to match names of resources to be excluded from the image.", type = OptionType.User)// public static final HostedOptionKey ExcludeResources = new HostedOptionKey<>(LocatableMultiOptionValue.Strings.build()); - @Option(help = "Create a Native Image registered resources json", type = OptionType.Debug) // - public static final HostedOptionKey DumpRegisteredResources = new HostedOptionKey<>(false); - + public static final String EMBEDDED_RESOURCES_FILE_NAME = "embedded-resources.json"; + @Option(help = "Create a " + EMBEDDED_RESOURCES_FILE_NAME + " file in the build directory. The output conforms to the JSON schema located at: " + + "docs/reference-manual/native-image/assets/embedded-resources-schema-v0.1.0.json", type = OptionType.User)// + public static final HostedOptionKey GenerateEmbeddedResourcesFile = new HostedOptionKey<>(false); } private boolean sealed = false; @@ -175,7 +175,7 @@ private record CompiledConditionalPattern(ConfigurationCondition condition, Reso @Platforms(Platform.HOSTED_ONLY.class) public static void declareResourceAsRegistered(Module module, String resource, String source) { - if (Options.DumpRegisteredResources.getValue()) { + if (Options.GenerateEmbeddedResourcesFile.getValue()) { Resources.ModuleResourceRecord key = createStorageKey(module, resource); synchronized (registeredResources) { registeredResources.computeIfAbsent(key, k -> new ArrayList<>()); @@ -634,38 +634,16 @@ boolean moduleNameMatches(String resourceContainerModuleName) { @Override public void afterAnalysis(AfterAnalysisAccess access) { sealed = true; - if (Options.DumpRegisteredResources.getValue()) { - List resourceInfoList = new ArrayList<>(); - EconomicMap resourceStorage = Resources.singleton().getResourceStorage(); - resourceStorage.getKeys().forEach(key -> { - Module module = key.module(); - String resourceName = key.resource(); - - ResourceStorageEntryBase storageEntry = resourceStorage.get(key); - List registeredEntrySources = registeredResources.get(key); - List sources = new ArrayList<>(); - - // if registeredEntrySource is null we are processing - if (registeredEntrySources == null) { - if (storageEntry != NEGATIVE_QUERY_MARKER) { - VMError.shouldNotReachHere("Resource: " + resourceName + - " from module: " + module + - " wasn't register from ResourcesFeature. It should never happen except for NEGATIVE_QUERIES in some cases"); - } - sources.add(new ResourceReporter.SourceSizePair("n/a", "NEGATIVE QUERY")); - } else { - for (int i = 0; i < registeredEntrySources.size(); i++) { - String source = registeredEntrySources.get(i); - String size = storageEntry == NEGATIVE_QUERY_MARKER ? "NEGATIVE QUERY" : String.valueOf(storageEntry.getData().get(i).length); - sources.add(new ResourceReporter.SourceSizePair(source, size)); - } - } + if (Options.GenerateEmbeddedResourcesFile.getValue()) { + Path reportLocation = NativeImageGenerator.generatedFiles(HostedOptionValues.singleton()).resolve("embedded-resources.json"); + EmbeddedResourceExporter resourceExporter = new EmbeddedResourceExporter(registeredResources); + ImageSingletons.add(EmbeddedResourceExporter.class, resourceExporter); - Boolean isDirectory = storageEntry == NEGATIVE_QUERY_MARKER ? null : storageEntry.isDirectory(); - resourceInfoList.add(new ResourceReporter.ResourceReportEntry(module, resourceName, sources, isDirectory)); - }); - - ResourceReporter.printReport(resourceInfoList); + try (JsonWriter writer = new JsonWriter(reportLocation)) { + EmbeddedResourceExporter.singleton().printReport(writer); + } catch (IOException e) { + throw VMError.shouldNotReachHere("Json writer cannot write to: " + reportLocation, e); + } } } From ca7d1738aec8d8c202f8abc7ac6a72d3a6b9d9c4 Mon Sep 17 00:00:00 2001 From: David Nestorovic Date: Tue, 26 Mar 2024 16:45:17 +0100 Subject: [PATCH 09/19] Make registered resources list always available and extract resource util functions --- .../com/oracle/svm/core/jdk/Resources.java | 7 +- .../svm/hosted/EmbeddedResourceExporter.java | 71 ++++++----- .../oracle/svm/hosted/ResourcesFeature.java | 110 ++--------------- .../svm/hosted/util/ResourcesUtils.java | 113 ++++++++++++++++++ 4 files changed, 163 insertions(+), 138 deletions(-) create mode 100644 substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/util/ResourcesUtils.java diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/Resources.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/Resources.java index 2c15a457fb8b..5addb8524de2 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/Resources.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/Resources.java @@ -43,6 +43,7 @@ import org.graalvm.collections.EconomicMap; import org.graalvm.collections.MapCursor; +import org.graalvm.nativeimage.ImageInfo; import org.graalvm.nativeimage.ImageSingletons; import org.graalvm.nativeimage.Platform; import org.graalvm.nativeimage.Platforms; @@ -142,6 +143,11 @@ public static String moduleName(Module module) { public static ModuleResourceRecord createStorageKey(Module module, String resourceName) { Module m = module != null && module.isNamed() ? module : null; + if (ImageInfo.inImageBuildtimeCode()) { + if (m != null) { + m = RuntimeModuleSupport.instance().getRuntimeModuleForHostedModule(m); + } + } return new ModuleResourceRecord(m, resourceName); } @@ -171,7 +177,6 @@ private void updateTimeStamp() { private void addEntry(Module module, String resourceName, boolean isDirectory, byte[] data, boolean fromJar, boolean isNegativeQuery) { VMError.guarantee(!BuildPhaseProvider.isAnalysisFinished(), "Trying to add a resource entry after analysis."); Module m = module != null && module.isNamed() ? module : null; - synchronized (resources) { ModuleResourceRecord key = createStorageKey(m, resourceName); ResourceStorageEntryBase entry = resources.get(key); diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/EmbeddedResourceExporter.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/EmbeddedResourceExporter.java index cc8842947b7f..1d0de1f405b3 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/EmbeddedResourceExporter.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/EmbeddedResourceExporter.java @@ -9,7 +9,6 @@ import java.util.concurrent.ConcurrentHashMap; import org.graalvm.collections.EconomicMap; -import org.graalvm.nativeimage.ImageSingletons; import org.graalvm.nativeimage.Platform; import org.graalvm.nativeimage.Platforms; @@ -22,25 +21,17 @@ @Platforms(Platform.HOSTED_ONLY.class) public class EmbeddedResourceExporter { - public record SourceSizePair(String source, String size) { - // NOTE: size is string because it could be NEGATIVE QUERY + public record SourceSizePair(String source, int size) { } - public record ResourceReportEntry(Module module, String resourceName, List entries, Boolean isDirectory) { + public record ResourceReportEntry(Module module, String resourceName, List entries, boolean isDirectory, boolean isMissing) { } - private final List resources; - - public static EmbeddedResourceExporter singleton() { - return ImageSingletons.lookup(EmbeddedResourceExporter.class); - } - - public EmbeddedResourceExporter(ConcurrentHashMap> registeredResources) { - this.resources = getResourceReportEntryList(registeredResources); - } - - public void printReport(JsonWriter writer) throws IOException { - JsonPrinter.printCollection(writer, this.resources, Comparator.comparing(EmbeddedResourceExporter.ResourceReportEntry::resourceName), EmbeddedResourceExporter::resourceReportElement); + public static void printReport(JsonWriter writer) throws IOException { + JsonPrinter.printCollection(writer, + getResourceReportEntryList(EmbeddedResourcesInfo.getRegisteredResources()), + Comparator.comparing(EmbeddedResourceExporter.ResourceReportEntry::resourceName), + EmbeddedResourceExporter::resourceReportElement); } private static void resourceReportElement(ResourceReportEntry p, JsonWriter w) throws IOException { @@ -52,10 +43,17 @@ private static void resourceReportElement(ResourceReportEntry p, JsonWriter w) t w.appendKeyValue("module", p.module().getName()).appendSeparator(); w.newline(); } - if (p.isDirectory() != null) { - w.appendKeyValue("directory", p.isDirectory()).appendSeparator(); + + if (p.isDirectory()) { + w.appendKeyValue("is_directory", true).appendSeparator(); + w.newline(); + } + + if (p.isMissing()) { + w.appendKeyValue("is_missing", true).appendSeparator(); w.newline(); } + w.quote("entries").append(":"); JsonPrinter.printCollection(w, p.entries(), Comparator.comparing(SourceSizePair::source), EmbeddedResourceExporter::sourceElement); w.unindent().newline().appendObjectEnd(); @@ -71,7 +69,7 @@ private static void sourceElement(SourceSizePair p, JsonWriter w) throws IOExcep w.unindent(); } - private List getResourceReportEntryList(ConcurrentHashMap> collection) { + private static List getResourceReportEntryList(ConcurrentHashMap> collection) { List resourceInfoList = new ArrayList<>(); EconomicMap resourceStorage = Resources.singleton().getResourceStorage(); resourceStorage.getKeys().forEach(key -> { @@ -80,26 +78,27 @@ private List getResourceReportEntryList(ConcurrentHashMap registeredEntrySources = collection.get(key); - List sources = new ArrayList<>(); - // if registeredEntrySource is null we are processing - if (registeredEntrySources == null) { - if (storageEntry != NEGATIVE_QUERY_MARKER) { - VMError.shouldNotReachHere("Resource: " + resourceName + - " from module: " + module + - " wasn't register from ResourcesFeature. It should never happen except for NEGATIVE_QUERIES in some cases"); - } - sources.add(new EmbeddedResourceExporter.SourceSizePair("n/a", "NEGATIVE QUERY")); - } else { - for (int i = 0; i < registeredEntrySources.size(); i++) { - String source = registeredEntrySources.get(i); - String size = storageEntry == NEGATIVE_QUERY_MARKER ? "NEGATIVE QUERY" : String.valueOf(storageEntry.getData().get(i).length); - sources.add(new EmbeddedResourceExporter.SourceSizePair(source, size)); - } + if (registeredEntrySources == null && storageEntry != NEGATIVE_QUERY_MARKER) { + throw VMError.shouldNotReachHere("Resource: " + resourceName + + " from module: " + module + + " wasn't register from ResourcesFeature. It should never happen except for NEGATIVE_QUERIES in some cases"); + } + + if (storageEntry == NEGATIVE_QUERY_MARKER) { + resourceInfoList.add(new ResourceReportEntry(module, resourceName, new ArrayList<>(), false, true)); + return; + } + + List sources = new ArrayList<>(); + for (int i = 0; i < registeredEntrySources.size(); i++) { + String source = registeredEntrySources.get(i); + int size = storageEntry.getData().get(i).length; + sources.add(new SourceSizePair(source, size)); } - Boolean isDirectory = storageEntry == NEGATIVE_QUERY_MARKER ? null : storageEntry.isDirectory(); - resourceInfoList.add(new EmbeddedResourceExporter.ResourceReportEntry(module, resourceName, sources, isDirectory)); + boolean isDirectory = storageEntry.isDirectory(); + resourceInfoList.add(new ResourceReportEntry(module, resourceName, sources, isDirectory, false)); }); return resourceInfoList; diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java index c5e23d1b7d74..01f165f99daf 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java @@ -26,18 +26,16 @@ package com.oracle.svm.hosted; import static com.oracle.svm.core.jdk.Resources.RESOURCES_INTERNAL_PATH_SEPARATOR; -import static com.oracle.svm.core.jdk.Resources.createStorageKey; +import static com.oracle.svm.hosted.EmbeddedResourcesInfo.declareResourceAsRegistered; import java.io.IOException; import java.io.InputStream; import java.lang.reflect.Method; import java.lang.reflect.Modifier; -import java.net.JarURLConnection; import java.net.URI; import java.net.URISyntaxException; import java.net.URL; import java.nio.file.FileSystem; -import java.nio.file.FileSystems; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; @@ -51,21 +49,15 @@ import java.util.Locale; import java.util.Optional; import java.util.Set; -import java.util.TreeSet; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.LongAdder; -import java.util.jar.JarEntry; -import java.util.jar.JarFile; import java.util.regex.Pattern; import java.util.stream.Collectors; -import java.util.stream.Stream; import org.graalvm.nativeimage.ImageSingletons; -import org.graalvm.nativeimage.Platform; -import org.graalvm.nativeimage.Platforms; import org.graalvm.nativeimage.hosted.RuntimeResourceAccess; import org.graalvm.nativeimage.impl.ConfigurationCondition; import org.graalvm.nativeimage.impl.RuntimeResourceSupport; @@ -99,6 +91,7 @@ import com.oracle.svm.hosted.jdk.localization.LocalizationFeature; import com.oracle.svm.hosted.reflect.NativeImageConditionResolver; import com.oracle.svm.hosted.snippets.SubstrateGraphBuilderPlugins; +import com.oracle.svm.hosted.util.ResourcesUtils; import com.oracle.svm.util.LogUtils; import com.oracle.svm.util.ModuleSupport; import com.oracle.svm.util.ReflectionUtil; @@ -171,21 +164,6 @@ private record CompiledConditionalPattern(ConfigurationCondition condition, Reso private Set resourcePatternWorkSet = Collections.newSetFromMap(new ConcurrentHashMap<>()); private final Set excludedResourcePatterns = Collections.newSetFromMap(new ConcurrentHashMap<>()); - @Platforms(Platform.HOSTED_ONLY.class) private static final ConcurrentHashMap> registeredResources = new ConcurrentHashMap<>(); - - @Platforms(Platform.HOSTED_ONLY.class) - public static void declareResourceAsRegistered(Module module, String resource, String source) { - if (Options.GenerateEmbeddedResourcesFile.getValue()) { - Resources.ModuleResourceRecord key = createStorageKey(module, resource); - synchronized (registeredResources) { - registeredResources.computeIfAbsent(key, k -> new ArrayList<>()); - if (!registeredResources.get(key).contains(source)) { - registeredResources.get(key).add(source); - } - } - } - } - private int loadedConfigurations; private ImageClassLoader imageClassLoader; @@ -296,7 +274,7 @@ private void processResourceFromModule(Module module, String resourcePath) { boolean isDirectory = Files.isDirectory(Path.of(resourcePath)); if (isDirectory) { - String content = getDirectoryContent(resourcePath, false); + String content = ResourcesUtils.getDirectoryContent(resourcePath, false); Resources.singleton().registerDirectoryResource(module, resourcePath, content, false); } else { InputStream is = module.getResourceAsStream(resourcePath); @@ -337,15 +315,16 @@ private void processResourceFromClasspath(String resourcePath) { alreadyProcessedResources.add(url.toString()); try { boolean fromJar = url.getProtocol().equalsIgnoreCase("jar"); - boolean isDirectory = resourceIsDirectory(url, fromJar, resourcePath); + boolean isDirectory = ResourcesUtils.resourceIsDirectory(url, fromJar, resourcePath); if (isDirectory) { - String content = getDirectoryContent(fromJar ? url.toString() : Paths.get(url.toURI()).toString(), fromJar); + String content = ResourcesUtils.getDirectoryContent(fromJar ? url.toString() : Paths.get(url.toURI()).toString(), fromJar); Resources.singleton().registerDirectoryResource(null, resourcePath, content, fromJar); } else { InputStream is = url.openStream(); registerResource(null, resourcePath, fromJar, is); } - String source = fromJar ? url.toURI().toString() : Path.of(url.getPath()).toUri().toString(); + + String source = ResourcesUtils.getResourceSource(url, resourcePath, fromJar); declareResourceAsRegistered(null, resourcePath, source); } catch (IOException e) { Resources.singleton().registerIOException(null, resourcePath, e, LinkAtBuildTimeSupport.singleton().packageOrClassAtBuildTime(resourcePath)); @@ -370,74 +349,6 @@ private void registerResource(Module module, String resourcePath, boolean fromJa throw new RuntimeException(e); } } - - /* Util functions for resource attributes calculations */ - private String urlToJarPath(URL url) { - try { - return ((JarURLConnection) url.openConnection()).getJarFileURL().toURI().getPath(); - } catch (IOException | URISyntaxException e) { - throw new RuntimeException(e); - } - } - - private boolean resourceIsDirectory(URL url, boolean fromJar, String resourcePath) throws IOException, URISyntaxException { - if (fromJar) { - try (JarFile jf = new JarFile(urlToJarPath(url))) { - return jf.getEntry(resourcePath).isDirectory(); - } - } else { - return Files.isDirectory(Path.of(url.toURI())); - } - } - - private String getDirectoryContent(String path, boolean fromJar) throws IOException { - Set content = new TreeSet<>(); - if (fromJar) { - try (JarFile jf = new JarFile(urlToJarPath(URI.create(path).toURL()))) { - String pathSeparator = FileSystems.getDefault().getSeparator(); - String directoryPath = path.split("!")[1]; - - // we are removing leading slash because jar entry names don't start with slash - if (directoryPath.startsWith(pathSeparator)) { - directoryPath = directoryPath.substring(1); - } - - Enumeration entries = jf.entries(); - while (entries.hasMoreElements()) { - String entry = entries.nextElement().getName(); - if (entry.startsWith(directoryPath)) { - String contentEntry = entry.substring(directoryPath.length()); - - // remove the leading slash - if (contentEntry.startsWith(pathSeparator)) { - contentEntry = contentEntry.substring(1); - } - - // prevent adding empty strings as a content - if (!contentEntry.isEmpty()) { - // get top level content only - int firstSlash = contentEntry.indexOf(pathSeparator); - if (firstSlash != -1) { - content.add(contentEntry.substring(0, firstSlash)); - } else { - content.add(contentEntry); - } - } - } - } - - } - } else { - try (Stream contentStream = Files.list(Path.of(path))) { - content = new TreeSet<>(contentStream - .map(Path::getFileName) - .map(Path::toString) - .toList()); - } - } - - return String.join(System.lineSeparator(), content); - } } @Override @@ -593,7 +504,7 @@ public void registerIOException(Module module, String resourceName, IOException @Override public void registerNegativeQuery(Module module, String resourceName) { - declareResourceAsRegistered(module, resourceName, "n/a"); + declareResourceAsRegistered(module, resourceName, ""); Resources.singleton().registerNegativeQuery(module, resourceName); } } @@ -636,11 +547,8 @@ public void afterAnalysis(AfterAnalysisAccess access) { sealed = true; if (Options.GenerateEmbeddedResourcesFile.getValue()) { Path reportLocation = NativeImageGenerator.generatedFiles(HostedOptionValues.singleton()).resolve("embedded-resources.json"); - EmbeddedResourceExporter resourceExporter = new EmbeddedResourceExporter(registeredResources); - ImageSingletons.add(EmbeddedResourceExporter.class, resourceExporter); - try (JsonWriter writer = new JsonWriter(reportLocation)) { - EmbeddedResourceExporter.singleton().printReport(writer); + EmbeddedResourceExporter.printReport(writer); } catch (IOException e) { throw VMError.shouldNotReachHere("Json writer cannot write to: " + reportLocation, e); } diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/util/ResourcesUtils.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/util/ResourcesUtils.java new file mode 100644 index 000000000000..7b203567b344 --- /dev/null +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/util/ResourcesUtils.java @@ -0,0 +1,113 @@ +package com.oracle.svm.hosted.util; + +import java.io.IOException; +import java.net.JarURLConnection; +import java.net.URI; +import java.net.URISyntaxException; +import java.net.URL; +import java.nio.file.FileSystems; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Enumeration; +import java.util.Set; +import java.util.TreeSet; +import java.util.jar.JarEntry; +import java.util.jar.JarFile; +import java.util.stream.Stream; + +public class ResourcesUtils { + + /** + * Returns jar path from the given url + */ + private static String urlToJarPath(URL url) { + try { + return ((JarURLConnection) url.openConnection()).getJarFileURL().toURI().getPath(); + } catch (IOException | URISyntaxException e) { + throw new RuntimeException(e); + } + } + + /** + * Returns directory that contains resource on the given url + */ + public static String getResourceSource(URL url, String resource, boolean fromJar) { + String source = fromJar ? Path.of(urlToJarPath(url)).toUri().toString() : Path.of(url.getPath()).toUri().toString(); + if (!fromJar) { + // -1 removes trailing slash from path of directory that contains resource + source = source.substring(0, source.length() - resource.length() - 1); + if (source.endsWith("/")) { + // if resource was directory we still have one slash at the end + source = source.substring(0, source.length() - 1); + } + } + + return source; + } + + /** + * Returns whether the given resource is directory or not + */ + public static boolean resourceIsDirectory(URL url, boolean fromJar, String resource) throws IOException, URISyntaxException { + if (fromJar) { + try (JarFile jf = new JarFile(urlToJarPath(url))) { + return jf.getEntry(resource).isDirectory(); + } + } else { + return Files.isDirectory(Path.of(url.toURI())); + } + } + + /** + * Returns directory content of the resource from the given path + */ + public static String getDirectoryContent(String path, boolean fromJar) throws IOException { + Set content = new TreeSet<>(); + if (fromJar) { + try (JarFile jf = new JarFile(urlToJarPath(URI.create(path).toURL()))) { + String pathSeparator = FileSystems.getDefault().getSeparator(); + String directoryPath = path.split("!")[1]; + + // we are removing leading slash because jar entry names don't start with slash + if (directoryPath.startsWith(pathSeparator)) { + directoryPath = directoryPath.substring(1); + } + + Enumeration entries = jf.entries(); + while (entries.hasMoreElements()) { + String entry = entries.nextElement().getName(); + if (entry.startsWith(directoryPath)) { + String contentEntry = entry.substring(directoryPath.length()); + + // remove the leading slash + if (contentEntry.startsWith(pathSeparator)) { + contentEntry = contentEntry.substring(1); + } + + // prevent adding empty strings as a content + if (!contentEntry.isEmpty()) { + // get top level content only + int firstSlash = contentEntry.indexOf(pathSeparator); + if (firstSlash != -1) { + content.add(contentEntry.substring(0, firstSlash)); + } else { + content.add(contentEntry); + } + } + } + } + + } + } else { + try (Stream contentStream = Files.list(Path.of(path))) { + content = new TreeSet<>(contentStream + .map(Path::getFileName) + .map(Path::toString) + .toList()); + } + } + + return String.join(System.lineSeparator(), content); + } + +} From 78ee909ad5222994c9ff60a5620847ab19a52d58 Mon Sep 17 00:00:00 2001 From: David Nestorovic Date: Tue, 26 Mar 2024 17:21:42 +0100 Subject: [PATCH 10/19] Make EmbeddedResourcesInfo singleton --- .../svm/hosted/EmbeddedResourceExporter.java | 2 +- .../svm/hosted/EmbeddedResourcesInfo.java | 38 +++++++++++++++++++ .../oracle/svm/hosted/ResourcesFeature.java | 11 +++--- 3 files changed, 45 insertions(+), 6 deletions(-) create mode 100644 substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/EmbeddedResourcesInfo.java diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/EmbeddedResourceExporter.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/EmbeddedResourceExporter.java index 1d0de1f405b3..e227d5958a76 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/EmbeddedResourceExporter.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/EmbeddedResourceExporter.java @@ -29,7 +29,7 @@ public record ResourceReportEntry(Module module, String resourceName, List> registeredResources = new ConcurrentHashMap<>(); + + public ConcurrentHashMap> getRegisteredResources() { + return registeredResources; + } + + public static EmbeddedResourcesInfo singleton() { + return ImageSingletons.lookup(EmbeddedResourcesInfo.class); + } + + public void declareResourceAsRegistered(Module module, String resource, String source) { + Resources.ModuleResourceRecord key = createStorageKey(module, resource); + synchronized (registeredResources) { + registeredResources.computeIfAbsent(key, k -> new ArrayList<>()); + if (!registeredResources.get(key).contains(source)) { + registeredResources.get(key).add(source); + } + } + } + +} diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java index 01f165f99daf..ce475c4f715b 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java @@ -26,7 +26,6 @@ package com.oracle.svm.hosted; import static com.oracle.svm.core.jdk.Resources.RESOURCES_INTERNAL_PATH_SEPARATOR; -import static com.oracle.svm.hosted.EmbeddedResourcesInfo.declareResourceAsRegistered; import java.io.IOException; import java.io.InputStream; @@ -198,7 +197,7 @@ public void addResource(Module module, String resourcePath) { @Override public void injectResource(Module module, String resourcePath, byte[] resourceContent) { - declareResourceAsRegistered(module, resourcePath, "INJECTED"); + EmbeddedResourcesInfo.singleton().declareResourceAsRegistered(module, resourcePath, "INJECTED"); Resources.singleton().registerResource(module, resourcePath, resourceContent); } @@ -284,7 +283,7 @@ private void processResourceFromModule(Module module, String resourcePath) { var resolvedModule = module.getLayer().configuration().findModule(module.getName()); if (resolvedModule.isPresent()) { Optional location = resolvedModule.get().reference().location(); - location.ifPresent(uri -> declareResourceAsRegistered(module, resourcePath, uri.toString())); + location.ifPresent(uri -> EmbeddedResourcesInfo.singleton().declareResourceAsRegistered(module, resourcePath, uri.toString())); } } catch (IOException e) { Resources.singleton().registerIOException(module, resourcePath, e, LinkAtBuildTimeSupport.singleton().packageOrClassAtBuildTime(resourcePath)); @@ -325,7 +324,7 @@ private void processResourceFromClasspath(String resourcePath) { } String source = ResourcesUtils.getResourceSource(url, resourcePath, fromJar); - declareResourceAsRegistered(null, resourcePath, source); + EmbeddedResourcesInfo.singleton().declareResourceAsRegistered(null, resourcePath, source); } catch (IOException e) { Resources.singleton().registerIOException(null, resourcePath, e, LinkAtBuildTimeSupport.singleton().packageOrClassAtBuildTime(resourcePath)); return; @@ -358,6 +357,8 @@ public void afterRegistration(AfterRegistrationAccess a) { ResourcesRegistryImpl resourcesRegistry = new ResourcesRegistryImpl(); ImageSingletons.add(ResourcesRegistry.class, resourcesRegistry); ImageSingletons.add(RuntimeResourceSupport.class, resourcesRegistry); + EmbeddedResourcesInfo embeddedResourcesInfo = new EmbeddedResourcesInfo(); + ImageSingletons.add(EmbeddedResourcesInfo.class, embeddedResourcesInfo); } private static ResourcesRegistryImpl resourceRegistryImpl() { @@ -504,7 +505,7 @@ public void registerIOException(Module module, String resourceName, IOException @Override public void registerNegativeQuery(Module module, String resourceName) { - declareResourceAsRegistered(module, resourceName, ""); + EmbeddedResourcesInfo.singleton().declareResourceAsRegistered(module, resourceName, ""); Resources.singleton().registerNegativeQuery(module, resourceName); } } From 4255e4e46f504555896eb97fec726ea0381adfff Mon Sep 17 00:00:00 2001 From: David Nestorovic Date: Tue, 26 Mar 2024 17:59:01 +0100 Subject: [PATCH 11/19] Add embedded-resources-schema --- .../embedded-resources-schema-v1.0.0.json | 56 +++++++++++++++++++ .../svm/hosted/EmbeddedResourcesInfo.java | 23 ++++++-- .../oracle/svm/hosted/ResourcesFeature.java | 2 +- 3 files changed, 75 insertions(+), 6 deletions(-) create mode 100644 docs/reference-manual/native-image/assets/embedded-resources-schema-v1.0.0.json diff --git a/docs/reference-manual/native-image/assets/embedded-resources-schema-v1.0.0.json b/docs/reference-manual/native-image/assets/embedded-resources-schema-v1.0.0.json new file mode 100644 index 000000000000..241e32ede4a7 --- /dev/null +++ b/docs/reference-manual/native-image/assets/embedded-resources-schema-v1.0.0.json @@ -0,0 +1,56 @@ +{ + "$schema": "https://json-schema.org/draft/2019-09/schema", + "$id": "https://github.com/oracle/graal/blob/master/docs/reference-manual/native-image/assets/embedded-resources-schema-v1.0.0.json", + "default": [], + "items": { + "properties": { + "name": { + "type": "string", + "title": "Name of the resource that was registered" + }, + "module": { + "type": "string", + "title": "Module of the resource that was registered" + }, + "is_directory": { + "type": "boolean", + "default": false, + "title": "Describes whether the registered resource is directory or not" + }, + "is_missing": { + "type": "boolean", + "default": false, + "title": "Describes whether the resource is missing on the system or not" + }, + "entries": { + "default": [], + "items": { + "properties": { + "origin": { + "type": "string", + "title": "Actual path to the resource" + }, + "size": { + "type": "integer", + "title": "Size of the resource expressed in bytes" + } + }, + "additionalProperties": false, + "type": "object", + "title": "Source of the resource defined with name and module properties" + }, + "type": "array", + "title": "List of sources for the resource defined with name and module properties" + } + }, + "required": [ + "name", + "entries" + ], + "additionalProperties": false, + "type": "object", + "title": "Resource that was registered" + }, + "type": "array", + "title": "JSON schema for the embedded-resources.json that shows all resources that were registered." +} \ No newline at end of file diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/EmbeddedResourcesInfo.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/EmbeddedResourcesInfo.java index 7a673cdf472b..e1d3377a3fea 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/EmbeddedResourcesInfo.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/EmbeddedResourcesInfo.java @@ -27,12 +27,25 @@ public static EmbeddedResourcesInfo singleton() { public void declareResourceAsRegistered(Module module, String resource, String source) { Resources.ModuleResourceRecord key = createStorageKey(module, resource); - synchronized (registeredResources) { - registeredResources.computeIfAbsent(key, k -> new ArrayList<>()); - if (!registeredResources.get(key).contains(source)) { - registeredResources.get(key).add(source); + registeredResources.compute(key, (k, v) -> { + if (v == null) { + ArrayList newValue = new ArrayList<>(); + newValue.add(source); + return newValue; } - } + + /* + * We have to avoid duplicated sources here. In case when declaring resource that comes + * from module as registered, we don't have information whether the resource is already + * registered or not. That check is performed later in {@link Resources.java#addEntry}, + * so we have to perform same check here, to avoid duplicates when collecting + * information about resource. + */ + if (!v.contains(source)) { + v.add(source); + } + return v; + }); } } diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java index ce475c4f715b..56bf8775165c 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java @@ -148,7 +148,7 @@ public static class Options { public static final String EMBEDDED_RESOURCES_FILE_NAME = "embedded-resources.json"; @Option(help = "Create a " + EMBEDDED_RESOURCES_FILE_NAME + " file in the build directory. The output conforms to the JSON schema located at: " + - "docs/reference-manual/native-image/assets/embedded-resources-schema-v0.1.0.json", type = OptionType.User)// + "docs/reference-manual/native-image/assets/embedded-resources-schema-v1.0.0.json", type = OptionType.User)// public static final HostedOptionKey GenerateEmbeddedResourcesFile = new HostedOptionKey<>(false); } From 1b5e4492063686a2f6db5a7a94e45366e822bc37 Mon Sep 17 00:00:00 2001 From: David Nestorovic Date: Wed, 27 Mar 2024 14:18:58 +0100 Subject: [PATCH 12/19] Add missing license headers --- .../svm/hosted/EmbeddedResourceExporter.java | 24 ++++++++++++++ .../svm/hosted/EmbeddedResourcesInfo.java | 24 ++++++++++++++ .../svm/hosted/util/ResourcesUtils.java | 32 ++++++++++++++++--- 3 files changed, 76 insertions(+), 4 deletions(-) diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/EmbeddedResourceExporter.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/EmbeddedResourceExporter.java index e227d5958a76..773020c017ff 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/EmbeddedResourceExporter.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/EmbeddedResourceExporter.java @@ -1,3 +1,27 @@ +/* + * Copyright (c) 2024, 2024, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ package com.oracle.svm.hosted; import static com.oracle.svm.core.jdk.Resources.NEGATIVE_QUERY_MARKER; diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/EmbeddedResourcesInfo.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/EmbeddedResourcesInfo.java index e1d3377a3fea..8defc8018018 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/EmbeddedResourcesInfo.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/EmbeddedResourcesInfo.java @@ -1,3 +1,27 @@ +/* + * Copyright (c) 2024, 2024, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ package com.oracle.svm.hosted; import static com.oracle.svm.core.jdk.Resources.createStorageKey; diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/util/ResourcesUtils.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/util/ResourcesUtils.java index 7b203567b344..ffefa13ea92b 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/util/ResourcesUtils.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/util/ResourcesUtils.java @@ -1,3 +1,27 @@ +/* + * Copyright (c) 2024, 2024, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ package com.oracle.svm.hosted.util; import java.io.IOException; @@ -18,7 +42,7 @@ public class ResourcesUtils { /** - * Returns jar path from the given url + * Returns jar path from the given url. */ private static String urlToJarPath(URL url) { try { @@ -29,7 +53,7 @@ private static String urlToJarPath(URL url) { } /** - * Returns directory that contains resource on the given url + * Returns directory that contains resource on the given url. */ public static String getResourceSource(URL url, String resource, boolean fromJar) { String source = fromJar ? Path.of(urlToJarPath(url)).toUri().toString() : Path.of(url.getPath()).toUri().toString(); @@ -46,7 +70,7 @@ public static String getResourceSource(URL url, String resource, boolean fromJar } /** - * Returns whether the given resource is directory or not + * Returns whether the given resource is directory or not. */ public static boolean resourceIsDirectory(URL url, boolean fromJar, String resource) throws IOException, URISyntaxException { if (fromJar) { @@ -59,7 +83,7 @@ public static boolean resourceIsDirectory(URL url, boolean fromJar, String resou } /** - * Returns directory content of the resource from the given path + * Returns directory content of the resource from the given path. */ public static String getDirectoryContent(String path, boolean fromJar) throws IOException { Set content = new TreeSet<>(); From 76c1583ffa16da60912fee97a951ab795a1560fd Mon Sep 17 00:00:00 2001 From: David Nestorovic Date: Thu, 28 Mar 2024 10:32:21 +0100 Subject: [PATCH 13/19] Use toURI instead of getPath to avoid errors on Windows --- .../svm/hosted/util/ResourcesUtils.java | 39 ++++++++++++------- 1 file changed, 25 insertions(+), 14 deletions(-) diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/util/ResourcesUtils.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/util/ResourcesUtils.java index ffefa13ea92b..efa633d00ace 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/util/ResourcesUtils.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/util/ResourcesUtils.java @@ -39,34 +39,37 @@ import java.util.jar.JarFile; import java.util.stream.Stream; +import com.oracle.svm.core.util.VMError; + public class ResourcesUtils { /** * Returns jar path from the given url. */ private static String urlToJarPath(URL url) { - try { - return ((JarURLConnection) url.openConnection()).getJarFileURL().toURI().getPath(); - } catch (IOException | URISyntaxException e) { - throw new RuntimeException(e); - } + return urlToJarUri(url).getPath(); } /** * Returns directory that contains resource on the given url. */ public static String getResourceSource(URL url, String resource, boolean fromJar) { - String source = fromJar ? Path.of(urlToJarPath(url)).toUri().toString() : Path.of(url.getPath()).toUri().toString(); - if (!fromJar) { - // -1 removes trailing slash from path of directory that contains resource - source = source.substring(0, source.length() - resource.length() - 1); - if (source.endsWith("/")) { - // if resource was directory we still have one slash at the end - source = source.substring(0, source.length() - 1); + try { + String source = fromJar ? urlToJarUri(url).toString() : url.toURI().toString(); + + if (!fromJar) { + // -1 removes trailing slash from path of directory that contains resource + source = source.substring(0, source.length() - resource.length() - 1); + if (source.endsWith("/")) { + // if resource was directory we still have one slash at the end + source = source.substring(0, source.length() - 1); + } } - } - return source; + return source; + } catch (URISyntaxException e) { + throw VMError.shouldNotReachHere("Cannot get uri from: " + url, e); + } } /** @@ -134,4 +137,12 @@ public static String getDirectoryContent(String path, boolean fromJar) throws IO return String.join(System.lineSeparator(), content); } + private static URI urlToJarUri(URL url) { + try { + return ((JarURLConnection) url.openConnection()).getJarFileURL().toURI(); + } catch (IOException | URISyntaxException e) { + throw new RuntimeException(e); + } + } + } From ad1e23748f77d4ea1aadd08bd5af52b692d8bfde Mon Sep 17 00:00:00 2001 From: David Nestorovic Date: Tue, 2 Apr 2024 15:59:13 +0200 Subject: [PATCH 14/19] Rename ModuleResourceRecord to ModuleResourceKey --- .../src/com/oracle/svm/core/jdk/Resources.java | 18 +++++++++--------- .../NativeImageResourceFileSystem.java | 2 +- .../svm/hosted/EmbeddedResourceExporter.java | 4 ++-- .../svm/hosted/EmbeddedResourcesInfo.java | 6 +++--- 4 files changed, 15 insertions(+), 15 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/Resources.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/Resources.java index 5addb8524de2..c96b02f132d6 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/Resources.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/Resources.java @@ -81,19 +81,19 @@ public static Resources singleton() { } /** - * The hosted map used to collect registered resources. Using a {@link ModuleResourceRecord} of + * The hosted map used to collect registered resources. Using a {@link ModuleResourceKey} of * (module, resourceName) provides implementations for {@code hashCode()} and {@code equals()} * needed for the map keys. Hosted module instances differ to runtime instances, so the map that * ends up in the image heap is computed after the runtime module instances have been computed * {see com.oracle.svm.hosted.ModuleLayerFeature}. */ - private final EconomicMap resources = ImageHeapMap.create(); + private final EconomicMap resources = ImageHeapMap.create(); private final EconomicMap requestedPatterns = ImageHeapMap.create(); public record RequestedPattern(String module, String resource) { } - public record ModuleResourceRecord(Module module, String resource) { + public record ModuleResourceKey(Module module, String resource) { } /** @@ -121,7 +121,7 @@ public record ModuleResourceRecord(Module module, String resource) { Resources() { } - public EconomicMap getResourceStorage() { + public EconomicMap getResourceStorage() { return resources; } @@ -141,19 +141,19 @@ public static String moduleName(Module module) { return module == null ? null : module.getName(); } - public static ModuleResourceRecord createStorageKey(Module module, String resourceName) { + public static ModuleResourceKey createStorageKey(Module module, String resourceName) { Module m = module != null && module.isNamed() ? module : null; if (ImageInfo.inImageBuildtimeCode()) { if (m != null) { m = RuntimeModuleSupport.instance().getRuntimeModuleForHostedModule(m); } } - return new ModuleResourceRecord(m, resourceName); + return new ModuleResourceKey(m, resourceName); } public static Set getIncludedResourcesModules() { return StreamSupport.stream(singleton().resources.getKeys().spliterator(), false) - .map(ModuleResourceRecord::module) + .map(ModuleResourceKey::module) .filter(Objects::nonNull) .map(Module::getName) .collect(Collectors.toSet()); @@ -178,7 +178,7 @@ private void addEntry(Module module, String resourceName, boolean isDirectory, b VMError.guarantee(!BuildPhaseProvider.isAnalysisFinished(), "Trying to add a resource entry after analysis."); Module m = module != null && module.isNamed() ? module : null; synchronized (resources) { - ModuleResourceRecord key = createStorageKey(m, resourceName); + ModuleResourceKey key = createStorageKey(m, resourceName); ResourceStorageEntryBase entry = resources.get(key); if (isNegativeQuery) { if (entry == null) { @@ -237,7 +237,7 @@ public void registerIOException(Module module, String resourceName, IOException LogUtils.warning("Resource " + resourceName + " from module " + moduleName(module) + " produced the following IOException: " + e.getClass().getTypeName() + ": " + e.getMessage()); } } - ModuleResourceRecord key = createStorageKey(module, resourceName); + ModuleResourceKey key = createStorageKey(module, resourceName); synchronized (resources) { updateTimeStamp(); resources.put(key, new ResourceExceptionEntry(e)); diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/resources/NativeImageResourceFileSystem.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/resources/NativeImageResourceFileSystem.java index 1f1ba248ecf3..e831a792dc9a 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/resources/NativeImageResourceFileSystem.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/resources/NativeImageResourceFileSystem.java @@ -656,7 +656,7 @@ private void update(Entry e) { } private void readAllEntries() { - MapCursor entries = Resources.singleton().getResourceStorage().getEntries(); + MapCursor entries = Resources.singleton().getResourceStorage().getEntries(); while (entries.advance()) { byte[] name = getBytes(entries.getKey().resource()); ResourceStorageEntryBase entry = entries.getValue(); diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/EmbeddedResourceExporter.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/EmbeddedResourceExporter.java index 773020c017ff..5bab14361b11 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/EmbeddedResourceExporter.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/EmbeddedResourceExporter.java @@ -93,9 +93,9 @@ private static void sourceElement(SourceSizePair p, JsonWriter w) throws IOExcep w.unindent(); } - private static List getResourceReportEntryList(ConcurrentHashMap> collection) { + private static List getResourceReportEntryList(ConcurrentHashMap> collection) { List resourceInfoList = new ArrayList<>(); - EconomicMap resourceStorage = Resources.singleton().getResourceStorage(); + EconomicMap resourceStorage = Resources.singleton().getResourceStorage(); resourceStorage.getKeys().forEach(key -> { Module module = key.module(); String resourceName = key.resource(); diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/EmbeddedResourcesInfo.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/EmbeddedResourcesInfo.java index 8defc8018018..5d4b03ef7b2f 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/EmbeddedResourcesInfo.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/EmbeddedResourcesInfo.java @@ -39,9 +39,9 @@ @Platforms(Platform.HOSTED_ONLY.class) public class EmbeddedResourcesInfo { - private final ConcurrentHashMap> registeredResources = new ConcurrentHashMap<>(); + private final ConcurrentHashMap> registeredResources = new ConcurrentHashMap<>(); - public ConcurrentHashMap> getRegisteredResources() { + public ConcurrentHashMap> getRegisteredResources() { return registeredResources; } @@ -50,7 +50,7 @@ public static EmbeddedResourcesInfo singleton() { } public void declareResourceAsRegistered(Module module, String resource, String source) { - Resources.ModuleResourceRecord key = createStorageKey(module, resource); + Resources.ModuleResourceKey key = createStorageKey(module, resource); registeredResources.compute(key, (k, v) -> { if (v == null) { ArrayList newValue = new ArrayList<>(); From 7c604b42dd307d59d0c4df2376510d6d30510ad7 Mon Sep 17 00:00:00 2001 From: David Nestorovic Date: Tue, 2 Apr 2024 17:32:54 +0200 Subject: [PATCH 15/19] Use EMBEDDED_RESOURCES_FILE_NAME constant for declaring file name in the location path --- .../native-image/assets/embedded-resources-schema-v1.0.0.json | 2 +- .../src/com/oracle/svm/hosted/ResourcesFeature.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/reference-manual/native-image/assets/embedded-resources-schema-v1.0.0.json b/docs/reference-manual/native-image/assets/embedded-resources-schema-v1.0.0.json index 241e32ede4a7..aff6871fbd32 100644 --- a/docs/reference-manual/native-image/assets/embedded-resources-schema-v1.0.0.json +++ b/docs/reference-manual/native-image/assets/embedded-resources-schema-v1.0.0.json @@ -28,7 +28,7 @@ "properties": { "origin": { "type": "string", - "title": "Actual path to the resource" + "title": "Resource path" }, "size": { "type": "integer", diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java index 56bf8775165c..1131b26c261c 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java @@ -547,7 +547,7 @@ boolean moduleNameMatches(String resourceContainerModuleName) { public void afterAnalysis(AfterAnalysisAccess access) { sealed = true; if (Options.GenerateEmbeddedResourcesFile.getValue()) { - Path reportLocation = NativeImageGenerator.generatedFiles(HostedOptionValues.singleton()).resolve("embedded-resources.json"); + Path reportLocation = NativeImageGenerator.generatedFiles(HostedOptionValues.singleton()).resolve(Options.EMBEDDED_RESOURCES_FILE_NAME); try (JsonWriter writer = new JsonWriter(reportLocation)) { EmbeddedResourceExporter.printReport(writer); } catch (IOException e) { From 26b9c60e3c4e664f23282baff36710eca62089a6 Mon Sep 17 00:00:00 2001 From: David Nestorovic Date: Thu, 4 Apr 2024 16:09:51 +0200 Subject: [PATCH 16/19] Collect resources info only when option is passed --- .../src/com/oracle/svm/hosted/EmbeddedResourcesInfo.java | 9 +++++---- .../src/com/oracle/svm/hosted/ResourcesFeature.java | 2 +- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/EmbeddedResourcesInfo.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/EmbeddedResourcesInfo.java index 5d4b03ef7b2f..3e2078fafc0c 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/EmbeddedResourcesInfo.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/EmbeddedResourcesInfo.java @@ -24,8 +24,6 @@ */ package com.oracle.svm.hosted; -import static com.oracle.svm.core.jdk.Resources.createStorageKey; - import java.util.ArrayList; import java.util.List; import java.util.concurrent.ConcurrentHashMap; @@ -50,7 +48,11 @@ public static EmbeddedResourcesInfo singleton() { } public void declareResourceAsRegistered(Module module, String resource, String source) { - Resources.ModuleResourceKey key = createStorageKey(module, resource); + if (!ResourcesFeature.Options.GenerateEmbeddedResourcesFile.getValue()) { + return; + } + + Resources.ModuleResourceKey key = Resources.createStorageKey(module, resource); registeredResources.compute(key, (k, v) -> { if (v == null) { ArrayList newValue = new ArrayList<>(); @@ -71,5 +73,4 @@ public void declareResourceAsRegistered(Module module, String resource, String s return v; }); } - } diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java index 1131b26c261c..ffb471943839 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java @@ -146,7 +146,7 @@ public static class Options { @Option(help = "Regexp to match names of resources to be excluded from the image.", type = OptionType.User)// public static final HostedOptionKey ExcludeResources = new HostedOptionKey<>(LocatableMultiOptionValue.Strings.build()); - public static final String EMBEDDED_RESOURCES_FILE_NAME = "embedded-resources.json"; + private static final String EMBEDDED_RESOURCES_FILE_NAME = "embedded-resources.json"; @Option(help = "Create a " + EMBEDDED_RESOURCES_FILE_NAME + " file in the build directory. The output conforms to the JSON schema located at: " + "docs/reference-manual/native-image/assets/embedded-resources-schema-v1.0.0.json", type = OptionType.User)// public static final HostedOptionKey GenerateEmbeddedResourcesFile = new HostedOptionKey<>(false); From 0895578485c28200be60f82e1f78539f5858e04d Mon Sep 17 00:00:00 2001 From: David Nestorovic Date: Mon, 8 Apr 2024 14:56:27 +0200 Subject: [PATCH 17/19] Extend embedded resources section in BuildOutput.md --- docs/reference-manual/native-image/BuildOutput.md | 3 +++ .../native-image/assets/embedded-resources-schema-v1.0.0.json | 2 +- substratevm/CHANGELOG.md | 1 + .../svm/core/jdk/resources/NativeImageResourceFileSystem.java | 2 +- .../src/com/oracle/svm/hosted/ResourcesFeature.java | 2 +- 5 files changed, 7 insertions(+), 3 deletions(-) diff --git a/docs/reference-manual/native-image/BuildOutput.md b/docs/reference-manual/native-image/BuildOutput.md index dffb2bf6e16d..e93037baa458 100644 --- a/docs/reference-manual/native-image/BuildOutput.md +++ b/docs/reference-manual/native-image/BuildOutput.md @@ -218,6 +218,9 @@ Therefore, this can also include `byte[]` objects from application code. ##### Embedded Resources Stored in `byte[]` The total size of all `byte[]` objects used for storing resources (for example, files accessed via `Class.getResource()`) within the native binary. The number of resources is shown in the [Heap](#glossary-image-heap) section. +A list of all resources including additional information such as their module, name, origin, and size are included in the [build reports](BuildOptions.md#build-output-and-build-report). +This information can also be requested in the JSON format using the `-H:+GenerateEmbeddedResourcesFile` option. +Such a JSON file validates against the JSON schema defined in [`embedded-resources-schema-v1.0.0.json`](https://github.com/oracle/graal/tree/master/docs/reference-manual/native-image/assets/embedded-resources-schema-v1.0.0.json). ##### Code Metadata Stored in `byte[]` The total size of all `byte[]` objects used for metadata for the [code area](#glossary-code-area). diff --git a/docs/reference-manual/native-image/assets/embedded-resources-schema-v1.0.0.json b/docs/reference-manual/native-image/assets/embedded-resources-schema-v1.0.0.json index aff6871fbd32..3d508d07ff77 100644 --- a/docs/reference-manual/native-image/assets/embedded-resources-schema-v1.0.0.json +++ b/docs/reference-manual/native-image/assets/embedded-resources-schema-v1.0.0.json @@ -15,7 +15,7 @@ "is_directory": { "type": "boolean", "default": false, - "title": "Describes whether the registered resource is directory or not" + "title": "Describes whether the registered resource is a directory or not" }, "is_missing": { "type": "boolean", diff --git a/substratevm/CHANGELOG.md b/substratevm/CHANGELOG.md index 8b3c5b39c908..e9c595f4dea1 100644 --- a/substratevm/CHANGELOG.md +++ b/substratevm/CHANGELOG.md @@ -16,6 +16,7 @@ This changelog summarizes major changes to GraalVM Native Image. * (GR-47832) Experimental support for upcalls from foreign code and other improvements to our implementation of the [Foreign Function & Memory API](https://github.com/oracle/graal/blob/master/docs/reference-manual/native-image/ForeignInterface.md) (part of "Project Panama", [JEP 454](https://openjdk.org/jeps/454)) on AMD64. Must be enabled with `-H:+ForeignAPISupport` (requiring `-H:+UnlockExperimentalVMOptions`). * (GR-52314) `-XX:MissingRegistrationReportingMode` can now be used on program invocation instead of as a build option, to avoid a rebuild when debugging missing registration errors. * (GR-51086) Introduce a new `--static-nolibc` API option as a replacement for the experimental `-H:±StaticExecutableWithDynamicLibC` option. +* (GR-52578) Print information about embedded resources into `embedded-resources.json` using the `-H:+GenerateEmbeddedResourcesFile` option. ## GraalVM for JDK 22 (Internal Version 24.0.0) * (GR-48304) Red Hat added support for the JFR event ThreadAllocationStatistics. diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/resources/NativeImageResourceFileSystem.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/resources/NativeImageResourceFileSystem.java index e831a792dc9a..8321db7613cd 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/resources/NativeImageResourceFileSystem.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/resources/NativeImageResourceFileSystem.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2021, 2024, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java index ffb471943839..fd5670aa8956 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015, 2023, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2015, 2024, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it From 8bf4abef34ce6b26429be2a31d140b514a4a12a7 Mon Sep 17 00:00:00 2001 From: David Nestorovic Date: Fri, 12 Apr 2024 11:04:07 +0200 Subject: [PATCH 18/19] Add embedded-resources.json in the list of produced artifacts --- .../src/com/oracle/svm/hosted/ResourcesFeature.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java index fd5670aa8956..693eb3cb4e32 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java @@ -61,6 +61,7 @@ import org.graalvm.nativeimage.impl.ConfigurationCondition; import org.graalvm.nativeimage.impl.RuntimeResourceSupport; +import com.oracle.svm.core.BuildArtifacts; import com.oracle.svm.core.ClassLoaderSupport; import com.oracle.svm.core.ClassLoaderSupport.ResourceCollector; import com.oracle.svm.core.MissingRegistrationUtils; @@ -553,6 +554,8 @@ public void afterAnalysis(AfterAnalysisAccess access) { } catch (IOException e) { throw VMError.shouldNotReachHere("Json writer cannot write to: " + reportLocation, e); } + + BuildArtifacts.singleton().add(BuildArtifacts.ArtifactType.BUILD_INFO, reportLocation); } } From 01ac29441dc5a07cfeb312c51cde2ae483e48e70 Mon Sep 17 00:00:00 2001 From: David Nestorovic Date: Fri, 12 Apr 2024 13:18:14 +0200 Subject: [PATCH 19/19] Do not print anything if data collection is disabled --- .../oracle/svm/hosted/EmbeddedResourceExporter.java | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/EmbeddedResourceExporter.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/EmbeddedResourceExporter.java index 5bab14361b11..73fc8ac37d29 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/EmbeddedResourceExporter.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/EmbeddedResourceExporter.java @@ -28,6 +28,7 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.Collections; import java.util.Comparator; import java.util.List; import java.util.concurrent.ConcurrentHashMap; @@ -41,6 +42,7 @@ import com.oracle.svm.core.util.VMError; import com.oracle.svm.core.util.json.JsonPrinter; import com.oracle.svm.core.util.json.JsonWriter; +import com.oracle.svm.util.LogUtils; @Platforms(Platform.HOSTED_ONLY.class) public class EmbeddedResourceExporter { @@ -94,6 +96,14 @@ private static void sourceElement(SourceSizePair p, JsonWriter w) throws IOExcep } private static List getResourceReportEntryList(ConcurrentHashMap> collection) { + if (collection.isEmpty()) { + LogUtils.warning("Attempting to write information about resources without data being collected. " + + "Either the GenerateEmbeddedResourcesFile hosted option is disabled " + + "or the application doesn't have any resource registered"); + + return Collections.emptyList(); + } + List resourceInfoList = new ArrayList<>(); EconomicMap resourceStorage = Resources.singleton().getResourceStorage(); resourceStorage.getKeys().forEach(key -> {