Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[api] Make resource loading compatible with java9 module #1541

Merged
merged 1 commit into from
Mar 23, 2022
Merged

[api] Make resource loading compatible with java9 module #1541

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
42 changes: 42 additions & 0 deletions api/src/main/java/ai/djl/util/ClassLoaderUtils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@

import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.lang.reflect.Constructor;
import java.net.URL;
import java.net.URLClassLoader;
Expand Down Expand Up @@ -167,4 +168,45 @@ public static ClassLoader getContextClassLoader() {
}
return cl;
}

/**
* Finds all the resources with the given name.
*
* @param name the resource name
* @return An enumeration of {@link java.net.URL URL} objects for the resource
* @throws IOException if I/O errors occur
*/
public static Enumeration<URL> getResources(String name) throws IOException {
return getContextClassLoader().getResources(name);
}

/**
* Finds the first resource in class path with the given name.
*
* @param name the resource name
* @return an enumeration of {@link java.net.URL URL} objects for the resource
* @throws IOException if I/O errors occur
*/
public static URL getResource(String name) throws IOException {
Enumeration<URL> en = getResources(name);
if (en.hasMoreElements()) {
return en.nextElement();
}
return null;
}

/**
* Returns an {@code InputStream} for reading from the resource.
*
* @param name the resource name
* @return an {@code InputStream} for reading
* @throws IOException if I/O errors occur
*/
public static InputStream getResourceAsStream(String name) throws IOException {
URL url = getResource(name);
if (url == null) {
throw new IOException("Resource not found in classpath: " + name);
}
return url.openStream();
}
}
22 changes: 12 additions & 10 deletions api/src/main/java/ai/djl/util/Platform.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -136,21 +136,23 @@ static Platform fromUrl(URL url) {
}

private static Platform fromSystem(String engine) {
String engineProp = '/' + engine + "-engine.properties";
String engineProp = engine + "-engine.properties";
String versionKey = engine + "_version";
Platform platform = fromSystem();
platform.placeholder = true;

URL url = Platform.class.getResource(engineProp);
if (url != null) {
try (InputStream is = url.openStream()) {
Properties prop = new Properties();
prop.load(is);
platform.version = prop.getProperty(versionKey);
platform.apiVersion = prop.getProperty("djl_version");
} catch (IOException e) {
throw new AssertionError("Failed to read property file: " + engineProp, e);
try {
URL url = ClassLoaderUtils.getResource(engineProp);
if (url != null) {
try (InputStream is = url.openStream()) {
Properties prop = new Properties();
prop.load(is);
platform.version = prop.getProperty(versionKey);
platform.apiVersion = prop.getProperty("djl_version");
}
}
} catch (IOException e) {
throw new AssertionError("Failed to read property file: " + engineProp, e);
}
return platform;
}
Expand Down
5 changes: 4 additions & 1 deletion basicdataset/src/main/java/ai/djl/basicdataset/tabular/AmesRandomAccess.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
import java.nio.file.Path;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import org.apache.commons.csv.CSVFormat;

Expand Down Expand Up @@ -237,7 +238,9 @@ public AmesRandomAccess build() {

private void parseFeatures() {
if (af == null) {
try (InputStream is = AmesRandomAccess.class.getResourceAsStream("ames.json");
try (InputStream is =
Objects.requireNonNull(
AmesRandomAccess.class.getResourceAsStream("ames.json"));
Reader reader = new InputStreamReader(is, StandardCharsets.UTF_8)) {
af = JsonUtils.GSON.fromJson(reader, AmesFeatures.class);
} catch (IOException e) {
Expand Down
15 changes: 5 additions & 10 deletions engines/dlr/dlr-engine/src/main/java/ai/djl/dlr/jni/LibUtils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
*/
package ai.djl.dlr.jni;

import ai.djl.util.ClassLoaderUtils;
import ai.djl.util.Platform;
import ai.djl.util.Utils;
import java.io.File;
Expand Down Expand Up @@ -93,12 +94,9 @@ private static String copyNativeLibraryFromClasspath(Platform platform) {
Files.createDirectories(dlrCacheRoot);
tmp = Files.createTempDirectory(dlrCacheRoot, "tmp");
for (String file : platform.getLibraries()) {
String libPath = "/native/lib/" + file;
String libPath = "native/lib/" + file;
logger.info("Extracting {} to cache ...", libPath);
try (InputStream is = LibUtils.class.getResourceAsStream(libPath)) {
if (is == null) {
throw new IllegalStateException("native library not found: " + libPath);
}
try (InputStream is = ClassLoaderUtils.getResourceAsStream(libPath)) {
Files.copy(is, tmp.resolve(file), StandardCopyOption.REPLACE_EXISTING);
}
}
Expand Down Expand Up @@ -165,11 +163,8 @@ private static String copyJniLibraryFromClasspath(Path nativeDir) {
}
Path tmp = null;
// both cpu & gpu share the same jnilib
String lib = "/jnilib/" + classifier + '/' + name;
try (InputStream is = LibUtils.class.getResourceAsStream(lib)) {
if (is == null) {
throw new UnsupportedOperationException("DLR is not supported by this platform");
}
String lib = "jnilib/" + classifier + '/' + name;
try (InputStream is = ClassLoaderUtils.getResourceAsStream(lib)) {
tmp = Files.createTempFile(nativeDir, "jni", "tmp");
Files.copy(is, tmp, StandardCopyOption.REPLACE_EXISTING);
Utils.moveQuietly(tmp, path);
Expand Down
8 changes: 3 additions & 5 deletions engines/mxnet/mxnet-engine/src/main/java/ai/djl/mxnet/jna/LibUtils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
*/
package ai.djl.mxnet.jna;

import ai.djl.util.ClassLoaderUtils;
import ai.djl.util.Platform;
import ai.djl.util.Utils;
import com.sun.jna.Library;
Expand Down Expand Up @@ -137,12 +138,9 @@ private static String loadLibraryFromClasspath(Platform platform) {
Files.createDirectories(cacheFolder);
tmp = Files.createTempDirectory(cacheFolder, "tmp");
for (String file : platform.getLibraries()) {
String libPath = "/native/lib/" + file;
String libPath = "native/lib/" + file;
logger.info("Extracting {} to cache ...", libPath);
try (InputStream is = LibUtils.class.getResourceAsStream(libPath)) {
if (is == null) {
throw new IllegalStateException("MXNet library not found: " + libPath);
}
try (InputStream is = ClassLoaderUtils.getResourceAsStream(libPath)) {
Files.copy(is, tmp.resolve(file), StandardCopyOption.REPLACE_EXISTING);
}
}
Expand Down
16 changes: 7 additions & 9 deletions engines/paddlepaddle/paddlepaddle-engine/src/main/java/ai/djl/paddlepaddle/jni/LibUtils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
*/
package ai.djl.paddlepaddle.jni;

import ai.djl.util.ClassLoaderUtils;
import ai.djl.util.Platform;
import ai.djl.util.Utils;
import java.io.File;
Expand Down Expand Up @@ -143,14 +144,11 @@ private static String copyJniLibraryFromClasspath(Path nativeDir) {

Path tmp = null;
// Paddle GPU and CPU share the same jni so file
String libPath = "/jnilib/" + classifier + "/cpu/" + name;
try (InputStream stream = LibUtils.class.getResourceAsStream(libPath)) {
String libPath = "jnilib/" + classifier + "/cpu/" + name;
try (InputStream is = ClassLoaderUtils.getResourceAsStream(libPath)) {
logger.info("Extracting {} to cache ...", libPath);
if (stream == null) {
throw new IllegalStateException("Paddle jni not found: " + libPath);
}
tmp = Files.createTempFile(nativeDir, "jni", "tmp");
Files.copy(stream, tmp, StandardCopyOption.REPLACE_EXISTING);
Files.copy(is, tmp, StandardCopyOption.REPLACE_EXISTING);
Utils.moveQuietly(tmp, path);
return path.toAbsolutePath().toString();
} catch (IOException e) {
Expand Down Expand Up @@ -191,17 +189,17 @@ private static String loadLibraryFromClasspath(Platform platform) {
Files.createDirectories(cacheFolder);
tmp = Files.createTempDirectory(cacheFolder, "tmp");
for (String file : platform.getLibraries()) {
String libPath = "/native/lib/" + file;
String libPath = "native/lib/" + file;
logger.info("Extracting {} to cache ...", file);
if (file.endsWith(".gz")) {
// FIXME: temporary workaround for paddlepaddle-native-cu102:2.0.2
String f = file.substring(0, file.length() - 3);
try (InputStream is =
new GZIPInputStream(LibUtils.class.getResourceAsStream(libPath))) {
new GZIPInputStream(ClassLoaderUtils.getResourceAsStream(libPath))) {
Files.copy(is, tmp.resolve(f), StandardCopyOption.REPLACE_EXISTING);
}
} else {
try (InputStream is = LibUtils.class.getResourceAsStream(libPath)) {
try (InputStream is = ClassLoaderUtils.getResourceAsStream(libPath)) {
Files.copy(is, tmp.resolve(file), StandardCopyOption.REPLACE_EXISTING);
}
}
Expand Down
24 changes: 11 additions & 13 deletions engines/pytorch/pytorch-engine/src/main/java/ai/djl/pytorch/jni/LibUtils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
*/
package ai.djl.pytorch.jni;

import ai.djl.util.ClassLoaderUtils;
import ai.djl.util.Platform;
import ai.djl.util.Utils;
import java.io.File;
Expand Down Expand Up @@ -205,11 +206,14 @@ private static Path findJniLibrary(LibTorch libTorch) {
}
version = matcher.group(1);

try (InputStream is = LibUtils.class.getResourceAsStream("/jnilib/pytorch.properties")) {
try {
URL url = ClassLoaderUtils.getResource("jnilib/pytorch.properties");
String jniVersion = null;
if (is != null) {
if (url != null) {
Properties prop = new Properties();
prop.load(is);
try (InputStream is = url.openStream()) {
prop.load(is);
}
jniVersion = prop.getProperty("jni_version");
if (jniVersion == null) {
throw new AssertionError("No PyTorch jni version found.");
Expand All @@ -228,12 +232,9 @@ private static Path findJniLibrary(LibTorch libTorch) {
}

Path tmp = null;
String libPath = "/jnilib/" + classifier + '/' + flavor + '/' + JNI_LIB_NAME;
String libPath = "jnilib/" + classifier + '/' + flavor + '/' + JNI_LIB_NAME;
logger.info("Extracting {} to cache ...", libPath);
try (InputStream is = LibUtils.class.getResourceAsStream(libPath)) {
if (is == null) {
throw new AssertionError("PyTorch jni not found: " + libPath);
}
try (InputStream is = ClassLoaderUtils.getResourceAsStream(libPath)) {
tmp = Files.createTempFile(dir, "jni", "tmp");
Files.copy(is, tmp, StandardCopyOption.REPLACE_EXISTING);
Utils.moveQuietly(tmp, path);
Expand Down Expand Up @@ -292,12 +293,9 @@ private static LibTorch copyNativeLibraryFromClasspath(Platform platform) {
Files.createDirectories(cacheDir);
tmp = Files.createTempDirectory(cacheDir, "tmp");
for (String file : platform.getLibraries()) {
String libPath = "/native/lib/" + file;
String libPath = "native/lib/" + file;
logger.info("Extracting {} to cache ...", libPath);
try (InputStream is = LibUtils.class.getResourceAsStream(libPath)) {
if (is == null) {
throw new IllegalStateException("PyTorch library not found: " + libPath);
}
try (InputStream is = ClassLoaderUtils.getResourceAsStream(libPath)) {
Files.copy(is, tmp.resolve(file), StandardCopyOption.REPLACE_EXISTING);
}
}
Expand Down
8 changes: 3 additions & 5 deletions ...tensorflow/tensorflow-engine/src/main/java/ai/djl/tensorflow/engine/javacpp/LibUtils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@

package ai.djl.tensorflow.engine.javacpp;

import ai.djl.util.ClassLoaderUtils;
import ai.djl.util.Platform;
import ai.djl.util.Utils;
import java.io.File;
Expand Down Expand Up @@ -104,12 +105,9 @@ private static String loadLibraryFromClasspath(Platform platform) {
Files.createDirectories(cacheFolder);
tmp = Files.createTempDirectory(cacheFolder, "tmp");
for (String file : platform.getLibraries()) {
String libPath = "/native/lib/" + file;
String libPath = "native/lib/" + file;
logger.info("Extracting {} to cache ...", libPath);
try (InputStream is = LibUtils.class.getResourceAsStream(libPath)) {
if (is == null) {
throw new IllegalStateException("Tensorflow library not found: " + libPath);
}
try (InputStream is = ClassLoaderUtils.getResourceAsStream(libPath)) {
Files.copy(is, tmp.resolve(file), StandardCopyOption.REPLACE_EXISTING);
}
}
Expand Down
8 changes: 3 additions & 5 deletions engines/tensorrt/src/main/java/ai/djl/tensorrt/jni/LibUtils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
*/
package ai.djl.tensorrt.jni;

import ai.djl.util.ClassLoaderUtils;
import ai.djl.util.Platform;
import ai.djl.util.Utils;
import java.io.IOException;
Expand Down Expand Up @@ -63,13 +64,10 @@ private static String copyJniLibraryFromClasspath() {
Path tmp = null;
String libPath = "/jnilib/" + classifier + "/" + name;
logger.info("Extracting {} to cache ...", libPath);
try (InputStream stream = LibUtils.class.getResourceAsStream(libPath)) {
if (stream == null) {
throw new IllegalStateException("TensorRT library not found: " + libPath);
}
try (InputStream is = ClassLoaderUtils.getResourceAsStream(libPath)) {
Files.createDirectories(dir);
tmp = Files.createTempFile(cacheDir, "jni", "tmp");
Files.copy(stream, tmp, StandardCopyOption.REPLACE_EXISTING);
Files.copy(is, tmp, StandardCopyOption.REPLACE_EXISTING);
Utils.moveQuietly(tmp, path);
return path.toAbsolutePath().toString();
} catch (IOException e) {
Expand Down
8 changes: 3 additions & 5 deletions engines/tflite/tflite-engine/src/main/java/ai/djl/tflite/engine/LibUtils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
*/
package ai.djl.tflite.engine;

import ai.djl.util.ClassLoaderUtils;
import ai.djl.util.Platform;
import ai.djl.util.Utils;
import java.io.IOException;
Expand Down Expand Up @@ -80,12 +81,9 @@ private static String loadLibraryFromClasspath(Platform platform) {
Files.createDirectories(cacheFolder);
tmp = Files.createTempDirectory(cacheFolder, "tmp");
for (String file : platform.getLibraries()) {
String libPath = "/native/lib/" + file;
String libPath = "native/lib/" + file;
logger.info("Extracting {} to cache ...", libPath);
try (InputStream is = LibUtils.class.getResourceAsStream(libPath)) {
if (is == null) {
throw new IllegalStateException("TFLite library not found: " + libPath);
}
try (InputStream is = ClassLoaderUtils.getResourceAsStream(libPath)) {
Files.copy(is, tmp.resolve(file), StandardCopyOption.REPLACE_EXISTING);
}
}
Expand Down
5 changes: 3 additions & 2 deletions extensions/aws-ai/src/main/java/ai/djl/aws/sagemaker/SageMaker.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
import java.nio.file.Paths;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.Objects;
import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;
import org.apache.commons.compress.compressors.gzip.GzipCompressorOutputStream;
Expand Down Expand Up @@ -302,7 +303,7 @@ private Path tar(Path dir) throws IOException {

private void addToTar(Path root, Path file, TarArchiveOutputStream tos) throws IOException {
Path relative = root.relativize(file);
String name = modelName + '/' + relative.toString();
String name = modelName + '/' + relative;
if (Files.isDirectory(file)) {
File[] files = file.toFile().listFiles();
if (files != null) {
Expand Down Expand Up @@ -433,7 +434,7 @@ private String getContainerImageName() {
}

private static String readPolicyDocument(String path) {
try (InputStream is = SageMaker.class.getResourceAsStream(path)) {
try (InputStream is = Objects.requireNonNull(SageMaker.class.getResourceAsStream(path))) {
return Utils.toString(is);
} catch (IOException e) {
throw new AssertionError("Failed to read " + path, e);
Expand Down
10 changes: 4 additions & 6 deletions extensions/fasttext/src/main/java/ai/djl/fasttext/jni/LibUtils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
*/
package ai.djl.fasttext.jni;

import ai.djl.util.ClassLoaderUtils;
import ai.djl.util.Platform;
import ai.djl.util.Utils;
import java.io.IOException;
Expand Down Expand Up @@ -63,14 +64,11 @@ private static String copyJniLibraryFromClasspath() {
return path.toAbsolutePath().toString();
}
Path tmp = null;
String libPath = "/native/lib/" + classifier + "/" + name;
try (InputStream stream = LibUtils.class.getResourceAsStream(libPath)) {
if (stream == null) {
throw new IllegalStateException("fastText library not found: " + libPath);
}
String libPath = "native/lib/" + classifier + "/" + name;
try (InputStream is = ClassLoaderUtils.getResourceAsStream(libPath)) {
Files.createDirectories(nativeDir.resolve(version));
tmp = Files.createTempFile(nativeDir, "jni", "tmp");
Files.copy(stream, tmp, StandardCopyOption.REPLACE_EXISTING);
Files.copy(is, tmp, StandardCopyOption.REPLACE_EXISTING);
Utils.moveQuietly(tmp, path);
return path.toAbsolutePath().toString();
} catch (IOException e) {
Expand Down
Loading