diff --git a/jqm-all/jqm-api/pom.xml b/jqm-all/jqm-api/pom.xml index 913fce194..b393fedab 100644 --- a/jqm-all/jqm-api/pom.xml +++ b/jqm-all/jqm-api/pom.xml @@ -1,36 +1,49 @@ - - 4.0.0 - - com.enioka.jqm - jqm-all - 2.2.10-SNAPSHOT - - jqm-api + + 4.0.0 + + com.enioka.jqm + jqm-all + 2.2.10-SNAPSHOT + + jqm-api - ${project.groupId}:${project.artifactId} - http://jqm.readthedocs.org - JQM public API interfaces + ${project.groupId}:${project.artifactId} + http://jqm.readthedocs.org + JQM public API interfaces - - - - maven-antrun-plugin - - - copy-xsd - - run - - compile - - - - - - - - + + + + maven-antrun-plugin + + + copy-xsd + + run + + compile + + + + + + + + - - + + org.apache.maven.plugins + maven-jar-plugin + + + + com.enioka.jqm.payload.api + + + + + + + diff --git a/jqm-all/jqm-engine/src/main/java/com/enioka/jqm/tools/JndiContext.java b/jqm-all/jqm-engine/src/main/java/com/enioka/jqm/tools/JndiContext.java index 96225c01a..fcfb5b411 100644 --- a/jqm-all/jqm-engine/src/main/java/com/enioka/jqm/tools/JndiContext.java +++ b/jqm-all/jqm-engine/src/main/java/com/enioka/jqm/tools/JndiContext.java @@ -20,20 +20,25 @@ import java.io.File; import java.lang.management.ManagementFactory; +import java.lang.module.Configuration; +import java.lang.module.ModuleDescriptor; +import java.lang.module.ModuleFinder; +import java.lang.module.ModuleReference; import java.lang.reflect.Method; import java.net.MalformedURLException; import java.net.URL; import java.net.URLClassLoader; +import java.nio.file.Path; import java.rmi.Remote; import java.rmi.registry.Registry; -import java.security.AccessController; -import java.security.PrivilegedAction; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; import java.util.Hashtable; import java.util.List; import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; import javax.management.MBeanServer; import javax.management.ObjectName; @@ -54,7 +59,7 @@ /** * This class implements a basic JNDI context - * + * */ class JndiContext extends InitialContext implements InitialContextFactoryBuilder, InitialContextFactory, NameParser { @@ -64,13 +69,14 @@ class JndiContext extends InitialContext implements InitialContextFactoryBuilder private List jmxNames = new ArrayList(); private Registry r = null; private ClassLoader extResources; + private ModuleLayer extLayer; /** * Will create a JNDI Context and register it as the initial context factory builder - * + * * @return the context * @throws NamingException - * on any issue during initial context factory builder registration + * on any issue during initial context factory builder registration */ static JndiContext createJndiContext() throws NamingException { @@ -98,7 +104,7 @@ static JndiContext createJndiContext() throws NamingException /** * Create a new Context - * + * * @throws NamingException */ private JndiContext() throws NamingException @@ -128,18 +134,26 @@ private JndiContext() throws NamingException // Create classloader final URL[] aUrls = urls.toArray(new URL[0]); + final Path[] aPaths = new Path[aUrls.length]; for (URL u : aUrls) { + aPaths[urls.indexOf(u)] = Path.of(u.getPath()); jqmlogger.trace(u.toString()); } - extResources = AccessController.doPrivileged(new PrivilegedAction() - { - @Override - public URLClassLoader run() - { - return new URLClassLoader(aUrls, getParentCl()); - } - }); + // TODO: test if java >=9 + /* + * extResources = AccessController.doPrivileged(new PrivilegedAction() { + * + * @Override public URLClassLoader run() { return new URLClassLoader(aUrls, getParentCl()); } }); + */ + + ModuleFinder moduleFinder = ModuleFinder.of(aPaths); + Set moduleNames = moduleFinder.findAll().stream().map(ModuleReference::descriptor).map(ModuleDescriptor::name) + .collect(Collectors.toUnmodifiableSet()); + Configuration cf = ModuleLayer.boot().configuration().resolveAndBind(ModuleFinder.of(), moduleFinder, moduleNames); + extLayer = ModuleLayer.boot().defineModulesWithOneLoader(cf, getParentCl()); + extResources = extLayer.modules().isEmpty() ? new URLClassLoader(new URL[0]) + : extLayer.modules().iterator().next().getClassLoader(); } else { @@ -356,7 +370,7 @@ public void bind(Name name, Object obj) throws NamingException /** * Will register the given Registry as a provider for the RMI: context. If there is already a registered Registry, the call is ignored. - * + * * @param r */ void registerRmiContext(Registry r) @@ -375,6 +389,11 @@ ClassLoader getExtCl() return this.extResources; } + public ModuleLayer getModuleLayer() + { + return this.extLayer; + } + @Override public void unbind(Name name) throws NamingException { @@ -427,4 +446,4 @@ private static ClassLoader getParentCl() throw new JqmInitError("Could not fetch Platform Class Loader", e); } } -} \ No newline at end of file +} diff --git a/jqm-all/jqm-engine/src/main/java/com/enioka/jqm/tools/RunningJobInstance.java b/jqm-all/jqm-engine/src/main/java/com/enioka/jqm/tools/RunningJobInstance.java index 0e704ea46..48b51c02f 100644 --- a/jqm-all/jqm-engine/src/main/java/com/enioka/jqm/tools/RunningJobInstance.java +++ b/jqm-all/jqm-engine/src/main/java/com/enioka/jqm/tools/RunningJobInstance.java @@ -425,6 +425,20 @@ public ClassLoader getExtensionClassloader() return extLoader; } + @Override + public ModuleLayer getExtensionModuleLayer() + { + try + { + return ((JndiContext) NamingManager.getInitialContext(null)).getModuleLayer(); + } + catch (NamingException e) + { + jqmlogger.warn("could not find ext directory class loader. It will not be used", e); + return null; + } + } + @Override public ClassLoader getEngineClassloader() { diff --git a/jqm-all/jqm-integration-tests/pom.xml b/jqm-all/jqm-integration-tests/pom.xml index 799572d1a..66586995f 100644 --- a/jqm-all/jqm-integration-tests/pom.xml +++ b/jqm-all/jqm-integration-tests/pom.xml @@ -104,7 +104,7 @@ jqm-api ${project.version} - + ${project.basedir}/ext diff --git a/jqm-all/jqm-integration-tests/src/test/java/com/enioka/jqm/tools/EngineJpmsTest.java b/jqm-all/jqm-integration-tests/src/test/java/com/enioka/jqm/tools/EngineJpmsTest.java new file mode 100644 index 000000000..792e49e1b --- /dev/null +++ b/jqm-all/jqm-integration-tests/src/test/java/com/enioka/jqm/tools/EngineJpmsTest.java @@ -0,0 +1,76 @@ +package com.enioka.jqm.tools; + +import org.junit.Assert; +import org.junit.Test; + +import com.enioka.jqm.api.JobRequest; +import com.enioka.jqm.test.helpers.CreationTools; +import com.enioka.jqm.test.helpers.TestHelpers; + +public class EngineJpmsTest extends JqmBaseTest +{ + @Test + public void testJpmsModuleStartsWithApi() + { + addAndStartEngine(); + + CreationTools.createJobDef(null, true, "com.enioka.jqm.tests.jpms/com.enioka.jqm.tests.jpms.SimpleJpmsPayload", null, + "jqm-tests/jqm-test-jpms/target/test.jar", TestHelpers.qVip, -1, "SimpleJpmsPayload", null, null, null, null, null, false, + cnx, null, false); + JobRequest.create("SimpleJpmsPayload", null).submit(); + + TestHelpers.waitFor(1, 10000, cnx); + + Assert.assertEquals(1, TestHelpers.getOkCount(cnx)); + Assert.assertEquals(0, TestHelpers.getNonOkCount(cnx)); + } + + @Test + public void testNonJpmsStartInClassicMode() + { + addAndStartEngine(); + + CreationTools.createJobDef(null, true, "pyl.Nothing", null, "jqm-tests/jqm-test-pyl-nodep/target/test.jar", TestHelpers.qVip, -1, + "SimpleNonJpmsPayload", null, null, null, null, null, false, cnx, null, false); + JobRequest.create("SimpleNonJpmsPayload", null).submit(); + + TestHelpers.waitFor(1, 10000, cnx); + + Assert.assertEquals(1, TestHelpers.getOkCount(cnx)); + Assert.assertEquals(0, TestHelpers.getNonOkCount(cnx)); + } + + @Test + public void testNonJpmsStartInJpmsMode() + { + addAndStartEngine(); + + // Note we are using an automatic module name here. + CreationTools.createJobDef(null, true, "test/pyl.Nothing", null, "jqm-tests/jqm-test-pyl-nodep/target/test.jar", TestHelpers.qVip, + -1, "SimpleNonJpmsPayload", null, null, null, null, null, false, cnx, null, false); + JobRequest.create("SimpleNonJpmsPayload", null).submit(); + + TestHelpers.waitFor(1, 10000, cnx); + + Assert.assertEquals(1, TestHelpers.getOkCount(cnx)); + Assert.assertEquals(0, TestHelpers.getNonOkCount(cnx)); + } + + @Test + public void testJpmsWithJndiCall() + { + // This tests checks the JNDI provider located inside the ext layer is accessible to the payload. + CreationTools.createJndiFile(cnx, "fs/test", "test resource", "/tmp"); + + JqmSimpleTest.create(cnx, "test/pyl.JndiFile", "jqm-test-pyl-nodep").run(this); + } + + @Test + public void testNonJpmsWithJndiCall() + { + // Sanity check. + CreationTools.createJndiFile(cnx, "fs/test", "test resource", "/tmp"); + + JqmSimpleTest.create(cnx, "pyl.JndiFile", "jqm-test-pyl-nodep").run(this); + } +} diff --git a/jqm-all/jqm-runner/jqm-runner-api/src/main/java/com/enioka/jqm/api/JobRunnerCallback.java b/jqm-all/jqm-runner/jqm-runner-api/src/main/java/com/enioka/jqm/api/JobRunnerCallback.java index 1ff2a7406..6ceff2e51 100644 --- a/jqm-all/jqm-runner/jqm-runner-api/src/main/java/com/enioka/jqm/api/JobRunnerCallback.java +++ b/jqm-all/jqm-runner/jqm-runner-api/src/main/java/com/enioka/jqm/api/JobRunnerCallback.java @@ -11,14 +11,14 @@ public interface JobRunnerCallback { /** * Should the runner use JMX? - * + * * @return */ public boolean isJmxEnabled(); /** * Generates the name the runner should use as a JMX bean if it registers a bean. - * + * * @return */ public String getJmxBeanName(); @@ -30,21 +30,23 @@ public interface JobRunnerCallback /** * Fetches a more or less accurate current run time of the running JI. - * + * * @return */ public Long getRunTimeSeconds(); /** * This CL contains /ext and has the bootstrap CL as parent. - * + * * @return */ public ClassLoader getExtensionClassloader(); + public ModuleLayer getExtensionModuleLayer(); + /** * This is the normal CL of the engine. It should never be visible to payloads. - * + * * @return */ public ClassLoader getEngineClassloader(); @@ -58,4 +60,4 @@ public interface JobRunnerCallback * How to contact the WS. */ public String getWebApiLocalUrl(DbConn cnx); -} \ No newline at end of file +} diff --git a/jqm-all/jqm-runner/jqm-runner-java/src/main/java/com/enioka/jqm/tools/JavaJobInstanceTracker.java b/jqm-all/jqm-runner/jqm-runner-java/src/main/java/com/enioka/jqm/tools/JavaJobInstanceTracker.java index 1008cda07..395a93e38 100644 --- a/jqm-all/jqm-runner/jqm-runner-java/src/main/java/com/enioka/jqm/tools/JavaJobInstanceTracker.java +++ b/jqm-all/jqm-runner/jqm-runner-java/src/main/java/com/enioka/jqm/tools/JavaJobInstanceTracker.java @@ -124,7 +124,7 @@ public State run() // Go! (launches the main function in the startup class designated in the manifest) try { - jobClassLoader.launchJar(job, job.getPrms(), clm, handler); + jobClassLoader.launchJar(job, job.getPrms(), clm, handler, engineCallback.getExtensionModuleLayer()); return State.ENDED; } catch (JqmKillException e) diff --git a/jqm-all/jqm-runner/jqm-runner-java/src/main/java/com/enioka/jqm/tools/ModuleManager.java b/jqm-all/jqm-runner/jqm-runner-java/src/main/java/com/enioka/jqm/tools/ModuleManager.java new file mode 100644 index 000000000..0733d028e --- /dev/null +++ b/jqm-all/jqm-runner/jqm-runner-java/src/main/java/com/enioka/jqm/tools/ModuleManager.java @@ -0,0 +1,115 @@ +package com.enioka.jqm.tools; + +import java.io.File; +import java.lang.module.Configuration; +import java.lang.module.ModuleDescriptor; +import java.lang.module.ModuleFinder; +import java.lang.module.ModuleReference; +import java.net.URISyntaxException; +import java.net.URLClassLoader; +import java.nio.file.Path; +import java.util.Set; +import java.util.function.Function; +import java.util.stream.Collectors; + +import org.apache.commons.io.FilenameUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.enioka.jqm.model.JobInstance; + +public class ModuleManager +{ + private static Logger jqmlogger = LoggerFactory.getLogger(ModuleManager.class); + + // This class would be a lambda in Java8+ + private static class ClMapper implements Function + { + private PayloadClassLoader cl; + + public ClMapper(PayloadClassLoader cl) + { + this.cl = cl; + } + + @Override + public ClassLoader apply(String t) + { + return cl; + } + } + + public static ModuleLayer createModuleLayerIfNeeded(PayloadClassLoader cl, ModuleLayer parentModuleLayer, JobInstance ji) + { + // TODO : cache layer if needed. + + // Test if the target is a module (just for information sake) + File jarFile = new File(FilenameUtils.concat(new File(ji.getNode().getRepo()).getAbsolutePath(), ji.getJD().getJarPath())); + ModuleFinder finder = ModuleFinder.of(jarFile.toPath()); + if (finder.findAll().isEmpty()) + { + jqmlogger.debug("Root {} is not a JPMS module, will be loaded as an automatic module", jarFile.getAbsolutePath()); + } + else + { + String mainModuleName = finder.findAll().iterator().next().descriptor().name(); + jqmlogger.debug("Root file {} is a JPMS module, using JPMS module layer. Root module name is {}", jarFile.getAbsolutePath(), + mainModuleName); + } + + // Convert URL to path (no streams in Java 6...) + Path[] paths = new Path[cl.getURLs().length]; + for (int i = 0; i < cl.getURLs().length; i++) + { + try + { + paths[i] = Path.of(cl.getURLs()[i].toURI()); + } + catch (URISyntaxException e) + { + throw new JqmPayloadException("wrong JPMS configuration", e); + } + } + if (cl.getParent() instanceof URLClassLoader) + { + jqmlogger.debug("JPMS module detected, adding parent classloader to module path"); + paths = new Path[cl.getURLs().length + ((URLClassLoader) cl.getParent()).getURLs().length]; + for (int i = 0; i < cl.getURLs().length; i++) + { + try + { + paths[i] = Path.of(cl.getURLs()[i].toURI()); + } + catch (URISyntaxException e) + { + throw new JqmPayloadException("wrong JPMS configuration", e); + } + } + for (int i = 0; i < ((URLClassLoader) cl.getParent()).getURLs().length; i++) + { + try + { + paths[i + cl.getURLs().length] = Path.of(((URLClassLoader) cl.getParent()).getURLs()[i].toURI()); + } + catch (URISyntaxException e) + { + throw new JqmPayloadException("wrong JPMS configuration", e); + } + } + } + + jqmlogger.debug("JPMS module path is {}", (Object) paths); + + // Add all files inside the JI path inside the new layer configuration as module roots. + ModuleFinder moduleFinder = ModuleFinder.of(paths); + Set moduleNames = moduleFinder.findAll().stream().map(ModuleReference::descriptor).map(ModuleDescriptor::name) + .collect(Collectors.toUnmodifiableSet()); + + // TODO: before/after according to child/parent first + Configuration cf = parentModuleLayer.configuration().resolveAndBind(moduleFinder, ModuleFinder.of(), moduleNames); + ModuleLayer layer = parentModuleLayer.defineModules(cf, new ClMapper(cl)); + + return layer; + } + +} diff --git a/jqm-all/jqm-runner/jqm-runner-java/src/main/java/com/enioka/jqm/tools/PayloadClassLoader.java b/jqm-all/jqm-runner/jqm-runner-java/src/main/java/com/enioka/jqm/tools/PayloadClassLoader.java index 67ba28d82..44114eab6 100644 --- a/jqm-all/jqm-runner/jqm-runner-java/src/main/java/com/enioka/jqm/tools/PayloadClassLoader.java +++ b/jqm-all/jqm-runner/jqm-runner-java/src/main/java/com/enioka/jqm/tools/PayloadClassLoader.java @@ -79,18 +79,19 @@ void extendUrls(URL jarUrl, URL[] libs) /** * Everything here can run without the database. - * + * * @param job - * the JI to run. + * the JI to run. * @param parameters - * already resolved runtime parameters + * already resolved runtime parameters * @param clm - * the CLM having created this CL. + * the CLM having created this CL. * @param h - * given as parameter because its constructor needs the database. + * given as parameter because its constructor needs the database. * @throws JqmEngineException */ - void launchJar(JobInstance job, Map parameters, ClassloaderManager clm, EngineApiProxy h) throws JobRunnerException + void launchJar(JobInstance job, Map parameters, ClassloaderManager clm, EngineApiProxy h, ModuleLayer parentModuleLayer) + throws JobRunnerException { // 1 - Create the proxy. Object proxy = null; @@ -116,8 +117,18 @@ void launchJar(JobInstance job, Map parameters, ClassloaderManag Class c = null; try { - // using payload CL, i.e. this very object - c = loadClass(classQualifiedName); + String[] classModuleSegments = classQualifiedName.split("/"); + if (classModuleSegments.length > 1) + { + ModuleManager mm = new ModuleManager(); + ModuleLayer ml = mm.createModuleLayerIfNeeded(this, parentModuleLayer, job); + c = ml.findLoader(classModuleSegments[0]).loadClass(classModuleSegments[1]); + } + else + { + // using payload CL, i.e. this very object + c = loadClass(classQualifiedName); + } } catch (Exception e) { diff --git a/jqm-all/jqm-tests/jqm-test-jpms/pom.xml b/jqm-all/jqm-tests/jqm-test-jpms/pom.xml new file mode 100644 index 000000000..985ca4242 --- /dev/null +++ b/jqm-all/jqm-tests/jqm-test-jpms/pom.xml @@ -0,0 +1,37 @@ + + + 4.0.0 + + com.enioka.jqm + jqm-tests + 2.2.10-SNAPSHOT + + + jqm-test-jpms + ${project.groupId}:${project.artifactId} + http://jqm.readthedocs.org + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.12.1 + + 1.9 + 1.9 + + + + + + + + com.enioka.jqm + jqm-api + ${project.version} + provided + + + diff --git a/jqm-all/jqm-tests/jqm-test-jpms/src/main/java/com/enioka/jqm/tests/jpms/JndiJpmsPayload.java b/jqm-all/jqm-tests/jqm-test-jpms/src/main/java/com/enioka/jqm/tests/jpms/JndiJpmsPayload.java new file mode 100644 index 000000000..4f1929d87 --- /dev/null +++ b/jqm-all/jqm-tests/jqm-test-jpms/src/main/java/com/enioka/jqm/tests/jpms/JndiJpmsPayload.java @@ -0,0 +1,23 @@ +package com.enioka.jqm.tests.jpms; + +import java.io.File; + +import javax.naming.NamingException; +import javax.naming.spi.NamingManager; + +public class JndiJpmsPayload implements Runnable +{ + public void run() + { + File o; + try + { + o = (File) NamingManager.getInitialContext(null).lookup("fs/test"); + } + catch (NamingException e) + { + throw new RuntimeException("could not get the test directory", e); + } + System.out.println(o.getAbsolutePath()); + } +} diff --git a/jqm-all/jqm-tests/jqm-test-jpms/src/main/java/com/enioka/jqm/tests/jpms/SimpleJpmsPayload.java b/jqm-all/jqm-tests/jqm-test-jpms/src/main/java/com/enioka/jqm/tests/jpms/SimpleJpmsPayload.java new file mode 100644 index 000000000..c7526c48c --- /dev/null +++ b/jqm-all/jqm-tests/jqm-test-jpms/src/main/java/com/enioka/jqm/tests/jpms/SimpleJpmsPayload.java @@ -0,0 +1,14 @@ +package com.enioka.jqm.tests.jpms; + +import com.enioka.jqm.api.JobManager; + +public class SimpleJpmsPayload implements Runnable +{ + public JobManager jobManager; + + public void run() + { + System.out.println("Hello world!"); + System.out.println("JI is " + jobManager.jobInstanceID()); + } +} diff --git a/jqm-all/jqm-tests/jqm-test-jpms/src/main/java/module-info.java b/jqm-all/jqm-tests/jqm-test-jpms/src/main/java/module-info.java new file mode 100644 index 000000000..fc3b49bd3 --- /dev/null +++ b/jqm-all/jqm-tests/jqm-test-jpms/src/main/java/module-info.java @@ -0,0 +1,7 @@ +module com.enioka.jqm.tests.jpms +{ + requires com.enioka.jqm.payload.api; + requires java.naming; + + exports com.enioka.jqm.tests.jpms; +} diff --git a/jqm-all/jqm-tests/pom.xml b/jqm-all/jqm-tests/pom.xml index 7a904da76..60bd6c84e 100644 --- a/jqm-all/jqm-tests/pom.xml +++ b/jqm-all/jqm-tests/pom.xml @@ -1,94 +1,96 @@ - - 4.0.0 - - com.enioka.jqm - jqm-all - 2.2.10-SNAPSHOT - + + 4.0.0 + + com.enioka.jqm + jqm-all + 2.2.10-SNAPSHOT + - jqm-tests - pom + jqm-tests + pom - ${project.groupId}:${project.artifactId} - http://jqm.readthedocs.org + ${project.groupId}:${project.artifactId} + http://jqm.readthedocs.org - - jqm-test-cl-isolation - jqm-test-datetimemaven - jqm-test-datetimemavenjarinlib - jqm-test-datetimemavennolibdef - jqm-test-datetimemavennopom - jqm-test-datetimemavennopomlib - jqm-test-em - jqm-test-jndijms-amq - jqm-test-providedapi - jqm-test-pyl - jqm-test-pyl-hibapi - jqm-test-pyl-nodep - jqm-test-spring-1 - jqm-test-spring-2 - + + jqm-test-cl-isolation + jqm-test-datetimemaven + jqm-test-datetimemavenjarinlib + jqm-test-datetimemavennolibdef + jqm-test-datetimemavennopom + jqm-test-datetimemavennopomlib + jqm-test-em + jqm-test-jndijms-amq + jqm-test-jpms + jqm-test-providedapi + jqm-test-pyl + jqm-test-pyl-hibapi + jqm-test-pyl-nodep + jqm-test-spring-1 + jqm-test-spring-2 + - - true - + + true + - - - - - - ../jqm-tests - - - - - - org.apache.maven.plugins - maven-dependency-plugin - 2.8 - - false - - - - copy - package - - copy - - - - - ${project.groupId} - ${project.artifactId} - ${project.version} - pom - true - pom.xml - - - ${project.groupId} - ${project.artifactId} - ${project.version} - jar - true - test.jar - - - ${project.build.directory} - - - - - - - - + + + + ../jqm-tests + + + + + + org.apache.maven.plugins + maven-dependency-plugin + 2.8 + + false + + + + copy + package + + copy + + + + + ${project.groupId} + ${project.artifactId} + ${project.version} + pom + true + pom.xml + + + ${project.groupId} + ${project.artifactId} + ${project.version} + jar + true + test.jar + + + ${project.build.directory} + + + + + + + + - \ No newline at end of file + diff --git a/jqm-all/jqm-wstst/pom.xml b/jqm-all/jqm-wstst/pom.xml index 7dfde07e1..cd97f2fb7 100644 --- a/jqm-all/jqm-wstst/pom.xml +++ b/jqm-all/jqm-wstst/pom.xml @@ -5,7 +5,7 @@ jqm-all com.enioka.jqm - 2.2.9-SNAPSHOT + 2.2.10-SNAPSHOT jqm-wstst