diff --git a/java/api/pom.xml b/java/api/pom.xml index 8972aba8c6ad..19628d6f285b 100644 --- a/java/api/pom.xml +++ b/java/api/pom.xml @@ -1,54 +1,53 @@ - - - org.ray.parent - ray-superpom - 1.0 - - 4.0.0 - - org.ray - ray-api - java api for ray - java api for ray - - - jar - - - - - org.ray - ray-common - 1.0 - - - - - commons-cli - commons-cli - 1.2 - - - - - org.apache.commons - commons-lang3 - 3.4 - - - de.ruedigermoeller - fst - 2.47 - - - org.apache.arrow - arrow-plasma - - - + + + org.ray.parent + ray-superpom + 1.0 + + 4.0.0 + + org.ray + ray-api + java api for ray + java api for ray + + + jar + + + + + org.ray + ray-common + 1.0 + + + + + commons-cli + commons-cli + 1.2 + + + + + org.apache.commons + commons-lang3 + 3.4 + + + de.ruedigermoeller + fst + 2.47 + + + org.apache.arrow + arrow-plasma + + diff --git a/java/api/src/main/java/org/ray/api/Ray.java b/java/api/src/main/java/org/ray/api/Ray.java index 38e14f1b844d..929758fe54b0 100644 --- a/java/api/src/main/java/org/ray/api/Ray.java +++ b/java/api/src/main/java/org/ray/api/Ray.java @@ -7,12 +7,14 @@ import org.ray.util.logger.RayLog; /** - * Ray API + * Ray API. */ public final class Ray extends Rpc { + private static RayApi impl = null; + /** - * initialize the current worker or the single-box cluster + * initialize the current worker or the single-box cluster. */ public static void init() { if (impl == null) { @@ -21,44 +23,44 @@ public static void init() { } /** - * Put obj into object store + * Put obj into object store. */ public static RayObject put(T obj) { return impl.put(obj); } - public static RayObject put(T obj, TM metadata) { + public static RayObject put(T obj, TMT metadata) { return impl.put(obj, metadata); } /** - * Get obj(s) from object store + * Get obj(s) from object store. */ static T get(UniqueID objectId) throws TaskExecutionException { return impl.get(objectId); } - static T getMeta(UniqueID objectId) throws TaskExecutionException { - return impl.getMeta(objectId); - } - static List get(List objectIds) throws TaskExecutionException { return impl.get(objectIds); } + static T getMeta(UniqueID objectId) throws TaskExecutionException { + return impl.getMeta(objectId); + } + static List getMeta(List objectIds) throws TaskExecutionException { return impl.getMeta(objectIds); } /** - * wait until timeout or enough RayObject are ready + * wait until timeout or enough RayObject are ready. * - * @param waitfor wait for who - * @param numReturns how many of ready is enough + * @param waitfor wait for who + * @param numReturns how many of ready is enough * @param timeoutMilliseconds in millisecond */ public static WaitResult wait(RayList waitfor, int numReturns, - int timeoutMilliseconds) { + int timeoutMilliseconds) { return impl.wait(waitfor, numReturns, timeoutMilliseconds); } @@ -77,7 +79,7 @@ public static WaitResult wait(RayObject waitfor, int timeoutMillisecon } /** - * create actor object + * create actor object. */ public static RayActor create(Class cls) { try { @@ -95,23 +97,21 @@ public static RayActor create(Class cls) { } /** - * get underlying runtime + * get underlying runtime. */ static RayApi internal() { return impl; } /** - * whether to use remote lambda + * whether to use remote lambda. */ public static boolean isRemoteLambda() { return impl.isRemoteLambda(); } - private static RayApi impl = null; - /** - * for ray's app's log + * for ray's app's log. */ public static DynamicLog getRappLogger() { return RayLog.rapp; diff --git a/java/api/src/main/java/org/ray/api/RayActor.java b/java/api/src/main/java/org/ray/api/RayActor.java index 6f00a6e781f1..30ea84346e75 100644 --- a/java/api/src/main/java/org/ray/api/RayActor.java +++ b/java/api/src/main/java/org/ray/api/RayActor.java @@ -7,7 +7,7 @@ import org.ray.util.Sha1Digestor; /** - * Ray actor abstraction + * Ray actor abstraction. */ public class RayActor extends RayObject implements Externalizable { @@ -40,7 +40,7 @@ public int increaseTaskCounter() { } /** - * Getter method for property taskCursor + * Getter method for property taskCursor. * * @return property value of taskCursor */ @@ -49,7 +49,7 @@ public UniqueID getTaskCursor() { } /** - * Setter method for property taskCursor + * Setter method for property taskCursor. * * @param taskCursor value to be assigned to property taskCursor */ @@ -57,11 +57,6 @@ public void setTaskCursor(UniqueID taskCursor) { this.taskCursor = taskCursor; } - public UniqueID computeNextActorHandleId() { - byte[] bytes = Sha1Digestor.digest(actorHandleId.id, ++forksNum); - return new UniqueID(bytes); - } - public UniqueID getActorHandleId() { return actorHandleId; } @@ -77,6 +72,11 @@ public void writeExternal(ObjectOutput out) throws IOException { out.writeObject(this.taskCursor); } + public UniqueID computeNextActorHandleId() { + byte[] bytes = Sha1Digestor.digest(actorHandleId.id, ++forksNum); + return new UniqueID(bytes); + } + @Override public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException { diff --git a/java/api/src/main/java/org/ray/api/RayApi.java b/java/api/src/main/java/org/ray/api/RayApi.java index 23c075d11215..6719a48e0ecb 100644 --- a/java/api/src/main/java/org/ray/api/RayApi.java +++ b/java/api/src/main/java/org/ray/api/RayApi.java @@ -7,95 +7,98 @@ import org.ray.util.exception.TaskExecutionException; /** - * Ray runtime abstraction + * Ray runtime abstraction. */ public interface RayApi { /** - * Put obj into object store + * Put obj into object store. * * @return RayObject */ RayObject put(T obj); - RayObject put(T obj, TM metadata); + RayObject put(T obj, TMT metadata); /** - * Get real obj from object store + * Get real obj from object store. */ T get(UniqueID objectId) throws TaskExecutionException; - T getMeta(UniqueID objectId) throws TaskExecutionException; - /** - * Get real objects from object store + * Get real objects from object store. * * @param objectIds list of ids of objects to get */ List get(List objectIds) throws TaskExecutionException; + T getMeta(UniqueID objectId) throws TaskExecutionException; + List getMeta(List objectIds) throws TaskExecutionException; /** - * wait until timeout or enough RayObjects are ready + * wait until timeout or enough RayObjects are ready. * - * @param waitfor wait for who + * @param waitfor wait for who * @param numReturns how many of ready is enough - * @param timeout in millisecond + * @param timeout in millisecond */ WaitResult wait(RayList waitfor, int numReturns, int timeout); /** - * create remote actor + * create remote actor. */ RayActor create(Class cls); /** - * submit a new task by invoking a remote function + * submit a new task by invoking a remote function. * - * @param taskId nil - * @param funcRun the target running function with @RayRemote + * @param taskId nil + * @param funcRun the target running function with @RayRemote * @param returnCount the number of to-be-returned objects from funcRun - * @param args arguments to this funcRun, can be its original form or RayObject + * @param args arguments to this funcRun, can be its original form or RayObject * @return a set of ray objects with their return ids */ RayObjects call(UniqueID taskId, Callable funcRun, int returnCount, Object... args); RayObjects call(UniqueID taskId, Class funcCls, Serializable lambda, int returnCount, - Object... args); + Object... args); /** * In some cases, we would like the return value of a remote function to be splitted into multiple * parts so that they are consumed by multiple further functions separately (potentially on * different machines). We therefore introduce this API so that developers can annotate the - * outputs with a set of labels (usually with Integer or String) + * outputs with a set of labels (usually with Integer or String). * - * @param taskId nil - * @param funcRun the target running function with @RayRemote + * @param taskId nil + * @param funcRun the target running function with @RayRemote * @param returnIds a set of labels to be used by the returned objects - * @param args arguments to this funcRun, can be its original form or RayObject + * @param args arguments to this funcRun, can be its original form or + * RayObject * @return a set of ray objects with their labels and return ids */ - RayMap callWithReturnLabels(UniqueID taskId, Callable funcRun, - Collection returnIds, Object... args); + RayMap callWithReturnLabels(UniqueID taskId, Callable funcRun, + Collection returnIds, Object... args); - RayMap callWithReturnLabels(UniqueID taskId, Class funcCls, - Serializable lambda, Collection returnids, Object... args); + RayMap callWithReturnLabels(UniqueID taskId, Class funcCls, + Serializable lambda, Collection returnids, + Object... args); /** - * a special case for the above RID-based labeling as <0...returnCount - 1> + * a special case for the above RID-based labeling as <0...returnCount - 1>. * - * @param taskId nil - * @param funcRun the target running function with @RayRemote + * @param taskId nil + * @param funcRun the target running function with @RayRemote * @param returnCount the number of to-be-returned objects from funcRun - * @param args arguments to this funcRun, can be its original form or RayObject + * @param args arguments to this funcRun, can be its original form or + * RayObject * @return an array of returned objects with their Unique ids */ RayList callWithReturnIndices(UniqueID taskId, Callable funcRun, Integer returnCount, - Object... args); + Object... args); RayList callWithReturnIndices(UniqueID taskId, Class funcCls, Serializable lambda, - Integer returnCount, Object... args); + Integer returnCount, Object... args); boolean isRemoteLambda(); } diff --git a/java/api/src/main/java/org/ray/api/RayList.java b/java/api/src/main/java/org/ray/api/RayList.java index 4d3d240181fa..5c9d38ca4060 100644 --- a/java/api/src/main/java/org/ray/api/RayList.java +++ b/java/api/src/main/java/org/ray/api/RayList.java @@ -38,15 +38,16 @@ public boolean contains(Object o) { return ids.contains(o); } - @RayDisabled - @Deprecated @Override - public Iterator iterator() { - throw new UnsupportedOperationException(); + public int indexOf(Object o) { + //throw new UnsupportedOperationException(); + return ids.indexOf(o); } - public Iterator> Iterator() { - return ids.iterator(); + @Override + public int lastIndexOf(Object o) { + //throw new UnsupportedOperationException(); + return ids.lastIndexOf(o); } @Override @@ -61,6 +62,30 @@ public T[] toArray(T[] a) { return ids.toArray(a); } + @Override + public E get(int index) { + return ids.get(index).get(); + } + + public List get() { + List objectIds = new ArrayList<>(); + for (RayObject id : ids) { + objectIds.add(id.getId()); + } + return Ray.get(objectIds); + } + + @RayDisabled + @Deprecated + @Override + public E set(int index, E element) { + throw new UnsupportedOperationException(); + } + + public RayObject set(int index, RayObject element) { + return ids.set(index, element); + } + @RayDisabled @Deprecated @Override @@ -68,10 +93,28 @@ public boolean add(E e) { throw new UnsupportedOperationException(); } + @RayDisabled + @Deprecated + @Override + public void add(int index, E element) { + throw new UnsupportedOperationException(); + } + public boolean add(RayObject e) { return ids.add(e); } + public void add(int index, RayObject element) { + ids.add(index, element); + } + + @RayDisabled + @Deprecated + @Override + public E remove(int index) { + throw new UnsupportedOperationException(); + } + @Override public boolean remove(Object o) { //throw new UnsupportedOperationException(); @@ -79,9 +122,9 @@ public boolean remove(Object o) { } @Override - public boolean containsAll(Collection c) { + public void clear() { //throw new UnsupportedOperationException(); - return ids.containsAll(c); + ids.clear(); } @RayDisabled @@ -110,104 +153,61 @@ public boolean retainAll(Collection c) { return ids.retainAll(c); } + @RayDisabled + @Deprecated @Override - public void clear() { - //throw new UnsupportedOperationException(); - ids.clear(); - } - - public List get() { - List objectIds = new ArrayList<>(); - for (RayObject id : ids) { - objectIds.add(id.getId()); - } - return Ray.get(objectIds); - } - - public List getMeta() { - List objectIds = new ArrayList<>(); - for (RayObject id : ids) { - objectIds.add(id.getId()); - } - return Ray.getMeta(objectIds); - } - - @Override - public E get(int index) { - return ids.get(index).get(); - } - - public TM getMeta(int index) { - return ids.get(index).getMeta(); - } - - public RayObject Get(int index) { - return ids.get(index); + public ListIterator listIterator(int index) { + throw new UnsupportedOperationException(); } @RayDisabled @Deprecated @Override - public E set(int index, E element) { + public ListIterator listIterator() { throw new UnsupportedOperationException(); } - public RayObject set(int index, RayObject element) { - return ids.set(index, element); - } - @RayDisabled @Deprecated @Override - public void add(int index, E element) { + public Iterator iterator() { throw new UnsupportedOperationException(); } - public void add(int index, RayObject element) { - ids.add(index, element); - } - @RayDisabled @Deprecated @Override - public E remove(int index) { + public List subList(int fromIndex, int toIndex) { throw new UnsupportedOperationException(); } - public RayObject Remove(int index) { - return ids.remove(index); + public Iterator> Iterator() { + return ids.iterator(); } @Override - public int indexOf(Object o) { + public boolean containsAll(Collection c) { //throw new UnsupportedOperationException(); - return ids.indexOf(o); + return ids.containsAll(c); } - @Override - public int lastIndexOf(Object o) { - //throw new UnsupportedOperationException(); - return ids.lastIndexOf(o); + public List getMeta() { + List objectIds = new ArrayList<>(); + for (RayObject id : ids) { + objectIds.add(id.getId()); + } + return Ray.getMeta(objectIds); } - @RayDisabled - @Deprecated - @Override - public ListIterator listIterator() { - throw new UnsupportedOperationException(); + public TMT getMeta(int index) { + return ids.get(index).getMeta(); } - @RayDisabled - @Deprecated - @Override - public ListIterator listIterator(int index) { - throw new UnsupportedOperationException(); + public RayObject Get(int index) { + return ids.get(index); } - @RayDisabled - @Deprecated - @Override - public List subList(int fromIndex, int toIndex) { - throw new UnsupportedOperationException(); + public RayObject Remove(int index) { + return ids.remove(index); } } diff --git a/java/api/src/main/java/org/ray/api/RayMap.java b/java/api/src/main/java/org/ray/api/RayMap.java index 031aaf7fe3dc..175a45190196 100644 --- a/java/api/src/main/java/org/ray/api/RayMap.java +++ b/java/api/src/main/java/org/ray/api/RayMap.java @@ -32,15 +32,8 @@ public boolean isEmpty() { } @Override - public boolean containsKey(Object key) { - //throw new UnsupportedOperationException(); - return ids.containsKey(key); - } - - @Override - public boolean containsValue(Object value) { - //throw new UnsupportedOperationException(); - return ids.containsValue(value); + public V get(Object key) { + return ids.get(key).get(); } // TODO: try to use multiple get @@ -52,26 +45,10 @@ public Map get() { return objs; } - public Map getMeta() { - Map metas = new HashMap<>(); - for (Map.Entry> id : ids.entrySet()) { - TM meta = id.getValue().getMeta(); - metas.put(id.getKey(), meta); - } - return metas; - } - @Override - public V get(Object key) { - return ids.get(key).get(); - } - - public TM getMeta(K key) { - return ids.get(key).getMeta(); - } - - public RayObject Get(K key) { - return ids.get(key); + public boolean containsKey(Object key) { + //throw new UnsupportedOperationException(); + return ids.containsKey(key); } @RayDisabled @@ -88,18 +65,14 @@ public RayObject put(K key, RayObject value) { @RayDisabled @Deprecated @Override - public V remove(Object key) { + public void putAll(Map m) { throw new UnsupportedOperationException(); } - public RayObject Remove(K key) { - return ids.remove(key); - } - @RayDisabled @Deprecated @Override - public void putAll(Map m) { + public V remove(Object key) { throw new UnsupportedOperationException(); } @@ -109,6 +82,12 @@ public void clear() { ids.clear(); } + @Override + public boolean containsValue(Object value) { + //throw new UnsupportedOperationException(); + return ids.containsValue(value); + } + @Override public Set keySet() { return ids.keySet(); @@ -121,10 +100,6 @@ public Collection values() { throw new UnsupportedOperationException(); } - public Collection> Values() { - return ids.values(); - } - @RayDisabled @Deprecated @Override @@ -132,6 +107,31 @@ public Set> entrySet() { throw new UnsupportedOperationException(); } + public Map getMeta() { + Map metas = new HashMap<>(); + for (Map.Entry> id : ids.entrySet()) { + TMT meta = id.getValue().getMeta(); + metas.put(id.getKey(), meta); + } + return metas; + } + + public TMT getMeta(K key) { + return ids.get(key).getMeta(); + } + + public RayObject Get(K key) { + return ids.get(key); + } + + public RayObject Remove(K key) { + return ids.remove(key); + } + + public Collection> Values() { + return ids.values(); + } + public Set>> EntrySet() { return ids.entrySet(); } diff --git a/java/api/src/main/java/org/ray/api/RayObject.java b/java/api/src/main/java/org/ray/api/RayObject.java index 2493e19d023a..d5695dc4e349 100644 --- a/java/api/src/main/java/org/ray/api/RayObject.java +++ b/java/api/src/main/java/org/ray/api/RayObject.java @@ -24,7 +24,7 @@ public T get() throws TaskExecutionException { return Ray.get(id); } - public TM getMeta() throws TaskExecutionException { + public TMT getMeta() throws TaskExecutionException { return Ray.getMeta(id); } diff --git a/java/api/src/main/java/org/ray/api/RayObjects.java b/java/api/src/main/java/org/ray/api/RayObjects.java index d56b2673e756..4dd492dc13fb 100644 --- a/java/api/src/main/java/org/ray/api/RayObjects.java +++ b/java/api/src/main/java/org/ray/api/RayObjects.java @@ -3,7 +3,7 @@ import org.apache.commons.lang3.ArrayUtils; /** - * Real object or ray future proxy for multiple returns + * Real object or ray future proxy for multiple returns. */ public class RayObjects { diff --git a/java/api/src/main/java/org/ray/api/RayRemote.java b/java/api/src/main/java/org/ray/api/RayRemote.java index 23c5399e9ed5..d3690809d4cd 100644 --- a/java/api/src/main/java/org/ray/api/RayRemote.java +++ b/java/api/src/main/java/org/ray/api/RayRemote.java @@ -6,15 +6,15 @@ import java.lang.annotation.Target; /** - * a ray remote function or class (as an actor) + * a ray remote function or class (as an actor). */ @Target({ElementType.METHOD, ElementType.TYPE}) @Retention(RetentionPolicy.RUNTIME) public @interface RayRemote { /** - * whether to use external I/O pool to execute the function + * whether to use external I/O pool to execute the function. */ - boolean externalIO() default false; + boolean externalIo() default false; } diff --git a/java/api/src/main/java/org/ray/api/RayService.java b/java/api/src/main/java/org/ray/api/RayService.java index 3dfefcf2915b..0012bee64445 100644 --- a/java/api/src/main/java/org/ray/api/RayService.java +++ b/java/api/src/main/java/org/ray/api/RayService.java @@ -1,8 +1,7 @@ package org.ray.api; /** - * a RPC service that represent the data processing services implemented in Ray - * + * a RPC service that represent the data processing services implemented in Ray. * it is programmed in other services but will be shipped and executed on ray machines at runtime */ public @interface RayService { diff --git a/java/api/src/main/java/org/ray/api/UniqueID.java b/java/api/src/main/java/org/ray/api/UniqueID.java index 97e2a298882a..08940b99181c 100644 --- a/java/api/src/main/java/org/ray/api/UniqueID.java +++ b/java/api/src/main/java/org/ray/api/UniqueID.java @@ -1,6 +1,5 @@ package org.ray.api; - import java.io.Serializable; import java.nio.ByteBuffer; import java.nio.IntBuffer; @@ -12,10 +11,9 @@ */ public class UniqueID implements Serializable { - private static final long serialVersionUID = 8588849129675565761L; - public static final int LENGTH = 20; - + public static final UniqueID nil = genNil(); + private static final long serialVersionUID = 8588849129675565761L; byte[] id; public UniqueID(byte[] id) { @@ -42,23 +40,26 @@ public UniqueID(String optionValue) { } } - @Override - public String toString() { - String s = ""; - String hex = "0123456789abcdef"; - for (int i = 0; i < LENGTH; i++) { - int val = id[i] & 0xff; - s += hex.charAt(val >> 4); - s += hex.charAt(val & 0xf); + public static UniqueID genNil() { + byte[] b = new byte[LENGTH]; + for (int i = 0; i < b.length; i++) { + b[i] = (byte) 0xFF; } - return s; + + return new UniqueID(b); + } + + public static UniqueID randomId() { + byte[] b = new byte[LENGTH]; + new Random().nextBytes(b); + return new UniqueID(b); } public byte[] getBytes() { return id; } - public ByteBuffer ToByteBuffer() { + public ByteBuffer toByteBuffer() { return ByteBuffer.wrap(id); } @@ -91,6 +92,18 @@ public boolean equals(Object obj) { return Arrays.equals(id, r.id); } + @Override + public String toString() { + String s = ""; + String hex = "0123456789abcdef"; + for (int i = 0; i < LENGTH; i++) { + int val = id[i] & 0xff; + s += hex.charAt(val >> 4); + s += hex.charAt(val & 0xf); + } + return s; + } + public boolean isNil() { for (byte b : id) { if (b != (byte) 0xFF) { @@ -99,21 +112,4 @@ public boolean isNil() { } return true; } - - public static final UniqueID nil = genNil(); - - public static UniqueID genNil() { - byte[] b = new byte[LENGTH]; - for (int i = 0; i < b.length; i++) { - b[i] = (byte) 0xFF; - } - - return new UniqueID(b); - } - - public static UniqueID randomID() { - byte[] b = new byte[LENGTH]; - new Random().nextBytes(b); - return new UniqueID(b); - } } diff --git a/java/api/src/main/java/org/ray/api/funcs/RayFunc_0_1.java b/java/api/src/main/java/org/ray/api/funcs/RayFunc_0_1.java index 123e9023fa23..2ba5d26345d9 100644 --- a/java/api/src/main/java/org/ray/api/funcs/RayFunc_0_1.java +++ b/java/api/src/main/java/org/ray/api/funcs/RayFunc_0_1.java @@ -6,8 +6,6 @@ @FunctionalInterface public interface RayFunc_0_1 extends RayFunc { - R0 apply() throws Throwable; - static R0 execute(Object[] args) throws Throwable { String name = (String) args[args.length - 2]; assert (name.equals(RayFunc_0_1.class.getName())); @@ -16,4 +14,6 @@ static R0 execute(Object[] args) throws Throwable { return f.apply(); } + R0 apply() throws Throwable; + } diff --git a/java/api/src/main/java/org/ray/api/funcs/RayFunc_0_2.java b/java/api/src/main/java/org/ray/api/funcs/RayFunc_0_2.java index 65fb65a198a7..51ca460b72fc 100644 --- a/java/api/src/main/java/org/ray/api/funcs/RayFunc_0_2.java +++ b/java/api/src/main/java/org/ray/api/funcs/RayFunc_0_2.java @@ -7,8 +7,6 @@ @FunctionalInterface public interface RayFunc_0_2 extends RayFunc { - MultipleReturns2 apply() throws Throwable; - static MultipleReturns2 execute(Object[] args) throws Throwable { String name = (String) args[args.length - 2]; assert (name.equals(RayFunc_0_2.class.getName())); @@ -17,4 +15,6 @@ static MultipleReturns2 execute(Object[] args) throws Throwable return f.apply(); } + MultipleReturns2 apply() throws Throwable; + } diff --git a/java/api/src/main/java/org/ray/api/funcs/RayFunc_0_3.java b/java/api/src/main/java/org/ray/api/funcs/RayFunc_0_3.java index e2b43e9013a0..27468941c740 100644 --- a/java/api/src/main/java/org/ray/api/funcs/RayFunc_0_3.java +++ b/java/api/src/main/java/org/ray/api/funcs/RayFunc_0_3.java @@ -7,8 +7,6 @@ @FunctionalInterface public interface RayFunc_0_3 extends RayFunc { - MultipleReturns3 apply() throws Throwable; - static MultipleReturns3 execute(Object[] args) throws Throwable { String name = (String) args[args.length - 2]; assert (name.equals(RayFunc_0_3.class.getName())); @@ -17,4 +15,6 @@ static MultipleReturns3 execute(Object[] args) throws T return f.apply(); } + MultipleReturns3 apply() throws Throwable; + } diff --git a/java/api/src/main/java/org/ray/api/funcs/RayFunc_0_4.java b/java/api/src/main/java/org/ray/api/funcs/RayFunc_0_4.java index 60c0691653d0..2554478ec0c5 100644 --- a/java/api/src/main/java/org/ray/api/funcs/RayFunc_0_4.java +++ b/java/api/src/main/java/org/ray/api/funcs/RayFunc_0_4.java @@ -7,8 +7,6 @@ @FunctionalInterface public interface RayFunc_0_4 extends RayFunc { - MultipleReturns4 apply() throws Throwable; - static MultipleReturns4 execute(Object[] args) throws Throwable { String name = (String) args[args.length - 2]; assert (name.equals(RayFunc_0_4.class.getName())); @@ -17,4 +15,6 @@ static MultipleReturns4 execute(Object[] args) return f.apply(); } + MultipleReturns4 apply() throws Throwable; + } diff --git a/java/api/src/main/java/org/ray/api/funcs/RayFunc_0_n.java b/java/api/src/main/java/org/ray/api/funcs/RayFunc_0_n.java index d651fb429eef..11af4498a4e1 100644 --- a/java/api/src/main/java/org/ray/api/funcs/RayFunc_0_n.java +++ b/java/api/src/main/java/org/ray/api/funcs/RayFunc_0_n.java @@ -6,16 +6,16 @@ import org.ray.api.internal.RayFunc; @FunctionalInterface -public interface RayFunc_0_n extends RayFunc { +public interface RayFunc_0_n extends RayFunc { - Map apply(Collection returnids) throws Throwable; - - static Map execute(Object[] args) throws Throwable { + static Map execute(Object[] args) throws Throwable { String name = (String) args[args.length - 2]; assert (name.equals(RayFunc_0_n.class.getName())); byte[] funcBytes = (byte[]) args[args.length - 1]; - RayFunc_0_n f = SerializationUtils.deserialize(funcBytes); - return f.apply((Collection) args[0]); + RayFunc_0_n f = SerializationUtils.deserialize(funcBytes); + return f.apply((Collection) args[0]); } + Map apply(Collection returnids) throws Throwable; + } diff --git a/java/api/src/main/java/org/ray/api/funcs/RayFunc_0_n_list.java b/java/api/src/main/java/org/ray/api/funcs/RayFunc_0_n_list.java index 1d2a008dea2a..2ec6e2acdc60 100644 --- a/java/api/src/main/java/org/ray/api/funcs/RayFunc_0_n_list.java +++ b/java/api/src/main/java/org/ray/api/funcs/RayFunc_0_n_list.java @@ -7,8 +7,6 @@ @FunctionalInterface public interface RayFunc_0_n_list extends RayFunc { - List apply() throws Throwable; - static List execute(Object[] args) throws Throwable { String name = (String) args[args.length - 2]; assert (name.equals(RayFunc_0_n_list.class.getName())); @@ -17,4 +15,6 @@ static List execute(Object[] args) throws Throwable { return f.apply(); } + List apply() throws Throwable; + } diff --git a/java/api/src/main/java/org/ray/api/funcs/RayFunc_1_1.java b/java/api/src/main/java/org/ray/api/funcs/RayFunc_1_1.java index 866c98f7ba0b..3341253191ec 100644 --- a/java/api/src/main/java/org/ray/api/funcs/RayFunc_1_1.java +++ b/java/api/src/main/java/org/ray/api/funcs/RayFunc_1_1.java @@ -6,8 +6,6 @@ @FunctionalInterface public interface RayFunc_1_1 extends RayFunc { - R0 apply(T0 t0) throws Throwable; - static R0 execute(Object[] args) throws Throwable { String name = (String) args[args.length - 2]; assert (name.equals(RayFunc_1_1.class.getName())); @@ -16,4 +14,6 @@ static R0 execute(Object[] args) throws Throwable { return f.apply((T0) args[0]); } + R0 apply(T0 t0) throws Throwable; + } diff --git a/java/api/src/main/java/org/ray/api/funcs/RayFunc_1_2.java b/java/api/src/main/java/org/ray/api/funcs/RayFunc_1_2.java index 6ec1117c3f63..cc561c87f6a7 100644 --- a/java/api/src/main/java/org/ray/api/funcs/RayFunc_1_2.java +++ b/java/api/src/main/java/org/ray/api/funcs/RayFunc_1_2.java @@ -7,8 +7,6 @@ @FunctionalInterface public interface RayFunc_1_2 extends RayFunc { - MultipleReturns2 apply(T0 t0) throws Throwable; - static MultipleReturns2 execute(Object[] args) throws Throwable { String name = (String) args[args.length - 2]; assert (name.equals(RayFunc_1_2.class.getName())); @@ -17,4 +15,6 @@ static MultipleReturns2 execute(Object[] args) throws Throw return f.apply((T0) args[0]); } + MultipleReturns2 apply(T0 t0) throws Throwable; + } diff --git a/java/api/src/main/java/org/ray/api/funcs/RayFunc_1_3.java b/java/api/src/main/java/org/ray/api/funcs/RayFunc_1_3.java index c7e78f859536..03846037908c 100644 --- a/java/api/src/main/java/org/ray/api/funcs/RayFunc_1_3.java +++ b/java/api/src/main/java/org/ray/api/funcs/RayFunc_1_3.java @@ -7,8 +7,6 @@ @FunctionalInterface public interface RayFunc_1_3 extends RayFunc { - MultipleReturns3 apply(T0 t0) throws Throwable; - static MultipleReturns3 execute(Object[] args) throws Throwable { String name = (String) args[args.length - 2]; assert (name.equals(RayFunc_1_3.class.getName())); @@ -17,4 +15,6 @@ static MultipleReturns3 execute(Object[] args) thro return f.apply((T0) args[0]); } + MultipleReturns3 apply(T0 t0) throws Throwable; + } diff --git a/java/api/src/main/java/org/ray/api/funcs/RayFunc_1_4.java b/java/api/src/main/java/org/ray/api/funcs/RayFunc_1_4.java index 29871e79a901..240f1500bb33 100644 --- a/java/api/src/main/java/org/ray/api/funcs/RayFunc_1_4.java +++ b/java/api/src/main/java/org/ray/api/funcs/RayFunc_1_4.java @@ -7,8 +7,6 @@ @FunctionalInterface public interface RayFunc_1_4 extends RayFunc { - MultipleReturns4 apply(T0 t0) throws Throwable; - static MultipleReturns4 execute(Object[] args) throws Throwable { String name = (String) args[args.length - 2]; @@ -18,4 +16,6 @@ static MultipleReturns4 execute(Object[] ar return f.apply((T0) args[0]); } + MultipleReturns4 apply(T0 t0) throws Throwable; + } diff --git a/java/api/src/main/java/org/ray/api/funcs/RayFunc_1_n.java b/java/api/src/main/java/org/ray/api/funcs/RayFunc_1_n.java index d057486a9f3d..9fccd780f7a4 100644 --- a/java/api/src/main/java/org/ray/api/funcs/RayFunc_1_n.java +++ b/java/api/src/main/java/org/ray/api/funcs/RayFunc_1_n.java @@ -6,16 +6,16 @@ import org.ray.api.internal.RayFunc; @FunctionalInterface -public interface RayFunc_1_n extends RayFunc { +public interface RayFunc_1_n extends RayFunc { - Map apply(Collection returnids, T0 t0) throws Throwable; - - static Map execute(Object[] args) throws Throwable { + static Map execute(Object[] args) throws Throwable { String name = (String) args[args.length - 2]; assert (name.equals(RayFunc_1_n.class.getName())); byte[] funcBytes = (byte[]) args[args.length - 1]; - RayFunc_1_n f = SerializationUtils.deserialize(funcBytes); - return f.apply((Collection) args[0], (T0) args[1]); + RayFunc_1_n f = SerializationUtils.deserialize(funcBytes); + return f.apply((Collection) args[0], (T0) args[1]); } + Map apply(Collection returnids, T0 t0) throws Throwable; + } diff --git a/java/api/src/main/java/org/ray/api/funcs/RayFunc_1_n_list.java b/java/api/src/main/java/org/ray/api/funcs/RayFunc_1_n_list.java index 8853c80f8f7e..cd85e1724a89 100644 --- a/java/api/src/main/java/org/ray/api/funcs/RayFunc_1_n_list.java +++ b/java/api/src/main/java/org/ray/api/funcs/RayFunc_1_n_list.java @@ -7,8 +7,6 @@ @FunctionalInterface public interface RayFunc_1_n_list extends RayFunc { - List apply(T0 t0) throws Throwable; - static List execute(Object[] args) throws Throwable { String name = (String) args[args.length - 2]; assert (name.equals(RayFunc_1_n_list.class.getName())); @@ -17,4 +15,6 @@ static List execute(Object[] args) throws Throwable { return f.apply((T0) args[0]); } + List apply(T0 t0) throws Throwable; + } diff --git a/java/api/src/main/java/org/ray/api/funcs/RayFunc_2_1.java b/java/api/src/main/java/org/ray/api/funcs/RayFunc_2_1.java index 0c9ed3fc3a08..6f4698dd8a5d 100644 --- a/java/api/src/main/java/org/ray/api/funcs/RayFunc_2_1.java +++ b/java/api/src/main/java/org/ray/api/funcs/RayFunc_2_1.java @@ -6,8 +6,6 @@ @FunctionalInterface public interface RayFunc_2_1 extends RayFunc { - R0 apply(T0 t0, T1 t1) throws Throwable; - static R0 execute(Object[] args) throws Throwable { String name = (String) args[args.length - 2]; assert (name.equals(RayFunc_2_1.class.getName())); @@ -16,4 +14,6 @@ static R0 execute(Object[] args) throws Throwable { return f.apply((T0) args[0], (T1) args[1]); } + R0 apply(T0 t0, T1 t1) throws Throwable; + } diff --git a/java/api/src/main/java/org/ray/api/funcs/RayFunc_2_2.java b/java/api/src/main/java/org/ray/api/funcs/RayFunc_2_2.java index c5faa49bd307..88030e8de804 100644 --- a/java/api/src/main/java/org/ray/api/funcs/RayFunc_2_2.java +++ b/java/api/src/main/java/org/ray/api/funcs/RayFunc_2_2.java @@ -7,8 +7,6 @@ @FunctionalInterface public interface RayFunc_2_2 extends RayFunc { - MultipleReturns2 apply(T0 t0, T1 t1) throws Throwable; - static MultipleReturns2 execute(Object[] args) throws Throwable { String name = (String) args[args.length - 2]; assert (name.equals(RayFunc_2_2.class.getName())); @@ -17,4 +15,6 @@ static MultipleReturns2 execute(Object[] args) throws T return f.apply((T0) args[0], (T1) args[1]); } + MultipleReturns2 apply(T0 t0, T1 t1) throws Throwable; + } diff --git a/java/api/src/main/java/org/ray/api/funcs/RayFunc_2_3.java b/java/api/src/main/java/org/ray/api/funcs/RayFunc_2_3.java index 49847b1c6966..3b33adffa8c1 100644 --- a/java/api/src/main/java/org/ray/api/funcs/RayFunc_2_3.java +++ b/java/api/src/main/java/org/ray/api/funcs/RayFunc_2_3.java @@ -7,8 +7,6 @@ @FunctionalInterface public interface RayFunc_2_3 extends RayFunc { - MultipleReturns3 apply(T0 t0, T1 t1) throws Throwable; - static MultipleReturns3 execute(Object[] args) throws Throwable { String name = (String) args[args.length - 2]; assert (name.equals(RayFunc_2_3.class.getName())); @@ -17,4 +15,6 @@ static MultipleReturns3 execute(Object[] args) return f.apply((T0) args[0], (T1) args[1]); } + MultipleReturns3 apply(T0 t0, T1 t1) throws Throwable; + } diff --git a/java/api/src/main/java/org/ray/api/funcs/RayFunc_2_4.java b/java/api/src/main/java/org/ray/api/funcs/RayFunc_2_4.java index bc40c02c3340..5ec4ff7f64aa 100644 --- a/java/api/src/main/java/org/ray/api/funcs/RayFunc_2_4.java +++ b/java/api/src/main/java/org/ray/api/funcs/RayFunc_2_4.java @@ -7,8 +7,6 @@ @FunctionalInterface public interface RayFunc_2_4 extends RayFunc { - MultipleReturns4 apply(T0 t0, T1 t1) throws Throwable; - static MultipleReturns4 execute(Object[] args) throws Throwable { String name = (String) args[args.length - 2]; @@ -18,4 +16,6 @@ static MultipleReturns4 execute(Object[ return f.apply((T0) args[0], (T1) args[1]); } + MultipleReturns4 apply(T0 t0, T1 t1) throws Throwable; + } diff --git a/java/api/src/main/java/org/ray/api/funcs/RayFunc_2_n.java b/java/api/src/main/java/org/ray/api/funcs/RayFunc_2_n.java index 1961ba67e360..4152591783a5 100644 --- a/java/api/src/main/java/org/ray/api/funcs/RayFunc_2_n.java +++ b/java/api/src/main/java/org/ray/api/funcs/RayFunc_2_n.java @@ -6,16 +6,16 @@ import org.ray.api.internal.RayFunc; @FunctionalInterface -public interface RayFunc_2_n extends RayFunc { +public interface RayFunc_2_n extends RayFunc { - Map apply(Collection returnids, T0 t0, T1 t1) throws Throwable; - - static Map execute(Object[] args) throws Throwable { + static Map execute(Object[] args) throws Throwable { String name = (String) args[args.length - 2]; assert (name.equals(RayFunc_2_n.class.getName())); byte[] funcBytes = (byte[]) args[args.length - 1]; - RayFunc_2_n f = SerializationUtils.deserialize(funcBytes); - return f.apply((Collection) args[0], (T0) args[1], (T1) args[2]); + RayFunc_2_n f = SerializationUtils.deserialize(funcBytes); + return f.apply((Collection) args[0], (T0) args[1], (T1) args[2]); } + Map apply(Collection returnids, T0 t0, T1 t1) throws Throwable; + } diff --git a/java/api/src/main/java/org/ray/api/funcs/RayFunc_2_n_list.java b/java/api/src/main/java/org/ray/api/funcs/RayFunc_2_n_list.java index 4c0591a471e1..3b88ce2e06c0 100644 --- a/java/api/src/main/java/org/ray/api/funcs/RayFunc_2_n_list.java +++ b/java/api/src/main/java/org/ray/api/funcs/RayFunc_2_n_list.java @@ -7,8 +7,6 @@ @FunctionalInterface public interface RayFunc_2_n_list extends RayFunc { - List apply(T0 t0, T1 t1) throws Throwable; - static List execute(Object[] args) throws Throwable { String name = (String) args[args.length - 2]; assert (name.equals(RayFunc_2_n_list.class.getName())); @@ -17,4 +15,6 @@ static List execute(Object[] args) throws Throwable { return f.apply((T0) args[0], (T1) args[1]); } + List apply(T0 t0, T1 t1) throws Throwable; + } diff --git a/java/api/src/main/java/org/ray/api/funcs/RayFunc_3_1.java b/java/api/src/main/java/org/ray/api/funcs/RayFunc_3_1.java index 2751f9eaa65d..30530544a0d1 100644 --- a/java/api/src/main/java/org/ray/api/funcs/RayFunc_3_1.java +++ b/java/api/src/main/java/org/ray/api/funcs/RayFunc_3_1.java @@ -6,8 +6,6 @@ @FunctionalInterface public interface RayFunc_3_1 extends RayFunc { - R0 apply(T0 t0, T1 t1, T2 t2) throws Throwable; - static R0 execute(Object[] args) throws Throwable { String name = (String) args[args.length - 2]; assert (name.equals(RayFunc_3_1.class.getName())); @@ -16,4 +14,6 @@ static R0 execute(Object[] args) throws Throwable { return f.apply((T0) args[0], (T1) args[1], (T2) args[2]); } + R0 apply(T0 t0, T1 t1, T2 t2) throws Throwable; + } diff --git a/java/api/src/main/java/org/ray/api/funcs/RayFunc_3_2.java b/java/api/src/main/java/org/ray/api/funcs/RayFunc_3_2.java index 880b89d47286..261c983ebf44 100644 --- a/java/api/src/main/java/org/ray/api/funcs/RayFunc_3_2.java +++ b/java/api/src/main/java/org/ray/api/funcs/RayFunc_3_2.java @@ -7,8 +7,6 @@ @FunctionalInterface public interface RayFunc_3_2 extends RayFunc { - MultipleReturns2 apply(T0 t0, T1 t1, T2 t2) throws Throwable; - static MultipleReturns2 execute(Object[] args) throws Throwable { String name = (String) args[args.length - 2]; assert (name.equals(RayFunc_3_2.class.getName())); @@ -17,4 +15,6 @@ static MultipleReturns2 execute(Object[] args) thro return f.apply((T0) args[0], (T1) args[1], (T2) args[2]); } + MultipleReturns2 apply(T0 t0, T1 t1, T2 t2) throws Throwable; + } diff --git a/java/api/src/main/java/org/ray/api/funcs/RayFunc_3_3.java b/java/api/src/main/java/org/ray/api/funcs/RayFunc_3_3.java index 97ec6af1e15f..68ff29978488 100644 --- a/java/api/src/main/java/org/ray/api/funcs/RayFunc_3_3.java +++ b/java/api/src/main/java/org/ray/api/funcs/RayFunc_3_3.java @@ -7,8 +7,6 @@ @FunctionalInterface public interface RayFunc_3_3 extends RayFunc { - MultipleReturns3 apply(T0 t0, T1 t1, T2 t2) throws Throwable; - static MultipleReturns3 execute(Object[] args) throws Throwable { String name = (String) args[args.length - 2]; @@ -18,4 +16,6 @@ static MultipleReturns3 execute(Object[] ar return f.apply((T0) args[0], (T1) args[1], (T2) args[2]); } + MultipleReturns3 apply(T0 t0, T1 t1, T2 t2) throws Throwable; + } diff --git a/java/api/src/main/java/org/ray/api/funcs/RayFunc_3_4.java b/java/api/src/main/java/org/ray/api/funcs/RayFunc_3_4.java index 977aedca24df..823fb8f5a69a 100644 --- a/java/api/src/main/java/org/ray/api/funcs/RayFunc_3_4.java +++ b/java/api/src/main/java/org/ray/api/funcs/RayFunc_3_4.java @@ -7,8 +7,6 @@ @FunctionalInterface public interface RayFunc_3_4 extends RayFunc { - MultipleReturns4 apply(T0 t0, T1 t1, T2 t2) throws Throwable; - static MultipleReturns4 execute(Object[] args) throws Throwable { String name = (String) args[args.length - 2]; @@ -18,4 +16,6 @@ static MultipleReturns4 execute(Obj return f.apply((T0) args[0], (T1) args[1], (T2) args[2]); } + MultipleReturns4 apply(T0 t0, T1 t1, T2 t2) throws Throwable; + } diff --git a/java/api/src/main/java/org/ray/api/funcs/RayFunc_3_n.java b/java/api/src/main/java/org/ray/api/funcs/RayFunc_3_n.java index a79edea3bf8b..b2dfadb19987 100644 --- a/java/api/src/main/java/org/ray/api/funcs/RayFunc_3_n.java +++ b/java/api/src/main/java/org/ray/api/funcs/RayFunc_3_n.java @@ -6,16 +6,16 @@ import org.ray.api.internal.RayFunc; @FunctionalInterface -public interface RayFunc_3_n extends RayFunc { +public interface RayFunc_3_n extends RayFunc { - Map apply(Collection returnids, T0 t0, T1 t1, T2 t2) throws Throwable; - - static Map execute(Object[] args) throws Throwable { + static Map execute(Object[] args) throws Throwable { String name = (String) args[args.length - 2]; assert (name.equals(RayFunc_3_n.class.getName())); byte[] funcBytes = (byte[]) args[args.length - 1]; - RayFunc_3_n f = SerializationUtils.deserialize(funcBytes); - return f.apply((Collection) args[0], (T0) args[1], (T1) args[2], (T2) args[3]); + RayFunc_3_n f = SerializationUtils.deserialize(funcBytes); + return f.apply((Collection) args[0], (T0) args[1], (T1) args[2], (T2) args[3]); } + Map apply(Collection returnids, T0 t0, T1 t1, T2 t2) throws Throwable; + } diff --git a/java/api/src/main/java/org/ray/api/funcs/RayFunc_3_n_list.java b/java/api/src/main/java/org/ray/api/funcs/RayFunc_3_n_list.java index de0976a67c46..cc85951f61e4 100644 --- a/java/api/src/main/java/org/ray/api/funcs/RayFunc_3_n_list.java +++ b/java/api/src/main/java/org/ray/api/funcs/RayFunc_3_n_list.java @@ -7,8 +7,6 @@ @FunctionalInterface public interface RayFunc_3_n_list extends RayFunc { - List apply(T0 t0, T1 t1, T2 t2) throws Throwable; - static List execute(Object[] args) throws Throwable { String name = (String) args[args.length - 2]; assert (name.equals(RayFunc_3_n_list.class.getName())); @@ -17,4 +15,6 @@ static List execute(Object[] args) throws Throwable { return f.apply((T0) args[0], (T1) args[1], (T2) args[2]); } + List apply(T0 t0, T1 t1, T2 t2) throws Throwable; + } diff --git a/java/api/src/main/java/org/ray/api/funcs/RayFunc_4_1.java b/java/api/src/main/java/org/ray/api/funcs/RayFunc_4_1.java index 352df12eadb6..fe42f7b00afb 100644 --- a/java/api/src/main/java/org/ray/api/funcs/RayFunc_4_1.java +++ b/java/api/src/main/java/org/ray/api/funcs/RayFunc_4_1.java @@ -6,8 +6,6 @@ @FunctionalInterface public interface RayFunc_4_1 extends RayFunc { - R0 apply(T0 t0, T1 t1, T2 t2, T3 t3) throws Throwable; - static R0 execute(Object[] args) throws Throwable { String name = (String) args[args.length - 2]; assert (name.equals(RayFunc_4_1.class.getName())); @@ -16,4 +14,6 @@ static R0 execute(Object[] args) throws Throwable { return f.apply((T0) args[0], (T1) args[1], (T2) args[2], (T3) args[3]); } + R0 apply(T0 t0, T1 t1, T2 t2, T3 t3) throws Throwable; + } diff --git a/java/api/src/main/java/org/ray/api/funcs/RayFunc_4_2.java b/java/api/src/main/java/org/ray/api/funcs/RayFunc_4_2.java index cc655e6489c0..7d2b3fbd4d26 100644 --- a/java/api/src/main/java/org/ray/api/funcs/RayFunc_4_2.java +++ b/java/api/src/main/java/org/ray/api/funcs/RayFunc_4_2.java @@ -7,8 +7,6 @@ @FunctionalInterface public interface RayFunc_4_2 extends RayFunc { - MultipleReturns2 apply(T0 t0, T1 t1, T2 t2, T3 t3) throws Throwable; - static MultipleReturns2 execute(Object[] args) throws Throwable { String name = (String) args[args.length - 2]; assert (name.equals(RayFunc_4_2.class.getName())); @@ -17,4 +15,6 @@ static MultipleReturns2 execute(Object[] args) return f.apply((T0) args[0], (T1) args[1], (T2) args[2], (T3) args[3]); } + MultipleReturns2 apply(T0 t0, T1 t1, T2 t2, T3 t3) throws Throwable; + } diff --git a/java/api/src/main/java/org/ray/api/funcs/RayFunc_4_3.java b/java/api/src/main/java/org/ray/api/funcs/RayFunc_4_3.java index c161c5aae9e0..3b61b1b7c9d1 100644 --- a/java/api/src/main/java/org/ray/api/funcs/RayFunc_4_3.java +++ b/java/api/src/main/java/org/ray/api/funcs/RayFunc_4_3.java @@ -7,8 +7,6 @@ @FunctionalInterface public interface RayFunc_4_3 extends RayFunc { - MultipleReturns3 apply(T0 t0, T1 t1, T2 t2, T3 t3) throws Throwable; - static MultipleReturns3 execute(Object[] args) throws Throwable { String name = (String) args[args.length - 2]; @@ -18,4 +16,6 @@ static MultipleReturns3 execute(Object[ return f.apply((T0) args[0], (T1) args[1], (T2) args[2], (T3) args[3]); } + MultipleReturns3 apply(T0 t0, T1 t1, T2 t2, T3 t3) throws Throwable; + } diff --git a/java/api/src/main/java/org/ray/api/funcs/RayFunc_4_4.java b/java/api/src/main/java/org/ray/api/funcs/RayFunc_4_4.java index ec9183835714..1ad160e80379 100644 --- a/java/api/src/main/java/org/ray/api/funcs/RayFunc_4_4.java +++ b/java/api/src/main/java/org/ray/api/funcs/RayFunc_4_4.java @@ -7,8 +7,6 @@ @FunctionalInterface public interface RayFunc_4_4 extends RayFunc { - MultipleReturns4 apply(T0 t0, T1 t1, T2 t2, T3 t3) throws Throwable; - static MultipleReturns4 execute(Object[] args) throws Throwable { String name = (String) args[args.length - 2]; @@ -18,4 +16,6 @@ static MultipleReturns4 execute return f.apply((T0) args[0], (T1) args[1], (T2) args[2], (T3) args[3]); } + MultipleReturns4 apply(T0 t0, T1 t1, T2 t2, T3 t3) throws Throwable; + } diff --git a/java/api/src/main/java/org/ray/api/funcs/RayFunc_4_n.java b/java/api/src/main/java/org/ray/api/funcs/RayFunc_4_n.java index 9ffc8c55ee6c..19c3fba6fe64 100644 --- a/java/api/src/main/java/org/ray/api/funcs/RayFunc_4_n.java +++ b/java/api/src/main/java/org/ray/api/funcs/RayFunc_4_n.java @@ -6,17 +6,17 @@ import org.ray.api.internal.RayFunc; @FunctionalInterface -public interface RayFunc_4_n extends RayFunc { +public interface RayFunc_4_n extends RayFunc { - Map apply(Collection returnids, T0 t0, T1 t1, T2 t2, T3 t3) throws Throwable; - - static Map execute(Object[] args) throws Throwable { + static Map execute(Object[] args) throws Throwable { String name = (String) args[args.length - 2]; assert (name.equals(RayFunc_4_n.class.getName())); byte[] funcBytes = (byte[]) args[args.length - 1]; - RayFunc_4_n f = SerializationUtils.deserialize(funcBytes); + RayFunc_4_n f = SerializationUtils.deserialize(funcBytes); return f - .apply((Collection) args[0], (T0) args[1], (T1) args[2], (T2) args[3], (T3) args[4]); + .apply((Collection) args[0], (T0) args[1], (T1) args[2], (T2) args[3], (T3) args[4]); } + Map apply(Collection returnids, T0 t0, T1 t1, T2 t2, T3 t3) throws Throwable; + } diff --git a/java/api/src/main/java/org/ray/api/funcs/RayFunc_4_n_list.java b/java/api/src/main/java/org/ray/api/funcs/RayFunc_4_n_list.java index 299411babfcc..4f9acf8b1350 100644 --- a/java/api/src/main/java/org/ray/api/funcs/RayFunc_4_n_list.java +++ b/java/api/src/main/java/org/ray/api/funcs/RayFunc_4_n_list.java @@ -7,8 +7,6 @@ @FunctionalInterface public interface RayFunc_4_n_list extends RayFunc { - List apply(T0 t0, T1 t1, T2 t2, T3 t3) throws Throwable; - static List execute(Object[] args) throws Throwable { String name = (String) args[args.length - 2]; assert (name.equals(RayFunc_4_n_list.class.getName())); @@ -17,4 +15,6 @@ static List execute(Object[] args) throws Throwable { return f.apply((T0) args[0], (T1) args[1], (T2) args[2], (T3) args[3]); } + List apply(T0 t0, T1 t1, T2 t2, T3 t3) throws Throwable; + } diff --git a/java/api/src/main/java/org/ray/api/funcs/RayFunc_5_1.java b/java/api/src/main/java/org/ray/api/funcs/RayFunc_5_1.java index 423565f8ef06..b0ad9f6dedc2 100644 --- a/java/api/src/main/java/org/ray/api/funcs/RayFunc_5_1.java +++ b/java/api/src/main/java/org/ray/api/funcs/RayFunc_5_1.java @@ -6,8 +6,6 @@ @FunctionalInterface public interface RayFunc_5_1 extends RayFunc { - R0 apply(T0 t0, T1 t1, T2 t2, T3 t3, T4 t4) throws Throwable; - static R0 execute(Object[] args) throws Throwable { String name = (String) args[args.length - 2]; assert (name.equals(RayFunc_5_1.class.getName())); @@ -16,4 +14,6 @@ static R0 execute(Object[] args) throws Throwable { return f.apply((T0) args[0], (T1) args[1], (T2) args[2], (T3) args[3], (T4) args[4]); } + R0 apply(T0 t0, T1 t1, T2 t2, T3 t3, T4 t4) throws Throwable; + } diff --git a/java/api/src/main/java/org/ray/api/funcs/RayFunc_5_2.java b/java/api/src/main/java/org/ray/api/funcs/RayFunc_5_2.java index fcd903e872ab..48d0a8b4030a 100644 --- a/java/api/src/main/java/org/ray/api/funcs/RayFunc_5_2.java +++ b/java/api/src/main/java/org/ray/api/funcs/RayFunc_5_2.java @@ -7,8 +7,6 @@ @FunctionalInterface public interface RayFunc_5_2 extends RayFunc { - MultipleReturns2 apply(T0 t0, T1 t1, T2 t2, T3 t3, T4 t4) throws Throwable; - static MultipleReturns2 execute(Object[] args) throws Throwable { String name = (String) args[args.length - 2]; @@ -18,4 +16,6 @@ static MultipleReturns2 execute(Object[] ar return f.apply((T0) args[0], (T1) args[1], (T2) args[2], (T3) args[3], (T4) args[4]); } + MultipleReturns2 apply(T0 t0, T1 t1, T2 t2, T3 t3, T4 t4) throws Throwable; + } diff --git a/java/api/src/main/java/org/ray/api/funcs/RayFunc_5_3.java b/java/api/src/main/java/org/ray/api/funcs/RayFunc_5_3.java index 1ba58f95b84c..757a903ec9f8 100644 --- a/java/api/src/main/java/org/ray/api/funcs/RayFunc_5_3.java +++ b/java/api/src/main/java/org/ray/api/funcs/RayFunc_5_3.java @@ -7,8 +7,6 @@ @FunctionalInterface public interface RayFunc_5_3 extends RayFunc { - MultipleReturns3 apply(T0 t0, T1 t1, T2 t2, T3 t3, T4 t4) throws Throwable; - static MultipleReturns3 execute(Object[] args) throws Throwable { String name = (String) args[args.length - 2]; @@ -18,4 +16,6 @@ static MultipleReturns3 execute(Obj return f.apply((T0) args[0], (T1) args[1], (T2) args[2], (T3) args[3], (T4) args[4]); } + MultipleReturns3 apply(T0 t0, T1 t1, T2 t2, T3 t3, T4 t4) throws Throwable; + } diff --git a/java/api/src/main/java/org/ray/api/funcs/RayFunc_5_4.java b/java/api/src/main/java/org/ray/api/funcs/RayFunc_5_4.java index 39f019310cde..c207744d1cac 100644 --- a/java/api/src/main/java/org/ray/api/funcs/RayFunc_5_4.java +++ b/java/api/src/main/java/org/ray/api/funcs/RayFunc_5_4.java @@ -7,8 +7,6 @@ @FunctionalInterface public interface RayFunc_5_4 extends RayFunc { - MultipleReturns4 apply(T0 t0, T1 t1, T2 t2, T3 t3, T4 t4) throws Throwable; - static MultipleReturns4 execute( Object[] args) throws Throwable { String name = (String) args[args.length - 2]; @@ -18,4 +16,6 @@ static MultipleReturns4 exe return f.apply((T0) args[0], (T1) args[1], (T2) args[2], (T3) args[3], (T4) args[4]); } + MultipleReturns4 apply(T0 t0, T1 t1, T2 t2, T3 t3, T4 t4) throws Throwable; + } diff --git a/java/api/src/main/java/org/ray/api/funcs/RayFunc_5_n.java b/java/api/src/main/java/org/ray/api/funcs/RayFunc_5_n.java index eea116f56d92..0219edd407e1 100644 --- a/java/api/src/main/java/org/ray/api/funcs/RayFunc_5_n.java +++ b/java/api/src/main/java/org/ray/api/funcs/RayFunc_5_n.java @@ -6,18 +6,19 @@ import org.ray.api.internal.RayFunc; @FunctionalInterface -public interface RayFunc_5_n extends RayFunc { +public interface RayFunc_5_n extends RayFunc { - Map apply(Collection returnids, T0 t0, T1 t1, T2 t2, T3 t3, T4 t4) throws Throwable; - - static Map execute(Object[] args) throws Throwable { + static Map execute(Object[] args) throws Throwable { String name = (String) args[args.length - 2]; assert (name.equals(RayFunc_5_n.class.getName())); byte[] funcBytes = (byte[]) args[args.length - 1]; - RayFunc_5_n f = SerializationUtils.deserialize(funcBytes); + RayFunc_5_n f = SerializationUtils.deserialize(funcBytes); return f - .apply((Collection) args[0], (T0) args[1], (T1) args[2], (T2) args[3], (T3) args[4], + .apply((Collection) args[0], (T0) args[1], (T1) args[2], (T2) args[3], (T3) args[4], (T4) args[5]); } + Map apply(Collection returnids, T0 t0, T1 t1, T2 t2, T3 t3, T4 t4) + throws Throwable; + } diff --git a/java/api/src/main/java/org/ray/api/funcs/RayFunc_5_n_list.java b/java/api/src/main/java/org/ray/api/funcs/RayFunc_5_n_list.java index 0f85cc160141..8a09755e0516 100644 --- a/java/api/src/main/java/org/ray/api/funcs/RayFunc_5_n_list.java +++ b/java/api/src/main/java/org/ray/api/funcs/RayFunc_5_n_list.java @@ -7,8 +7,6 @@ @FunctionalInterface public interface RayFunc_5_n_list extends RayFunc { - List apply(T0 t0, T1 t1, T2 t2, T3 t3, T4 t4) throws Throwable; - static List execute(Object[] args) throws Throwable { String name = (String) args[args.length - 2]; assert (name.equals(RayFunc_5_n_list.class.getName())); @@ -17,4 +15,6 @@ static List execute(Object[] args) throws Throwable { return f.apply((T0) args[0], (T1) args[1], (T2) args[2], (T3) args[3], (T4) args[4]); } + List apply(T0 t0, T1 t1, T2 t2, T3 t3, T4 t4) throws Throwable; + } diff --git a/java/api/src/main/java/org/ray/api/funcs/RayFunc_6_1.java b/java/api/src/main/java/org/ray/api/funcs/RayFunc_6_1.java index b0e54690b108..2dc10fe77df8 100644 --- a/java/api/src/main/java/org/ray/api/funcs/RayFunc_6_1.java +++ b/java/api/src/main/java/org/ray/api/funcs/RayFunc_6_1.java @@ -6,8 +6,6 @@ @FunctionalInterface public interface RayFunc_6_1 extends RayFunc { - R0 apply(T0 t0, T1 t1, T2 t2, T3 t3, T4 t4, T5 t5) throws Throwable; - static R0 execute(Object[] args) throws Throwable { String name = (String) args[args.length - 2]; assert (name.equals(RayFunc_6_1.class.getName())); @@ -17,4 +15,6 @@ static R0 execute(Object[] args) throws Throwable { .apply((T0) args[0], (T1) args[1], (T2) args[2], (T3) args[3], (T4) args[4], (T5) args[5]); } + R0 apply(T0 t0, T1 t1, T2 t2, T3 t3, T4 t4, T5 t5) throws Throwable; + } diff --git a/java/api/src/main/java/org/ray/api/funcs/RayFunc_6_2.java b/java/api/src/main/java/org/ray/api/funcs/RayFunc_6_2.java index 38a33840f02d..853aafb1ceab 100644 --- a/java/api/src/main/java/org/ray/api/funcs/RayFunc_6_2.java +++ b/java/api/src/main/java/org/ray/api/funcs/RayFunc_6_2.java @@ -7,8 +7,6 @@ @FunctionalInterface public interface RayFunc_6_2 extends RayFunc { - MultipleReturns2 apply(T0 t0, T1 t1, T2 t2, T3 t3, T4 t4, T5 t5) throws Throwable; - static MultipleReturns2 execute(Object[] args) throws Throwable { String name = (String) args[args.length - 2]; @@ -19,4 +17,6 @@ static MultipleReturns2 execute(Object[ .apply((T0) args[0], (T1) args[1], (T2) args[2], (T3) args[3], (T4) args[4], (T5) args[5]); } + MultipleReturns2 apply(T0 t0, T1 t1, T2 t2, T3 t3, T4 t4, T5 t5) throws Throwable; + } diff --git a/java/api/src/main/java/org/ray/api/funcs/RayFunc_6_3.java b/java/api/src/main/java/org/ray/api/funcs/RayFunc_6_3.java index 93b4823db1f6..aafa90f204f6 100644 --- a/java/api/src/main/java/org/ray/api/funcs/RayFunc_6_3.java +++ b/java/api/src/main/java/org/ray/api/funcs/RayFunc_6_3.java @@ -7,8 +7,6 @@ @FunctionalInterface public interface RayFunc_6_3 extends RayFunc { - MultipleReturns3 apply(T0 t0, T1 t1, T2 t2, T3 t3, T4 t4, T5 t5) throws Throwable; - static MultipleReturns3 execute(Object[] args) throws Throwable { String name = (String) args[args.length - 2]; @@ -19,4 +17,6 @@ static MultipleReturns3 execute .apply((T0) args[0], (T1) args[1], (T2) args[2], (T3) args[3], (T4) args[4], (T5) args[5]); } + MultipleReturns3 apply(T0 t0, T1 t1, T2 t2, T3 t3, T4 t4, T5 t5) throws Throwable; + } diff --git a/java/api/src/main/java/org/ray/api/funcs/RayFunc_6_4.java b/java/api/src/main/java/org/ray/api/funcs/RayFunc_6_4.java index 7db9f0bf6cde..0c401279bb64 100644 --- a/java/api/src/main/java/org/ray/api/funcs/RayFunc_6_4.java +++ b/java/api/src/main/java/org/ray/api/funcs/RayFunc_6_4.java @@ -7,8 +7,6 @@ @FunctionalInterface public interface RayFunc_6_4 extends RayFunc { - MultipleReturns4 apply(T0 t0, T1 t1, T2 t2, T3 t3, T4 t4, T5 t5) throws Throwable; - static MultipleReturns4 execute( Object[] args) throws Throwable { String name = (String) args[args.length - 2]; @@ -20,4 +18,6 @@ static MultipleReturns4 .apply((T0) args[0], (T1) args[1], (T2) args[2], (T3) args[3], (T4) args[4], (T5) args[5]); } + MultipleReturns4 apply(T0 t0, T1 t1, T2 t2, T3 t3, T4 t4, T5 t5) throws Throwable; + } diff --git a/java/api/src/main/java/org/ray/api/funcs/RayFunc_6_n.java b/java/api/src/main/java/org/ray/api/funcs/RayFunc_6_n.java index f755141410e5..e0fd9c52eacf 100644 --- a/java/api/src/main/java/org/ray/api/funcs/RayFunc_6_n.java +++ b/java/api/src/main/java/org/ray/api/funcs/RayFunc_6_n.java @@ -6,19 +6,19 @@ import org.ray.api.internal.RayFunc; @FunctionalInterface -public interface RayFunc_6_n extends RayFunc { +public interface RayFunc_6_n extends RayFunc { - Map apply(Collection returnids, T0 t0, T1 t1, T2 t2, T3 t3, T4 t4, T5 t5) - throws Throwable; - - static Map execute(Object[] args) throws Throwable { + static Map execute(Object[] args) throws Throwable { String name = (String) args[args.length - 2]; assert (name.equals(RayFunc_6_n.class.getName())); byte[] funcBytes = (byte[]) args[args.length - 1]; - RayFunc_6_n f = SerializationUtils.deserialize(funcBytes); + RayFunc_6_n f = SerializationUtils.deserialize(funcBytes); return f - .apply((Collection) args[0], (T0) args[1], (T1) args[2], (T2) args[3], (T3) args[4], + .apply((Collection) args[0], (T0) args[1], (T1) args[2], (T2) args[3], (T3) args[4], (T4) args[5], (T5) args[6]); } + Map apply(Collection returnids, T0 t0, T1 t1, T2 t2, T3 t3, T4 t4, T5 t5) + throws Throwable; + } diff --git a/java/api/src/main/java/org/ray/api/funcs/RayFunc_6_n_list.java b/java/api/src/main/java/org/ray/api/funcs/RayFunc_6_n_list.java index 123f3abcc940..98218cf7027f 100644 --- a/java/api/src/main/java/org/ray/api/funcs/RayFunc_6_n_list.java +++ b/java/api/src/main/java/org/ray/api/funcs/RayFunc_6_n_list.java @@ -7,8 +7,6 @@ @FunctionalInterface public interface RayFunc_6_n_list extends RayFunc { - List apply(T0 t0, T1 t1, T2 t2, T3 t3, T4 t4, T5 t5) throws Throwable; - static List execute(Object[] args) throws Throwable { String name = (String) args[args.length - 2]; assert (name.equals(RayFunc_6_n_list.class.getName())); @@ -18,4 +16,6 @@ static List execute(Object[] args) throws Throwab .apply((T0) args[0], (T1) args[1], (T2) args[2], (T3) args[3], (T4) args[4], (T5) args[5]); } + List apply(T0 t0, T1 t1, T2 t2, T3 t3, T4 t4, T5 t5) throws Throwable; + } diff --git a/java/api/src/main/java/org/ray/api/internal/Callable.java b/java/api/src/main/java/org/ray/api/internal/Callable.java index d0c500130fe5..e52f083cab68 100644 --- a/java/api/src/main/java/org/ray/api/internal/Callable.java +++ b/java/api/src/main/java/org/ray/api/internal/Callable.java @@ -1,7 +1,7 @@ package org.ray.api.internal; /** - * hold the remote call + * hold the remote call. */ public interface Callable { diff --git a/java/api/src/main/java/org/ray/api/internal/RayFunc.java b/java/api/src/main/java/org/ray/api/internal/RayFunc.java index a8d0dde01c9d..8928eec11db0 100644 --- a/java/api/src/main/java/org/ray/api/internal/RayFunc.java +++ b/java/api/src/main/java/org/ray/api/internal/RayFunc.java @@ -3,7 +3,7 @@ import java.io.Serializable; /** - * Base of the ray remote function + * Base of the ray remote function. */ public interface RayFunc extends Serializable { diff --git a/java/api/src/main/java/org/ray/api/returns/MultipleReturns.java b/java/api/src/main/java/org/ray/api/returns/MultipleReturns.java index 15bd79b1ee9c..eda0ba2f6384 100644 --- a/java/api/src/main/java/org/ray/api/returns/MultipleReturns.java +++ b/java/api/src/main/java/org/ray/api/returns/MultipleReturns.java @@ -1,13 +1,13 @@ package org.ray.api.returns; /** - * Multiple return objects for user's method + * Multiple return objects for user's method. */ public class MultipleReturns { protected final Object[] values; - public MultipleReturns(Object values[]) { + public MultipleReturns(Object[] values) { this.values = values; } diff --git a/java/api/src/main/java/org/ray/api/returns/MultipleReturns2.java b/java/api/src/main/java/org/ray/api/returns/MultipleReturns2.java index 6f898decfeb2..90f4cf9f2689 100644 --- a/java/api/src/main/java/org/ray/api/returns/MultipleReturns2.java +++ b/java/api/src/main/java/org/ray/api/returns/MultipleReturns2.java @@ -4,7 +4,7 @@ public class MultipleReturns2 extends MultipleReturns { public MultipleReturns2(R0 r0, R1 r1) { - super(new Object[]{r0, r1}); + super(new Object[] {r0, r1}); } public R0 get0() { diff --git a/java/api/src/main/java/org/ray/api/returns/MultipleReturns3.java b/java/api/src/main/java/org/ray/api/returns/MultipleReturns3.java index 7f1ce4c72466..2940377502bf 100644 --- a/java/api/src/main/java/org/ray/api/returns/MultipleReturns3.java +++ b/java/api/src/main/java/org/ray/api/returns/MultipleReturns3.java @@ -4,7 +4,7 @@ public class MultipleReturns3 extends MultipleReturns { public MultipleReturns3(R0 r0, R1 r1, R2 r2) { - super(new Object[]{r0, r1, r2}); + super(new Object[] {r0, r1, r2}); } public R0 get0() { diff --git a/java/api/src/main/java/org/ray/api/returns/MultipleReturns4.java b/java/api/src/main/java/org/ray/api/returns/MultipleReturns4.java index bb9b7fe17d62..581a682e0ba5 100644 --- a/java/api/src/main/java/org/ray/api/returns/MultipleReturns4.java +++ b/java/api/src/main/java/org/ray/api/returns/MultipleReturns4.java @@ -4,7 +4,7 @@ public class MultipleReturns4 extends MultipleReturns { public MultipleReturns4(R0 r0, R1 r1, R2 r2, R3 r3) { - super(new Object[]{r0, r1, r2, r3}); + super(new Object[] {r0, r1, r2, r3}); } public R0 get0() { diff --git a/java/api/src/main/java/org/ray/api/returns/RayObjects2.java b/java/api/src/main/java/org/ray/api/returns/RayObjects2.java index bb2e3b89cfb2..1afd1fd510a4 100644 --- a/java/api/src/main/java/org/ray/api/returns/RayObjects2.java +++ b/java/api/src/main/java/org/ray/api/returns/RayObjects2.java @@ -11,7 +11,7 @@ public RayObjects2(UniqueID[] ids) { super(ids); } - public RayObjects2(RayObject objs[]) { + public RayObjects2(RayObject[] objs) { super(objs); } diff --git a/java/api/src/main/java/org/ray/api/returns/RayObjects3.java b/java/api/src/main/java/org/ray/api/returns/RayObjects3.java index 7b9dc62ac8c7..486c4e1dcb60 100644 --- a/java/api/src/main/java/org/ray/api/returns/RayObjects3.java +++ b/java/api/src/main/java/org/ray/api/returns/RayObjects3.java @@ -11,7 +11,7 @@ public RayObjects3(UniqueID[] ids) { super(ids); } - public RayObjects3(RayObject objs[]) { + public RayObjects3(RayObject[] objs) { super(objs); } diff --git a/java/api/src/main/java/org/ray/api/returns/RayObjects4.java b/java/api/src/main/java/org/ray/api/returns/RayObjects4.java index e045f4e5db8c..ce6f130f81dc 100644 --- a/java/api/src/main/java/org/ray/api/returns/RayObjects4.java +++ b/java/api/src/main/java/org/ray/api/returns/RayObjects4.java @@ -11,7 +11,7 @@ public RayObjects4(UniqueID[] ids) { super(ids); } - public RayObjects4(RayObject objs[]) { + public RayObjects4(RayObject[] objs) { super(objs); } diff --git a/java/checkstyle-suppressions.xml b/java/checkstyle-suppressions.xml new file mode 100644 index 000000000000..f84d9e1ca12d --- /dev/null +++ b/java/checkstyle-suppressions.xml @@ -0,0 +1,22 @@ + + + + + + + + + + + + + + + + + + + + diff --git a/java/checkstyle.xml b/java/checkstyle.xml new file mode 100644 index 000000000000..88f970e04d03 --- /dev/null +++ b/java/checkstyle.xml @@ -0,0 +1,240 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/java/common/pom.xml b/java/common/pom.xml index c6b19f45761b..b3985f2f8c7a 100644 --- a/java/common/pom.xml +++ b/java/common/pom.xml @@ -1,40 +1,40 @@ - - - org.ray.parent - ray-superpom - 1.0 - - 4.0.0 + + + org.ray.parent + ray-superpom + 1.0 + + 4.0.0 - org.ray - ray-common - java common and util for ray - java common and util for ray - + org.ray + ray-common + java common and util for ray + java common and util for ray + - jar + jar - - - log4j - log4j - 1.2.17 - - - quartz - quartz - 1.5.2 - - - org.ini4j - ini4j - 0.5.2 - - + + + log4j + log4j + 1.2.17 + + + quartz + quartz + 1.5.2 + + + org.ini4j + ini4j + 0.5.2 + + diff --git a/java/common/src/main/java/org/ray/util/CommonUtil.java b/java/common/src/main/java/org/ray/util/CommonUtil.java index 85744153459a..052798d613f8 100644 --- a/java/common/src/main/java/org/ray/util/CommonUtil.java +++ b/java/common/src/main/java/org/ray/util/CommonUtil.java @@ -3,14 +3,14 @@ import java.util.Random; /** - * Common utilities + * Common utilities. */ public class CommonUtil { private static final Random seed = new Random(); /** - * Get random number between 0 and (max-1) + * Get random number between 0 and (max-1). */ public static int getRandom(int max) { return Math.abs(seed.nextInt() % max); diff --git a/java/common/src/main/java/org/ray/util/NetworkUtil.java b/java/common/src/main/java/org/ray/util/NetworkUtil.java index 268959552e7e..d5e48999c4aa 100644 --- a/java/common/src/main/java/org/ray/util/NetworkUtil.java +++ b/java/common/src/main/java/org/ray/util/NetworkUtil.java @@ -53,10 +53,8 @@ public static boolean isPortAvailable(int port) { ds.setReuseAddress(true); return true; } catch (IOException ignored) { + /* should not be thrown */ + return false; } - - /* should not be thrown */ - - return false; } } diff --git a/java/common/src/main/java/org/ray/util/ObjectUtil.java b/java/common/src/main/java/org/ray/util/ObjectUtil.java index e4d33c8626c7..bbee032616d5 100644 --- a/java/common/src/main/java/org/ray/util/ObjectUtil.java +++ b/java/common/src/main/java/org/ray/util/ObjectUtil.java @@ -7,7 +7,8 @@ public class ObjectUtil { public static T newObject(Class cls) { try { return cls.getConstructor().newInstance(); - } catch (InstantiationException | IllegalAccessException | NoSuchMethodException | InvocationTargetException e) { + } catch (InstantiationException | IllegalAccessException | NoSuchMethodException + | InvocationTargetException e) { e.printStackTrace(); return null; } diff --git a/java/common/src/main/java/org/ray/util/StringUtil.java b/java/common/src/main/java/org/ray/util/StringUtil.java index ecd235951e14..213d46eb21e4 100644 --- a/java/common/src/main/java/org/ray/util/StringUtil.java +++ b/java/common/src/main/java/org/ray/util/StringUtil.java @@ -6,29 +6,15 @@ public class StringUtil { - // Holds the start of an element and which brace started it. - private static class Start { - - // The brace number from the braces string in use. - final int brace; - // The position in the string it was seen. - final int pos; - - // Constructor. - public Start(int brace, int pos) { - this.brace = brace; - this.pos = pos; - } - } - /** - * @param s input string + * split. + * @param s input string * @param splitters common splitters - * @param open open braces - * @param close close braces + * @param open open braces + * @param close close braces * @return output array list */ - public static Vector Split(String s, String splitters, String open, String close) { + public static Vector split(String s, String splitters, String open, String close) { // The splits. Vector split = new Vector<>(); // The stack. @@ -130,5 +116,20 @@ public static String mergeArray(T[] objs, String concatenator) { } return objs.length == 0 ? "" : sb.substring(0, sb.length() - concatenator.length()); } + + // Holds the start of an element and which brace started it. + private static class Start { + + // The brace number from the braces string in use. + final int brace; + // The position in the string it was seen. + final int pos; + + // Constructor. + public Start(int brace, int pos) { + this.brace = brace; + this.pos = pos; + } + } } diff --git a/java/common/src/main/java/org/ray/util/SystemUtil.java b/java/common/src/main/java/org/ray/util/SystemUtil.java index e05b65a29366..f43b5870f383 100644 --- a/java/common/src/main/java/org/ray/util/SystemUtil.java +++ b/java/common/src/main/java/org/ray/util/SystemUtil.java @@ -6,10 +6,13 @@ import org.ray.util.logger.RayLog; /** - * some utilities for system process + * some utilities for system process. */ public class SystemUtil { + static final ReentrantLock pidlock = new ReentrantLock(); + static Integer pid; + public static String userHome() { return System.getProperty("user.home"); } @@ -36,9 +39,6 @@ public static boolean startWithJar(String clsName) { } } - static Integer pid; - static final ReentrantLock pidlock = new ReentrantLock(); - public static int pid() { if (pid == null) { pidlock.lock(); diff --git a/java/common/src/main/java/org/ray/util/config/AConfig.java b/java/common/src/main/java/org/ray/util/config/AConfig.java index e17a11c3cf19..546e1d64347c 100644 --- a/java/common/src/main/java/org/ray/util/config/AConfig.java +++ b/java/common/src/main/java/org/ray/util/config/AConfig.java @@ -13,24 +13,24 @@ public @interface AConfig { /** - * comments for this configuration field + * comments for this configuration field. */ String comment(); /** * when the config is an array list, a splitter set is specified, e.g., " \t" to use ' ' and '\t' - * as possible splits + * as possible splits. */ String splitters() default ", \t"; /** - * indirect with value as the new section name, the field name remains the same + * indirect with value as the new section name, the field name remains the same. */ String defaultIndirectSectionName() default ""; /** * see ConfigReader.getIndirectStringArray this config tells which is the default - * indirectSectionName in that function + * indirectSectionName in that function. */ String defaultArrayIndirectSectionName() default ""; } diff --git a/java/common/src/main/java/org/ray/util/config/ConfigReader.java b/java/common/src/main/java/org/ray/util/config/ConfigReader.java index aea8e2ce7c10..889e72d85154 100644 --- a/java/common/src/main/java/org/ray/util/config/ConfigReader.java +++ b/java/common/src/main/java/org/ray/util/config/ConfigReader.java @@ -43,10 +43,6 @@ public ConfigReader(String filePath, String updateConfigStr) throws Exception { } - public String filePath() { - return file; - } - private void loadConfigFile(String filePath) throws Exception { this.currentUseConfig.filePath = filePath; @@ -106,46 +102,16 @@ private void updateConfigFile(String updateConfigStr) { } } - public CurrentUseConfig getCurrentUseConfig() { - return currentUseConfig; - } - - private synchronized String getOriValue(String sectionKey, String configKey, T defaultValue, - String deptr) { - if (null == deptr) { - throw new RuntimeException("desc must not be empty of the key:" + configKey); - } - Profile.Section section = ini.get(sectionKey); - String oriValue = null; - if (section != null && section.containsKey(configKey)) { - oriValue = section.get(configKey); - } - - if (!currentUseConfig.sectionMap.containsKey(sectionKey)) { - ConfigSection configSection = new ConfigSection(); - configSection.sectionKey = sectionKey; - updateConfigSection(configSection, configKey, defaultValue, deptr, oriValue); - currentUseConfig.sectionMap.put(sectionKey, configSection); - } else if (!currentUseConfig.sectionMap.get(sectionKey).itemMap.containsKey(configKey)) { - ConfigSection configSection = currentUseConfig.sectionMap.get(sectionKey); - updateConfigSection(configSection, configKey, defaultValue, deptr, oriValue); - } - return oriValue; + public String filePath() { + return file; } - private void updateConfigSection(ConfigSection configSection, String configKey, - T defaultValue, String deptr, String oriValue) { - ConfigItem configItem = new ConfigItem<>(); - configItem.defaultValue = defaultValue; - configItem.key = configKey; - configItem.oriValue = oriValue; - configItem.desc = deptr; - configSection.itemMap.put(configKey, configItem); + public CurrentUseConfig getCurrentUseConfig() { + return currentUseConfig; } - public String getStringValue(String sectionKey, String configKey, String defaultValue, - String dsptr) { + String dsptr) { String value = getOriValue(sectionKey, configKey, defaultValue, dsptr); if (value != null) { return value; @@ -155,7 +121,7 @@ public String getStringValue(String sectionKey, String configKey, String default } public boolean getBooleanValue(String sectionKey, String configKey, boolean defaultValue, - String dsptr) { + String dsptr) { String value = getOriValue(sectionKey, configKey, defaultValue, dsptr); if (value != null) { if (value.length() == 0) { @@ -181,6 +147,39 @@ public int getIntegerValue(String sectionKey, String configKey, int defaultValue } } + private synchronized String getOriValue(String sectionKey, String configKey, T defaultValue, + String deptr) { + if (null == deptr) { + throw new RuntimeException("desc must not be empty of the key:" + configKey); + } + Profile.Section section = ini.get(sectionKey); + String oriValue = null; + if (section != null && section.containsKey(configKey)) { + oriValue = section.get(configKey); + } + + if (!currentUseConfig.sectionMap.containsKey(sectionKey)) { + ConfigSection configSection = new ConfigSection(); + configSection.sectionKey = sectionKey; + updateConfigSection(configSection, configKey, defaultValue, deptr, oriValue); + currentUseConfig.sectionMap.put(sectionKey, configSection); + } else if (!currentUseConfig.sectionMap.get(sectionKey).itemMap.containsKey(configKey)) { + ConfigSection configSection = currentUseConfig.sectionMap.get(sectionKey); + updateConfigSection(configSection, configKey, defaultValue, deptr, oriValue); + } + return oriValue; + } + + private void updateConfigSection(ConfigSection configSection, String configKey, + T defaultValue, String deptr, String oriValue) { + ConfigItem configItem = new ConfigItem<>(); + configItem.defaultValue = defaultValue; + configItem.key = configKey; + configItem.oriValue = oriValue; + configItem.desc = deptr; + configSection.itemMap.put(configKey, configItem); + } + public long getLongValue(String sectionKey, String configKey, long defaultValue, String dsptr) { String value = getOriValue(sectionKey, configKey, defaultValue, dsptr); if (value != null) { @@ -195,7 +194,7 @@ public long getLongValue(String sectionKey, String configKey, long defaultValue, } public double getDoubleValue(String sectionKey, String configKey, double defaultValue, - String dsptr) { + String dsptr) { String value = getOriValue(sectionKey, configKey, defaultValue, dsptr); if (value != null) { if (value.length() == 0) { @@ -210,7 +209,7 @@ public double getDoubleValue(String sectionKey, String configKey, double default public int[] getIntegerArray(String sectionKey, String configKey, int[] defaultValue, - String dsptr) { + String dsptr) { String value = getOriValue(sectionKey, configKey, defaultValue, dsptr); int[] array = defaultValue; if (value != null) { @@ -226,24 +225,22 @@ public int[] getIntegerArray(String sectionKey, String configKey, int[] defaultV /** * get a string list from a whole section as keys e.g., [core] data_dirs = local.dirs # or * cluster.dirs - * * [local.dirs] /home/xxx/1 /home/yyy/2 - * * [cluster.dirs] ... * - * @param sectionKey e.g., core - * @param configKey e.g., data_dirs + * @param sectionKey e.g., core + * @param configKey e.g., data_dirs * @param indirectSectionName e.g., cluster.dirs * @return string list */ public String[] getIndirectStringArray(String sectionKey, String configKey, - String indirectSectionName, String dsptr) { + String indirectSectionName, String dsptr) { String s = getStringValue(sectionKey, configKey, indirectSectionName, dsptr); Profile.Section section = ini.get(s); if (section == null) { - return new String[]{}; + return new String[] {}; } else { - return section.keySet().toArray(new String[]{}); + return section.keySet().toArray(new String[] {}); } } @@ -326,8 +323,9 @@ public void readObject(String sectionKey, T obj, T defaultValues) { String sv = getStringValue(section, fld.getName(), defaultFldValue.toString(), comment); Object v; try { - v = fld.getType().getConstructor(new Class[]{String.class}).newInstance(sv); - } catch (NoSuchMethodException | SecurityException | InstantiationException | InvocationTargetException e) { + v = fld.getType().getConstructor(new Class[] {String.class}).newInstance(sv); + } catch (NoSuchMethodException | SecurityException | InstantiationException + | InvocationTargetException e) { System.err.println( section + "." + fld.getName() + "'s format (" + sv + ") is invalid, default to " + defaultFldValue.toString()); @@ -340,7 +338,7 @@ public void readObject(String sectionKey, T obj, T defaultValues) { if (null == ss) { fld.set(obj, defaultFldValue); } else { - Vector ls = StringUtil.Split(ss, splitters, "", ""); + Vector ls = StringUtil.split(ss, splitters, "", ""); if (ccls.equals(boolean.class)) { boolean[] v = ObjectUtil .toBooleanArray(ls.stream().map(Boolean::parseBoolean).toArray()); @@ -357,7 +355,7 @@ public void readObject(String sectionKey, T obj, T defaultValues) { } else if (ccls.equals(String.class)) { String[] v; if (StringUtil.isNullOrEmpty(defaultArrayIndirectSectionName)) { - v = ls.toArray(new String[]{}); + v = ls.toArray(new String[] {}); } else { v = this .getIndirectStringArray(section, fld.getName(), diff --git a/java/common/src/main/java/org/ray/util/config/ConfigSection.java b/java/common/src/main/java/org/ray/util/config/ConfigSection.java index 95f5c1b4c256..10831a516049 100644 --- a/java/common/src/main/java/org/ray/util/config/ConfigSection.java +++ b/java/common/src/main/java/org/ray/util/config/ConfigSection.java @@ -8,7 +8,6 @@ */ public class ConfigSection { - public String sectionKey; - public final Map> itemMap = new ConcurrentHashMap<>(); + public String sectionKey; } diff --git a/java/common/src/main/java/org/ray/util/config/CurrentUseConfig.java b/java/common/src/main/java/org/ray/util/config/CurrentUseConfig.java index 4367fa163d1a..c2083e04a25c 100644 --- a/java/common/src/main/java/org/ray/util/config/CurrentUseConfig.java +++ b/java/common/src/main/java/org/ray/util/config/CurrentUseConfig.java @@ -9,8 +9,7 @@ */ public class CurrentUseConfig { - public String filePath; - public final Map sectionMap = new ConcurrentHashMap<>(); + public String filePath; } diff --git a/java/common/src/main/java/org/ray/util/generator/Composition.java b/java/common/src/main/java/org/ray/util/generator/Composition.java index 46c59278738e..f8e1afd7efee 100644 --- a/java/common/src/main/java/org/ray/util/generator/Composition.java +++ b/java/common/src/main/java/org/ray/util/generator/Composition.java @@ -9,20 +9,8 @@ */ public class Composition { - public static class TR { - - public final int Tcount; - public final int Rcount; - - public TR(int tcount, int rcount) { - super(); - Tcount = tcount; - Rcount = rcount; - } - } - - public static List calculate(int maxT, int maxR) { - List ret = new ArrayList<>(); + public static List calculate(int maxT, int maxR) { + List ret = new ArrayList<>(); for (int t = 0; t <= maxT; t++) { // <= 0 for dynamic return count @@ -30,9 +18,21 @@ public static List calculate(int maxT, int maxR) { // -1 for call_n returns RayObject<>[N] for (int r = -1; r <= maxR; r++) { - ret.add(new TR(t, r)); + ret.add(new Tr(t, r)); } } return ret; } + + public static class Tr { + + public final int tcount; + public final int rcount; + + public Tr(int tcount, int rcount) { + super(); + this.tcount = tcount; + this.rcount = rcount; + } + } } diff --git a/java/common/src/main/java/org/ray/util/generator/FuncsGenerator.java b/java/common/src/main/java/org/ray/util/generator/FuncsGenerator.java index 70c7bb190729..b2a5f46204cf 100644 --- a/java/common/src/main/java/org/ray/util/generator/FuncsGenerator.java +++ b/java/common/src/main/java/org/ray/util/generator/FuncsGenerator.java @@ -2,7 +2,7 @@ import java.io.IOException; import org.ray.util.FileUtil; -import org.ray.util.generator.Composition.TR; +import org.ray.util.generator.Composition.Tr; /** * Generate all classes in org.ray.api.funcs @@ -16,10 +16,10 @@ public static void main(String[] args) throws IOException { } private static void generate(String rootdir) throws IOException { - for (TR tr : Composition.calculate(Share.MAX_T, Share.MAX_R)) { - String str = build(tr.Tcount, tr.Rcount); - String file = rootdir + "/RayFunc_" + tr.Tcount + "_" - + (tr.Rcount <= 0 ? (tr.Rcount == 0 ? "n" : "n_list") : tr.Rcount) + ".java"; + for (Tr tr : Composition.calculate(Share.MAX_T, Share.MAX_R)) { + String str = build(tr.tcount, tr.rcount); + String file = rootdir + "/RayFunc_" + tr.tcount + "_" + + (tr.rcount <= 0 ? (tr.rcount == 0 ? "n" : "n_list") : tr.rcount) + ".java"; FileUtil.overrideFile(file, str); System.err.println("override " + file); } @@ -34,20 +34,21 @@ private static void generate(String rootdir) throws IOException { * * public static R0 execute(Object[] args) throws Throwable { String name = * (String)args[args.length - 2]; assert (name.equals(RayFunc_0_1.class.getName())); byte[] - * funcBytes = (byte[])args[args.length - 1]; RayFunc_0_1 f = (RayFunc_0_1)SerializationUtils.deserialize(funcBytes); + * funcBytes = (byte[])args[args.length - 1]; RayFunc_0_1 f = (RayFunc_0_1) + * SerializationUtils.deserialize(funcBytes); * return f.apply(); } } */ - private static String build(int Tcount, int Rcount) { + private static String build(int tcount, int rcount) { StringBuilder sb = new StringBuilder(); String tname = - "Ray" + "Func_" + Tcount + "_" + (Rcount <= 0 ? (Rcount == 0 ? "n" : "n_list") : Rcount); - String gname = tname + "<" + Share.buildClassDeclare(Tcount, Rcount) + ">"; + "Ray" + "Func_" + tcount + "_" + (rcount <= 0 ? (rcount == 0 ? "n" : "n_list") : rcount); + final String gname = tname + "<" + Share.buildClassDeclare(tcount, rcount) + ">"; sb.append("package org.ray.api.funcs;").append("\n"); - if (Rcount > 1) { + if (rcount > 1) { sb.append("import org.ray.api.returns.*;").append("\n"); } - if (Rcount <= 0) { + if (rcount <= 0) { sb.append("import java.util.Collection;").append("\n"); sb.append("import java.util.List;").append("\n"); sb.append("import java.util.Map;").append("\n"); @@ -60,14 +61,14 @@ private static String build(int Tcount, int Rcount) { sb.append("@FunctionalInterface").append("\n"); sb.append("public interface ").append(gname).append(" extends RayFunc {") .append("\n"); - sb.append("\t").append(Share.buildFuncReturn(Rcount)).append(" apply(") - .append(Rcount == 0 ? ("Collection returnids" + (Tcount > 0 ? ", " : "")) : "") - .append(Share.buildParameter(Tcount, "T", null)).append(") throws Throwable;") + sb.append("\t").append(Share.buildFuncReturn(rcount)).append(" apply(") + .append(rcount == 0 ? ("Collection returnids" + (tcount > 0 ? ", " : "")) : "") + .append(Share.buildParameter(tcount, "T", null)).append(") throws Throwable;") .append("\n"); sb.append("\t\n"); - sb.append("\tpublic static " + "<").append(Share.buildClassDeclare(Tcount, Rcount)) - .append(">").append(" ").append(Share.buildFuncReturn(Rcount)) + sb.append("\tpublic static " + "<").append(Share.buildClassDeclare(tcount, rcount)) + .append(">").append(" ").append(Share.buildFuncReturn(rcount)) .append(" execute(Object[] args) throws Throwable {").append("\n"); sb.append("\t\tString name = (String)args[args.length - 2];").append("\n"); sb.append("\t\tassert (name.equals(").append(tname).append(".class.getName()));").append("\n"); @@ -75,8 +76,8 @@ private static String build(int Tcount, int Rcount) { sb.append("\t\t").append(gname).append(" f = SerializationUtils.deserialize(funcBytes);") .append("\n"); sb.append("\t\treturn f.apply(") - .append(Rcount == 0 ? ("(Collection)args[0]" + (Tcount > 0 ? ", " : "")) : "") - .append(Share.buildParameterUse2(Tcount, Rcount == 0 ? 1 : 0, "T", "args[", "]")) + .append(rcount == 0 ? ("(Collection)args[0]" + (tcount > 0 ? ", " : "")) : "") + .append(Share.buildParameterUse2(tcount, rcount == 0 ? 1 : 0, "T", "args[", "]")) .append(");").append("\n"); sb.append("\t}").append("\n"); sb.append("\t\n"); diff --git a/java/common/src/main/java/org/ray/util/generator/MultipleReturnGenerator.java b/java/common/src/main/java/org/ray/util/generator/MultipleReturnGenerator.java index e6d2c83bf69e..b68452be62cb 100644 --- a/java/common/src/main/java/org/ray/util/generator/MultipleReturnGenerator.java +++ b/java/common/src/main/java/org/ray/util/generator/MultipleReturnGenerator.java @@ -20,35 +20,25 @@ public static void main(String[] args) throws IOException { } /** - * package org.ray.api.returns; - * - * public class MultipleReturns2 extends MultipleReturns { - * - * public MultipleReturns2(R0 r0, R1 r1) { super(new Object[] { r0, r1 }); } - * - * public R0 get0() { return (R0) this.values[0]; } - * - * public R1 get1() { return (R1) this.values[1]; } - * - * } + * package org.ray.api.returns. */ - private static String build(int Rcount) { + private static String build(int rcount) { StringBuilder sb = new StringBuilder(); sb.append("package org.ray.api.returns;").append("\n"); sb.append("import org.ray.api.*;").append("\n"); sb.append("@SuppressWarnings(\"unchecked\")"); - sb.append("public class MultipleReturns").append(Rcount).append("<") - .append(Share.buildClassDeclare(0, Rcount)).append("> extends MultipleReturns {") + sb.append("public class MultipleReturns").append(rcount).append("<") + .append(Share.buildClassDeclare(0, rcount)).append("> extends MultipleReturns {") .append("\n"); - sb.append("\tpublic MultipleReturns").append(Rcount).append("(") - .append(Share.buildParameter(Rcount, "R", null)).append(") {") + sb.append("\tpublic MultipleReturns").append(rcount).append("(") + .append(Share.buildParameter(rcount, "R", null)).append(") {") .append("\n"); - sb.append("\t\tsuper(new Object[] { ").append(Share.buildParameterUse(Rcount, "R")) + sb.append("\t\tsuper(new Object[] { ").append(Share.buildParameterUse(rcount, "R")) .append(" });") .append("\n"); sb.append("\t}").append("\n"); - for (int k = 0; k < Rcount; k++) { + for (int k = 0; k < rcount; k++) { sb.append(buildGetter(k)); } diff --git a/java/common/src/main/java/org/ray/util/generator/RayObjectsGenerator.java b/java/common/src/main/java/org/ray/util/generator/RayObjectsGenerator.java index 5af1a67516ec..35a444905602 100644 --- a/java/common/src/main/java/org/ray/util/generator/RayObjectsGenerator.java +++ b/java/common/src/main/java/org/ray/util/generator/RayObjectsGenerator.java @@ -35,23 +35,23 @@ public static void main(String[] args) throws IOException { * * public RayObject r1() { return objs[1]; } } */ - private static String build(int Rcount) { + private static String build(int rcount) { StringBuilder sb = new StringBuilder(); sb.append("package org.ray.api.returns;\n"); sb.append("import org.ray.api.*;\n"); sb.append("import org.ray.spi.model.UniqueID;\n"); sb.append("@SuppressWarnings({\"rawtypes\", \"unchecked\"})"); - sb.append("public class RayObjects").append(Rcount).append("<") - .append(Share.buildClassDeclare(0, Rcount)).append("> extends RayObjects {") + sb.append("public class RayObjects").append(rcount).append("<") + .append(Share.buildClassDeclare(0, rcount)).append("> extends RayObjects {") .append("\n"); - sb.append("\tpublic RayObjects").append(Rcount).append("(UniqueID[] ids) {").append("\n"); + sb.append("\tpublic RayObjects").append(rcount).append("(UniqueID[] ids) {").append("\n"); sb.append("\t\tsuper(ids);").append("\n"); sb.append("\t}").append("\n"); - sb.append("\tpublic RayObjects").append(Rcount).append("(RayObject objs[]) {").append("\n"); + sb.append("\tpublic RayObjects").append(rcount).append("(RayObject objs[]) {").append("\n"); sb.append("\t\tsuper(objs);").append("\n"); sb.append("\t}").append("\n"); - for (int k = 0; k < Rcount; k++) { + for (int k = 0; k < rcount; k++) { sb.append(buildGetter(k)); } @@ -59,9 +59,6 @@ private static String build(int Rcount) { return sb.toString(); } - /** - * public RayObject r0() { return objs[0]; } - */ private static String buildGetter(int index) { return "\tpublic RayObject r" + index + "() {\n" + "\t\treturn objs[" + index + "];\n" diff --git a/java/common/src/main/java/org/ray/util/generator/RpcGenerator.java b/java/common/src/main/java/org/ray/util/generator/RpcGenerator.java index f7112b49b518..ec2195d5f818 100644 --- a/java/common/src/main/java/org/ray/util/generator/RpcGenerator.java +++ b/java/common/src/main/java/org/ray/util/generator/RpcGenerator.java @@ -4,7 +4,7 @@ import java.util.HashSet; import java.util.Set; import org.ray.util.FileUtil; -import org.ray.util.generator.Composition.TR; +import org.ray.util.generator.Composition.Tr; /** * Generate Rpc.java @@ -29,90 +29,75 @@ private static String build() { sb.append("@SuppressWarnings({\"rawtypes\", \"unchecked\"})\n"); sb.append("class Rpc {\n"); - for (TR tr : Composition.calculate(Share.MAX_T, Share.MAX_R)) { - buildCall(sb, tr.Tcount, tr.Rcount); + for (Tr tr : Composition.calculate(Share.MAX_T, Share.MAX_R)) { + buildCall(sb, tr.tcount, tr.rcount); } sb.append("}\n"); return sb.toString(); } - private static void buildCall(StringBuilder sb, int Tcount, int Rcount) { - for (Set whichTisFuture : whichTisFutureComposition(Tcount)) { - sb.append(buildCall(Tcount, Rcount, whichTisFuture)); + private static void buildCall(StringBuilder sb, int tcount, int rcount) { + for (Set whichTisFuture : whichTisFutureComposition(tcount)) { + sb.append(buildCall(tcount, rcount, whichTisFuture)); } } - /** - * public static RayObject call(RayFunc_1_1 f, RayObject arg) { return - * Ray.runtime().rpc(() -> f.apply(null), arg).objs[0]; } - */ - private static String buildCall(int Tcount, int Rcount, Set whichTisFuture) { + private static String buildCall(int tcount, int rcount, Set whichTisFuture) { StringBuilder sb = new StringBuilder(); - String parameter = (Tcount == 0 ? "" - : ", " + Share.buildParameter(Tcount, "T", whichTisFuture)); - sb.append("\tpublic static <").append(Share.buildClassDeclare(Tcount, Rcount)).append("> ") - .append(Share.buildRpcReturn(Rcount)).append(" call") - .append(Rcount == 1 ? "" : (Rcount <= 0 ? "_n" : ("_" + Rcount))).append("(RayFunc_") - .append(Tcount).append("_") - .append(Rcount <= 0 ? (Rcount == 0 ? "n" : "n_list") : Rcount).append("<") - .append(Share.buildClassDeclare(Tcount, Rcount)).append("> f").append( - Rcount <= 0 ? (Rcount == 0 ? ", Collection returnids" : ", Integer returnCount") + String parameter = (tcount == 0 ? "" + : ", " + Share.buildParameter(tcount, "T", whichTisFuture)); + sb.append("\tpublic static <").append(Share.buildClassDeclare(tcount, rcount)).append("> ") + .append(Share.buildRpcReturn(rcount)).append(" call") + .append(rcount == 1 ? "" : (rcount <= 0 ? "_n" : ("_" + rcount))).append("(RayFunc_") + .append(tcount).append("_") + .append(rcount <= 0 ? (rcount == 0 ? "n" : "n_list") : rcount).append("<") + .append(Share.buildClassDeclare(tcount, rcount)).append("> f").append( + rcount <= 0 ? (rcount == 0 ? ", Collection returnids" : ", Integer returnCount") : "").append(parameter).append(") {\n"); - - /* - * public static RayObject call(RayFunc_0_1 f) { - if (Ray.Parameters().remoteLambda()) { - return Ray.internal().call(RayFunc_0_1.class, f, 1).objs[0]; - } - else { - return Ray.internal().call(() -> f.apply(), 1).objs[0]; - } - } - */ String nulls = Share.buildRepeat("null", - Tcount + (Rcount == 0 ? 1/*for first arg map*/ : 0)); - String parameterUse = (Tcount == 0 ? "" : (", " + Share.buildParameterUse(Tcount, "T"))); + tcount + (rcount == 0 ? 1/*for first arg map*/ : 0)); + String parameterUse = (tcount == 0 ? "" : (", " + Share.buildParameterUse(tcount, "T"))); String labmdaUse = "RayFunc_" - + Tcount + "_" + (Rcount <= 0 ? (Rcount == 0 ? "n" : "n_list") : Rcount) + + tcount + "_" + (rcount <= 0 ? (rcount == 0 ? "n" : "n_list") : rcount) + ".class, f"; sb.append("\t\tif (Ray.Parameters().remoteLambda()) {\n"); - if (Rcount == 1) { + if (rcount == 1) { sb.append("\t\t\treturn Ray.internal().call(null, ").append(labmdaUse).append(", 1") .append(parameterUse).append(").objs[0];") .append("\n"); - } else if (Rcount == 0) { + } else if (rcount == 0) { sb.append("\t\t\treturn Ray.internal().callWithReturnLabels(null, ") .append(labmdaUse).append(", returnids").append(parameterUse).append(");") .append("\n"); - } else if (Rcount < 0) { + } else if (rcount < 0) { sb.append("\t\t\treturn Ray.internal().callWithReturnIndices(null, ") .append(labmdaUse).append(", returnCount").append(parameterUse).append(");") .append("\n"); } else { - sb.append("\t\t\treturn new RayObjects").append(Rcount) - .append("(Ray.internal().call(null, ").append(labmdaUse).append(", ").append(Rcount) + sb.append("\t\t\treturn new RayObjects").append(rcount) + .append("(Ray.internal().call(null, ").append(labmdaUse).append(", ").append(rcount) .append(parameterUse).append(").objs);") .append("\n"); } sb.append("\t\t} else {\n"); - if (Rcount == 1) { + if (rcount == 1) { sb.append("\t\t\treturn Ray.internal().call(null, () -> f.apply(").append(nulls) .append("), 1").append(parameterUse).append(").objs[0];") .append("\n"); - } else if (Rcount == 0) { + } else if (rcount == 0) { sb.append("\t\t\treturn Ray.internal().callWithReturnLabels(null, () -> f.apply(") .append(nulls).append("), returnids").append(parameterUse).append(");") .append("\n"); - } else if (Rcount < 0) { + } else if (rcount < 0) { sb.append("\t\t\treturn Ray.internal().callWithReturnIndices(null, () -> f.apply(") .append(nulls).append("), returnCount").append(parameterUse).append(");") .append("\n"); } else { - sb.append("\t\t\treturn new RayObjects").append(Rcount) + sb.append("\t\t\treturn new RayObjects").append(rcount) .append("(Ray.internal().call(null, () -> f.apply(").append(nulls).append("), ") - .append(Rcount).append(parameterUse).append(").objs);") + .append(rcount).append(parameterUse).append(").objs);") .append("\n"); } sb.append("\t\t}\n"); @@ -120,34 +105,34 @@ private static String buildCall(int Tcount, int Rcount, Set whichTisFut return sb.toString(); } - private static Set> whichTisFutureComposition(int Tcount) { + private static Set> whichTisFutureComposition(int tcount) { Set> ret = new HashSet<>(); - Set N = new HashSet<>(); - for (int k = 0; k < Tcount; k++) { - N.add(k); + Set n = new HashSet<>(); + for (int k = 0; k < tcount; k++) { + n.add(k); } - for (int k = 0; k <= Tcount; k++) { - ret.addAll(CNn(N, k)); + for (int k = 0; k <= tcount; k++) { + ret.addAll(cnn(n, k)); } return ret; } //pick n numbers in N - private static Set> CNn(Set N, int n) { + private static Set> cnn(Set bigN, int n) { C c = new C(); for (int k = 0; k < n; k++) { - c.mul(N); + c.mul(bigN); } - return c.v; + return c.vc; } static class C { - Set> v; + Set> vc; public C() { - v = new HashSet<>(); - v.add(new HashSet<>()); + vc = new HashSet<>(); + vc.add(new HashSet<>()); } void mul(Set ns) { @@ -155,12 +140,12 @@ void mul(Set ns) { for (int n : ns) { ret.addAll(mul(n)); } - this.v = ret; + this.vc = ret; } Set> mul(int n) { Set> ret = new HashSet<>(); - for (Set s : v) { + for (Set s : vc) { if (s.contains(n)) { continue; } diff --git a/java/common/src/main/java/org/ray/util/generator/Share.java b/java/common/src/main/java/org/ray/util/generator/Share.java index 44acf70eb65b..e63376a0b32e 100644 --- a/java/common/src/main/java/org/ray/util/generator/Share.java +++ b/java/common/src/main/java/org/ray/util/generator/Share.java @@ -3,7 +3,7 @@ import java.util.Set; /** - * Share util for generators + * Share util for generators. */ public class Share { @@ -11,20 +11,20 @@ public class Share { public static final int MAX_R = 4; /** - * T0, T1, T2, T3, R + * T0, T1, T2, T3, R. */ - public static String buildClassDeclare(int Tcount, int Rcount) { + public static String buildClassDeclare(int tcount, int rcount) { StringBuilder sb = new StringBuilder(); - for (int k = 0; k < Tcount; k++) { + for (int k = 0; k < tcount; k++) { sb.append("T").append(k).append(", "); } - if (Rcount == 0) { + if (rcount == 0) { sb.append("R, RID"); - } else if (Rcount < 0) { - assert (Rcount == -1); + } else if (rcount < 0) { + assert (rcount == -1); sb.append("R"); } else { - for (int k = 0; k < Rcount; k++) { + for (int k = 0; k < rcount; k++) { sb.append("R").append(k).append(", "); } sb.deleteCharAt(sb.length() - 1); @@ -34,16 +34,16 @@ public static String buildClassDeclare(int Tcount, int Rcount) { } /** - * T0 t0, T1 t1, T2 t2, T3 t3 + * T0 t0, T1 t1, T2 t2, T3 t3. */ - public static String buildParameter(int Tcount, String TR, Set whichTisFuture) { + public static String buildParameter(int tcount, String tr, Set whichTisFuture) { StringBuilder sb = new StringBuilder(); - for (int k = 0; k < Tcount; k++) { + for (int k = 0; k < tcount; k++) { sb.append(whichTisFuture != null && whichTisFuture.contains(k) - ? "RayObject<" + (TR + k) + ">" : (TR + k)).append(" ").append(TR.toLowerCase()) + ? "RayObject<" + (tr + k) + ">" : (tr + k)).append(" ").append(tr.toLowerCase()) .append(k).append(", "); } - if (Tcount > 0) { + if (tcount > 0) { sb.deleteCharAt(sb.length() - 1); sb.deleteCharAt(sb.length() - 1); } @@ -51,77 +51,72 @@ public static String buildParameter(int Tcount, String TR, Set whichTis } /** - * t0, t1, t2 + * t0, t1, t2. */ - public static String buildParameterUse(int Tcount, String TR) { + public static String buildParameterUse(int tcount, String tr) { StringBuilder sb = new StringBuilder(); - for (int k = 0; k < Tcount; k++) { - sb.append(TR.toLowerCase()).append(k).append(", "); + for (int k = 0; k < tcount; k++) { + sb.append(tr.toLowerCase()).append(k).append(", "); } - if (Tcount > 0) { + if (tcount > 0) { sb.deleteCharAt(sb.length() - 1); sb.deleteCharAt(sb.length() - 1); } return sb.toString(); } - public static String buildParameterUse2(int Tcount, int startIndex, String TR, String pre, - String post) { + public static String buildParameterUse2(int tcount, int startIndex, String tr, String pre, + String post) { StringBuilder sb = new StringBuilder(); - for (int k = 0; k < Tcount; k++) { - sb.append("(").append(TR).append(k).append(")").append(pre).append(k + startIndex) + for (int k = 0; k < tcount; k++) { + sb.append("(").append(tr).append(k).append(")").append(pre).append(k + startIndex) .append(post).append(", "); } - if (Tcount > 0) { + if (tcount > 0) { sb.deleteCharAt(sb.length() - 1); sb.deleteCharAt(sb.length() - 1); } return sb.toString(); } - /** - * MultipleReturns2 apply(); R0 - */ - public static String buildFuncReturn(int Rcount) { - if (Rcount == 0) { + public static String buildFuncReturn(int rcount) { + if (rcount == 0) { return "Map"; - } else if (Rcount < 0) { - assert (-1 == Rcount); + } else if (rcount < 0) { + assert (-1 == rcount); return "List"; } - if (Rcount == 1) { + if (rcount == 1) { return "R0"; } StringBuilder sb = new StringBuilder(); - for (int k = 0; k < Rcount; k++) { + for (int k = 0; k < rcount; k++) { sb.append("R").append(k).append(", "); } sb.deleteCharAt(sb.length() - 1); sb.deleteCharAt(sb.length() - 1); - return "MultipleReturns" + Rcount + "<" + sb.toString() + ">"; + return "MultipleReturns" + rcount + "<" + sb.toString() + ">"; } - /** - */ - public static String buildRpcReturn(int Rcount) { - if (Rcount == 0) { + public static String buildRpcReturn(int rcount) { + if (rcount == 0) { return "RayMap"; - } else if (Rcount < 0) { - assert (Rcount == -1); + } else if (rcount < 0) { + assert (rcount == -1); return "RayList"; } - if (Rcount == 1) { + if (rcount == 1) { return "RayObject"; } StringBuilder sb = new StringBuilder(); - for (int k = 0; k < Rcount; k++) { + for (int k = 0; k < rcount; k++) { sb.append("R").append(k).append(", "); } sb.deleteCharAt(sb.length() - 1); sb.deleteCharAt(sb.length() - 1); - return "RayObjects" + Rcount + "<" + sb.toString() + ">"; + return "RayObjects" + rcount + "<" + sb.toString() + ">"; } diff --git a/java/common/src/main/java/org/ray/util/logger/ConsoleLogger.java b/java/common/src/main/java/org/ray/util/logger/ConsoleLogger.java index c3d0c590860d..b04fe5e48f68 100644 --- a/java/common/src/main/java/org/ray/util/logger/ConsoleLogger.java +++ b/java/common/src/main/java/org/ray/util/logger/ConsoleLogger.java @@ -3,7 +3,7 @@ import org.apache.log4j.Logger; /** - * A logger which prints output to console + * A logger which prints output to console. */ public class ConsoleLogger extends Logger { @@ -15,32 +15,32 @@ protected ConsoleLogger(String name, Logger realLogger) { } @Override - public void info(Object log) { - realLogger.info("(" + this.getName() + ") " + log); + public void debug(Object log) { + realLogger.debug("(" + this.getName() + ") " + log); } @Override - public void warn(Object log) { - realLogger.warn("(" + this.getName() + ") " + log); + public void error(Object log) { + realLogger.error("(" + this.getName() + ") " + log); } @Override - public void warn(Object log, Throwable e) { - realLogger.warn("(" + this.getName() + ") " + log, e); + public void error(Object log, Throwable e) { + realLogger.error("(" + this.getName() + ") " + log, e); } @Override - public void debug(Object log) { - realLogger.debug("(" + this.getName() + ") " + log); + public void info(Object log) { + realLogger.info("(" + this.getName() + ") " + log); } @Override - public void error(Object log) { - realLogger.error("(" + this.getName() + ") " + log); + public void warn(Object log) { + realLogger.warn("(" + this.getName() + ") " + log); } @Override - public void error(Object log, Throwable e) { - realLogger.error("(" + this.getName() + ") " + log, e); + public void warn(Object log, Throwable e) { + realLogger.warn("(" + this.getName() + ") " + log, e); } } diff --git a/java/common/src/main/java/org/ray/util/logger/DynamicLog.java b/java/common/src/main/java/org/ray/util/logger/DynamicLog.java index bf308fa00ec3..d8ac18e060bb 100644 --- a/java/common/src/main/java/org/ray/util/logger/DynamicLog.java +++ b/java/common/src/main/java/org/ray/util/logger/DynamicLog.java @@ -7,7 +7,7 @@ import org.ray.util.CommonUtil; /** - * Dynamic logger without properties configuration file + * Dynamic logger without properties configuration file. */ public class DynamicLog { @@ -16,12 +16,11 @@ public class DynamicLog { private static LogLevel logLevel = LogLevel.DEBUG; private static Boolean logLevelSetFlag = false; + private static Map sampleStatis = new ConcurrentHashMap<>(); + private final String key; - /** - * set the context prefix for all logs - */ - public static void setContextPrefix(String prefix) { - PREFIX.set(prefix); + private DynamicLog(String key) { + this.key = key; } public static String getContextPrefix() { @@ -29,7 +28,14 @@ public static String getContextPrefix() { } /** - * set the level for all logs + * set the context prefix for all logs. + */ + public static void setContextPrefix(String prefix) { + PREFIX.set(prefix); + } + + /** + * set the level for all logs. */ public static void setLogLevel(String level) { if (logLevelSetFlag) { /* one shot, avoid the risk of multithreading */ @@ -39,13 +45,17 @@ public static void setLogLevel(String level) { logLevel = LogLevel.of(level); } - private static LogLevel getenumLogLevel() { - return logLevel; + public static DynamicLog registerName(String name) { + return DynamicLogNameRegister.registerName(name); + } + + public static Collection values() { + return DynamicLogNameRegister.names.values(); } @Override - public String toString() { - return this.getKey(); + public int hashCode() { + return this.toString().hashCode(); } @Override @@ -54,19 +64,12 @@ public boolean equals(Object o) { } @Override - public int hashCode() { - return this.toString().hashCode(); + public String toString() { + return this.getKey(); } - private String wrap(String level, String log) { - StackTraceElement stes[] = Thread.currentThread().getStackTrace(); - String ret = "[" + level + "]" + "[" + stes[3].getFileName() + ":" + stes[3].getLineNumber() - + "] - " + (log == null ? "" : log); - String prefix = PREFIX.get(); - if (prefix != null) { - ret = "[" + prefix + "]" + ret; - } - return ret; + public String getKey() { + return this.key; } public void debug(String log) { @@ -74,18 +77,33 @@ public void debug(String log) { return; } log = wrap("debug", log); - Logger loggers[] = DynamicLogManager.getLogs(this); + Logger[] loggers = DynamicLogManager.getLogs(this); for (Logger logger : loggers) { logger.debug(log); } } + private static LogLevel getenumLogLevel() { + return logLevel; + } + + private String wrap(String level, String log) { + StackTraceElement[] stes = Thread.currentThread().getStackTrace(); + String ret = "[" + level + "]" + "[" + stes[3].getFileName() + ":" + stes[3].getLineNumber() + + "] - " + (log == null ? "" : log); + String prefix = PREFIX.get(); + if (prefix != null) { + ret = "[" + prefix + "]" + ret; + } + return ret; + } + public void info(String log) { if (!getenumLogLevel().needLog(LogLevel.INFO)) { return; } log = wrap("info", log); - Logger loggers[] = DynamicLogManager.getLogs(this); + Logger[] loggers = DynamicLogManager.getLogs(this); for (Logger logger : loggers) { logger.info(log); } @@ -96,7 +114,7 @@ public void warn(String log) { return; } log = wrap("warn", log); - Logger loggers[] = DynamicLogManager.getLogs(this); + Logger[] loggers = DynamicLogManager.getLogs(this); for (Logger logger : loggers) { logger.warn(log); } @@ -107,36 +125,36 @@ public void warn(String log, Throwable e) { return; } log = wrap("warn", log); - Logger loggers[] = DynamicLogManager.getLogs(this); + Logger[] loggers = DynamicLogManager.getLogs(this); for (Logger logger : loggers) { logger.warn(log, e); } } - public void error(String log) { + public void error(String log, Throwable e) { if (!getenumLogLevel().needLog(LogLevel.ERROR)) { return; } log = wrap("error", log); - Logger loggers[] = DynamicLogManager.getLogs(this); + if (e == null) { + error(log); + return; + } + + Logger[] loggers = DynamicLogManager.getLogs(this); for (Logger logger : loggers) { - logger.error(log); + logger.error(log, e); } } - public void error(String log, Throwable e) { + public void error(String log) { if (!getenumLogLevel().needLog(LogLevel.ERROR)) { return; } log = wrap("error", log); - if (e == null) { - error(log); - return; - } - - Logger loggers[] = DynamicLogManager.getLogs(this); + Logger[] loggers = DynamicLogManager.getLogs(this); for (Logger logger : loggers) { - logger.error(log, e); + logger.error(log); } } @@ -149,53 +167,21 @@ public void error(Throwable e) { error(log); return; } - Logger loggers[] = DynamicLogManager.getLogs(this); + Logger[] loggers = DynamicLogManager.getLogs(this); for (Logger logger : loggers) { logger.error(log, e); } } - //statistic for sampling - private static class SampleStatis { - - int total; - - public boolean gamble() { - int randomRange; - - if (total < 100) { - randomRange = 1; - } else if (total < 1000) { - randomRange = 1000; - } else if (total < 100000) { - randomRange = 10000; - } else if (total < 1000000) { - randomRange = 100000; - } else { - total = 0; - randomRange = 1; - } - if (CommonUtil.getRandom(randomRange) == 0) { - total++; - return true; - } else { - total++; - return false; - } - } - } - - private static Map sampleStatis = new ConcurrentHashMap<>(); - /** - * Print sample error log + * Print sample error log. */ public boolean sampleError(Object sampleKeyO, String log, Throwable e) { String sampleKey = sampleKeyO.toString(); try { SampleStatis ss = sampleStatis.computeIfAbsent(sampleKey, k -> new SampleStatis()); if (ss.gamble()) { - Logger loggers[] = DynamicLogManager.getLogs(this); + Logger[] loggers = DynamicLogManager.getLogs(this); for (Logger logger : loggers) { if (e != null) { logger.error("[" + sampleKey + "] - " + log, e); @@ -214,26 +200,38 @@ public boolean sampleError(Object sampleKeyO, String log, Throwable e) { } } - private final String key; - - private DynamicLog(String key) { - this.key = key; - } - - public String getKey() { - return this.key; - } - public String getDefaultLogFileName() { return this.key + ".log"; } - public static DynamicLog registerName(String name) { - return DynamicLogNameRegister.registerName(name); - } + //statistic for sampling + private static class SampleStatis { - public static Collection values() { - return DynamicLogNameRegister.names.values(); + int total; + + public boolean gamble() { + int randomRange; + + if (total < 100) { + randomRange = 1; + } else if (total < 1000) { + randomRange = 1000; + } else if (total < 100000) { + randomRange = 10000; + } else if (total < 1000000) { + randomRange = 100000; + } else { + total = 0; + randomRange = 1; + } + if (CommonUtil.getRandom(randomRange) == 0) { + total++; + return true; + } else { + total++; + return false; + } + } } public static class DynamicLogNameRegister { diff --git a/java/common/src/main/java/org/ray/util/logger/DynamicLogManager.java b/java/common/src/main/java/org/ray/util/logger/DynamicLogManager.java index d684e7a0e883..f320afe1ba24 100644 --- a/java/common/src/main/java/org/ray/util/logger/DynamicLogManager.java +++ b/java/common/src/main/java/org/ray/util/logger/DynamicLogManager.java @@ -11,27 +11,26 @@ import org.ray.util.SystemUtil; /** - * Manager for dynamic loggers + * Manager for dynamic loggers. */ public class DynamicLogManager { + protected static final String DAY_DATE_PATTERN = "'.'yyyy-MM-dd"; + private static final int LOG_CACHE_SIZE = 32 * 1024; + // private final static String HOUR_DATE_PATTERN = "' + // .'yyyy-MM-dd_HH"; + // private final static String GBK = "GBK"; + private static final String DAILY_APPENDER_NAME = "_DAILY_APPENDER_NAME"; + // private final static String CONSOLE_APPENDER_NAME = + // "_CONSOLE_APPENDER_NAME"; + private static final String LAYOUT_PATTERN = "%d [%t]%m%n"; + private static final ConcurrentHashMap loggers = new ConcurrentHashMap<>(); //whether to print the log on std(ie. console) public static boolean printOnStd = false; //the root directory of log files public static String logsDir; public static String logsSuffix; public static Level level = Level.DEBUG; //Level.INFO; - /** */ - private static final int LOG_CACHE_SIZE = 32 * 1024; - protected final static String DAY_DATE_PATTERN = "'.'yyyy-MM-dd"; - // private final static String HOUR_DATE_PATTERN = "'.'yyyy-MM-dd_HH"; - // private final static String GBK = "GBK"; - private final static String DAILY_APPENDER_NAME = "_DAILY_APPENDER_NAME"; - // private final static String CONSOLE_APPENDER_NAME = "_CONSOLE_APPENDER_NAME"; - private final static String LAYOUT_PATTERN = "%d [%t]%m%n"; - - private static final ConcurrentHashMap loggers = new ConcurrentHashMap<>(); - private static int MAX_FILE_NUM = 10; private static String MAX_FILE_SIZE = "500MB"; @@ -98,13 +97,13 @@ public static Logger[] getLogs(DynamicLog dynLog) { } } } - return new Logger[]{logger}; + return new Logger[] {logger}; } private static Logger initLogger(DynamicLog dynLog) { if (printOnStd) { Logger reallogger = Logger.getLogger(dynLog.getKey()); - ConsoleLogger logger = new ConsoleLogger(dynLog.getKey(), reallogger); + final ConsoleLogger logger = new ConsoleLogger(dynLog.getKey(), reallogger); PatternLayout layout = new PatternLayout(LAYOUT_PATTERN); ConsoleAppender appender = new ConsoleAppender(layout, ConsoleAppender.SYSTEM_OUT); reallogger.removeAllAppenders(); @@ -122,7 +121,7 @@ private static Logger initLogger(DynamicLog dynLog) { } protected static Logger makeLogger(String loggerName, String filename) { - Logger logger = Logger.getLogger(loggerName); + final Logger logger = Logger.getLogger(loggerName); PatternLayout layout = new PatternLayout(LAYOUT_PATTERN); File dir = new File(logsDir); if (!dir.exists()) { diff --git a/java/common/src/main/java/org/ray/util/logger/RayLog.java b/java/common/src/main/java/org/ray/util/logger/RayLog.java index 62a8bfa54c7a..3a71a56c6e04 100644 --- a/java/common/src/main/java/org/ray/util/logger/RayLog.java +++ b/java/common/src/main/java/org/ray/util/logger/RayLog.java @@ -1,17 +1,17 @@ package org.ray.util.logger; /** - * Dynamic loggers in Ray + * Dynamic loggers in Ray. */ public class RayLog { /** - * for ray itself + * for ray itself. */ public static final DynamicLog core = DynamicLog.registerName("core"); /** - * for ray's app's log + * for ray's app's log. */ public static DynamicLog rapp = core; //DynamicLog.registerName("rapp"); } diff --git a/java/common/src/main/java/org/ray/util/logger/TimedFlushDailyRollingFileAppender.java b/java/common/src/main/java/org/ray/util/logger/TimedFlushDailyRollingFileAppender.java index 282c4d131370..d0a4c3257932 100644 --- a/java/common/src/main/java/org/ray/util/logger/TimedFlushDailyRollingFileAppender.java +++ b/java/common/src/main/java/org/ray/util/logger/TimedFlushDailyRollingFileAppender.java @@ -7,11 +7,15 @@ import org.apache.log4j.RollingFileAppender; /** - * Normal log appender + * Normal log appender. */ public class TimedFlushDailyRollingFileAppender extends RollingFileAppender { - private final static Set all = new HashSet<>(); + private static final Set all = new HashSet<>(); + + static { + new TimedFlushLogThread().start(); + } public TimedFlushDailyRollingFileAppender() { super(); @@ -27,10 +31,6 @@ public TimedFlushDailyRollingFileAppender(Layout layout, String filename) throws } } - static { - new TimedFlushLogThread().start(); - } - private void flush() { try { if (!checkEntryConditions()) { diff --git a/java/example/pom.xml b/java/example/pom.xml index d6f253f91a0d..705711dd37da 100644 --- a/java/example/pom.xml +++ b/java/example/pom.xml @@ -1,102 +1,102 @@ - - - org.ray.parent - ray-superpom - 1.0 - - 4.0.0 + + + org.ray.parent + ray-superpom + 1.0 + + 4.0.0 - org.ray - ray-example + org.ray + ray-example - java example for ray - java example for ray - + java example for ray + java example for ray + - jar + jar - - - org.ray - ray-api - 1.0 - + + + org.ray + ray-api + 1.0 + - - org.ray - ray-runtime-common - 1.0 - + + org.ray + ray-runtime-common + 1.0 + - - org.ray - ray-runtime-native - 1.0 - + + org.ray + ray-runtime-native + 1.0 + - - org.ray - ray-runtime-dev - 1.0 - + + org.ray + ray-runtime-dev + 1.0 + - - org.ray - ray-common - 1.0 - + + org.ray + ray-common + 1.0 + - - org.ray - ray-hook - 1.0 - - - - - - org.apache.maven.plugins - maven-surefire-plugin - 2.20.1 - - - ${basedir}/../ray.config.ini - - -ea - -Djava.library.path=${basedir}/../../build/src/plasma:${basedir}/../../build/src/local_scheduler - -noverify - -DlogOutput=console - - ${basedir}/src/main/java/ - ${project.build.directory}/classes/ - - + + org.ray + ray-hook + 1.0 + + + + + + org.apache.maven.plugins + maven-surefire-plugin + 2.20.1 + + + ${basedir}/../ray.config.ini + + -ea + -Djava.library.path=${basedir}/../../build/src/plasma:${basedir}/../../build/src/local_scheduler + -noverify + -DlogOutput=console + + ${basedir}/src/main/java/ + ${project.build.directory}/classes/ + + - - org.apache.maven.plugins - maven-dependency-plugin - - - copy-dependencies - package - - copy-dependencies - - - ${basedir}/lib - false - false - true - - - - + + org.apache.maven.plugins + maven-dependency-plugin + + + copy-dependencies + package + + copy-dependencies + + + ${basedir}/lib + false + false + true + + + + - - + + diff --git a/java/example/src/main/java/org/ray/example/HelloWorld.java b/java/example/src/main/java/org/ray/example/HelloWorld.java index bdfaef068ac6..632ea5517943 100644 --- a/java/example/src/main/java/org/ray/example/HelloWorld.java +++ b/java/example/src/main/java/org/ray/example/HelloWorld.java @@ -1,22 +1,15 @@ package org.ray.example; +import java.io.Serializable; import org.ray.api.Ray; import org.ray.api.RayObject; import org.ray.api.RayRemote; import org.ray.core.RayRuntime; import org.ray.util.logger.RayLog; -import java.io.Serializable; - -public class HelloWorld implements Serializable{ +public class HelloWorld implements Serializable { - public static String sayHelloWorld() { - RayObject hello = Ray.call(HelloWorld::sayHello); - RayObject world = Ray.call(HelloWorld::sayWorld); - return Ray.call(HelloWorld::merge, hello, world).get(); - } - @RayRemote public static String sayHello() { String ret = "he"; @@ -39,18 +32,22 @@ public static String merge(String hello, String world) { public static void main(String[] args) throws Exception { try { - Ray.init(); - String helloWorld = HelloWorld.sayHelloWorld(); - RayLog.rapp.info(helloWorld); - assert helloWorld.equals("hello,world!"); - } - catch (Throwable t) { + Ray.init(); + String helloWorld = HelloWorld.sayHelloWorld(); + RayLog.rapp.info(helloWorld); + assert helloWorld.equals("hello,world!"); + } catch (Throwable t) { t.printStackTrace(); - } - finally { + } finally { RayRuntime.getInstance().cleanUp(); } } + + public static String sayHelloWorld() { + RayObject hello = Ray.call(HelloWorld::sayHello); + RayObject world = Ray.call(HelloWorld::sayWorld); + return Ray.call(HelloWorld::merge, hello, world).get(); + } } diff --git a/java/hook/pom.xml b/java/hook/pom.xml index b45a1c9ec482..60c8bbdde864 100644 --- a/java/hook/pom.xml +++ b/java/hook/pom.xml @@ -1,88 +1,88 @@ - - - org.ray.parent - ray-superpom - 1.0 - - 4.0.0 + + + org.ray.parent + ray-superpom + 1.0 + + 4.0.0 - org.ray - ray-hook - java api hook for ray - java api hook for ray - + org.ray + ray-hook + java api hook for ray + java api hook for ray + - jar + jar - + - - - org.ow2.asm - asm - 6.0 - + + + org.ow2.asm + asm + 6.0 + - - commons-codec - commons-codec - 1.4 - + + commons-codec + commons-codec + 1.4 + - - commons-io - commons-io - 2.5 - + + commons-io + commons-io + 2.5 + - - org.ray - ray-common - 1.0 - + + org.ray + ray-common + 1.0 + - + - - - - maven-jar-plugin - 2.5 - - - - org.ray.hook.Agent - true - - - - + + + + maven-jar-plugin + 2.5 + + + + org.ray.hook.Agent + true + + + + - - maven-shade-plugin - 3.1.0 - - - package - - shade - - - - - org.objectweb.asm - agent.org.objectweb.asm - - - false - - - - - - + + maven-shade-plugin + 3.1.0 + + + package + + shade + + + + + org.objectweb.asm + agent.org.objectweb.asm + + + false + + + + + + diff --git a/java/hook/src/main/java/org/ray/hook/ClassAdapter.java b/java/hook/src/main/java/org/ray/hook/ClassAdapter.java index b8fa1662f077..5099b30343c2 100644 --- a/java/hook/src/main/java/org/ray/hook/ClassAdapter.java +++ b/java/hook/src/main/java/org/ray/hook/ClassAdapter.java @@ -6,21 +6,10 @@ public class ClassAdapter { - public static class Result { - - public byte[] classBuffer; - public Set changedMethods; - } - public static Result hookClass(ClassLoader loader, String className, byte[] classfileBuffer) { // we have to comment out this quick filter as this is not accurate // e.g., org/ray/api/test/ActorTest$Adder.class is skipped!!! // even worse, this is non-deterministic... - /* - if (detectBody.contains("org/ray/hook/")) { - return classfileBuffer; - } - */ ClassReader reader = new ClassReader(classfileBuffer); ClassWriter writer = new ClassWriter(reader, 0); @@ -46,4 +35,10 @@ public static Result hookClass(ClassLoader loader, String className, byte[] clas rr.classBuffer = result; return rr; } + + public static class Result { + + public byte[] classBuffer; + public Set changedMethods; + } } diff --git a/java/hook/src/main/java/org/ray/hook/ClassDetectVisitor.java b/java/hook/src/main/java/org/ray/hook/ClassDetectVisitor.java index f733fc71495b..ea5f10d61a7e 100644 --- a/java/hook/src/main/java/org/ray/hook/ClassDetectVisitor.java +++ b/java/hook/src/main/java/org/ray/hook/ClassDetectVisitor.java @@ -11,20 +11,16 @@ import org.objectweb.asm.Type; /** - * rewrite phase 1 + * rewrite phase 1. */ public class ClassDetectVisitor extends ClassVisitor { + static int count = 0; final String className; final Set rayMethods = new HashSet<>(); + final ClassLoader loader; boolean isActor = false; - static int count = 0; int actorCalls = 0; - final ClassLoader loader; - - public int actorCalls() { - return actorCalls; - } public ClassDetectVisitor(ClassLoader loader, ClassVisitor origin, String className) { super(Opcodes.ASM6, origin); @@ -32,15 +28,12 @@ public ClassDetectVisitor(ClassLoader loader, ClassVisitor origin, String classN this.loader = loader; } - public Set detectedMethods() { - return rayMethods; + public int actorCalls() { + return actorCalls; } - @Override - public void visitInnerClass(String name, String outerName, - String innerName, int access) { - // System.err.println("visist inner class " + outerName + "$" + innerName); - super.visitInnerClass(name, outerName, innerName, access); + public Set detectedMethods() { + return rayMethods; } @Override @@ -51,20 +44,16 @@ public AnnotationVisitor visitAnnotation(String desc, boolean visible) { return super.visitAnnotation(desc, visible); } - private void visitRayMethod(int access, String name, String mdesc) { - if (name.equals("")) { - return; - } - - MethodId m = new MethodId(className, name, mdesc, (access & Opcodes.ACC_STATIC) != 0, loader); - rayMethods.add(m); - //System.err.println("Visit " + m.toString()); - count++; + @Override + public void visitInnerClass(String name, String outerName, + String innerName, int access) { + // System.err.println("visist inner class " + outerName + "$" + innerName); + super.visitInnerClass(name, outerName, innerName, access); } @Override public MethodVisitor visitMethod(int access, String name, String mdesc, String signature, - String[] exceptions) { + String[] exceptions) { //System.out.println("Visit " + className + "." + name); if (isActor && (access & Opcodes.ACC_PUBLIC) != 0) { visitRayMethod(access, name, mdesc); @@ -81,41 +70,9 @@ public AnnotationVisitor visitAnnotation(String adesc, boolean visible) { return super.visitAnnotation(adesc, visible); } - private boolean isValidCallParameterOrReturnType(Type t) { - if (t.equals(Type.VOID_TYPE)) { - return false; - } - if (t.equals(Type.BOOLEAN_TYPE)) { - return false; - } - if (t.equals(Type.CHAR_TYPE)) { - return false; - } - if (t.equals(Type.BYTE_TYPE)) { - return false; - } - if (t.equals(Type.SHORT_TYPE)) { - return false; - } - if (t.equals(Type.INT_TYPE)) { - return false; - } - if (t.equals(Type.FLOAT_TYPE)) { - return false; - } - if (t.equals(Type.LONG_TYPE)) { - return false; - } - if (t.equals(Type.DOUBLE_TYPE)) { - return false; - } - - return true; - } - @Override public void visitInvokeDynamicInsn(String name, String desc, Handle bsm, - Object... bsmArgs) { + Object... bsmArgs) { // fix all actor calls from InvokeVirtual to InvokeStatic if (desc.contains("org/ray/api/funcs/RayFunc_")) { @@ -151,7 +108,8 @@ public void visitInvokeDynamicInsn(String name, String desc, Handle bsm, dsptr, h.isInterface()); bsmArgs[i] = newh; - //System.err.println("Change ray.call from " + h + " -> " + newh + ", isInterface = " + h.isInterface()); + //System.err.println("Change ray.call from " + h + " -> " + newh + ", isInterface + // = " + h.isInterface()); ++actorCalls; } } @@ -159,6 +117,49 @@ public void visitInvokeDynamicInsn(String name, String desc, Handle bsm, } super.visitInvokeDynamicInsn(name, desc, bsm, bsmArgs); } + + private boolean isValidCallParameterOrReturnType(Type t) { + if (t.equals(Type.VOID_TYPE)) { + return false; + } + if (t.equals(Type.BOOLEAN_TYPE)) { + return false; + } + if (t.equals(Type.CHAR_TYPE)) { + return false; + } + if (t.equals(Type.BYTE_TYPE)) { + return false; + } + if (t.equals(Type.SHORT_TYPE)) { + return false; + } + if (t.equals(Type.INT_TYPE)) { + return false; + } + if (t.equals(Type.FLOAT_TYPE)) { + return false; + } + if (t.equals(Type.LONG_TYPE)) { + return false; + } + if (t.equals(Type.DOUBLE_TYPE)) { + return false; + } + + return true; + } }; } + + private void visitRayMethod(int access, String name, String mdesc) { + if (name.equals("")) { + return; + } + + MethodId m = new MethodId(className, name, mdesc, (access & Opcodes.ACC_STATIC) != 0, loader); + rayMethods.add(m); + //System.err.println("Visit " + m.toString()); + count++; + } } diff --git a/java/hook/src/main/java/org/ray/hook/ClassOverrideVisitor.java b/java/hook/src/main/java/org/ray/hook/ClassOverrideVisitor.java index 58024d6982f1..9bb703e3055d 100644 --- a/java/hook/src/main/java/org/ray/hook/ClassOverrideVisitor.java +++ b/java/hook/src/main/java/org/ray/hook/ClassOverrideVisitor.java @@ -7,75 +7,33 @@ import org.objectweb.asm.Opcodes; /** - * rewrite phase 2 + * rewrite phase 2. */ public class ClassOverrideVisitor extends ClassVisitor { final String className; final Set rayRemoteMethods; - MethodVisitor ClinitVisitor; - - // init the static added field in - // static { - // assign value to _hashOf_XXX - // } - class StaticBlockVisitor extends MethodVisitor { - - StaticBlockVisitor(MethodVisitor mv) { - super(Opcodes.ASM6, mv); - } - - @Override - public void visitCode() { - super.visitCode(); - - // assign value for added hash fields within - for (MethodId m : rayRemoteMethods) { - byte[] hash = m.getSha1Hash(); - insertByteArray(hash); - mv.visitFieldInsn(Opcodes.PUTSTATIC, className, m.getStaticHashValueFieldName(), "[B"); - - System.out.println("assign field: " + m.getStaticHashValueFieldName() + " = " + MethodId - .toHexHashString(hash)); - } - } - - private void insertByteArray(byte[] bytes) { - int length = bytes.length; - assert (length < Short.MAX_VALUE); - mv.visitIntInsn(Opcodes.SIPUSH, length); - mv.visitIntInsn(Opcodes.NEWARRAY, Opcodes.T_BYTE); - mv.visitInsn(Opcodes.DUP); - for (int i = 0; i < length; ++i) { - mv.visitIntInsn(Opcodes.BIPUSH, i); - mv.visitIntInsn(Opcodes.BIPUSH, bytes[i]); - mv.visitInsn(Opcodes.BASTORE); - if (i < (length - 1)) { - mv.visitInsn(Opcodes.DUP); - } - } - } - - } + MethodVisitor clinitVisitor; public ClassOverrideVisitor(ClassVisitor origin, String className, - Set rayRemoteMethods) { + Set rayRemoteMethods) { super(Opcodes.ASM6, origin); this.className = className; this.rayRemoteMethods = rayRemoteMethods; - this.ClinitVisitor = null; + this.clinitVisitor = null; } @Override public MethodVisitor visitMethod(int access, String name, String desc, String signature, - String[] exceptions) { - if ("".equals(name) && ClinitVisitor == null) { + String[] exceptions) { + if ("".equals(name) && clinitVisitor == null) { MethodVisitor mv = super.visitMethod(access, name, desc, signature, exceptions); - ClinitVisitor = new StaticBlockVisitor(mv); - return ClinitVisitor;// dispatch the ASM modifications (assign values to the preComputedxxx static field) to the ClinitVisitor + clinitVisitor = new StaticBlockVisitor(mv); + return clinitVisitor;// dispatch the ASM modifications (assign values to the preComputedxxx + // static field) to the clinitVisitor } - ClassVisitor this_ = this; + ClassVisitor current = this; MethodId m = new MethodId(className, name, desc, (access & Opcodes.ACC_STATIC) != 0, null); if (rayRemoteMethods.contains(m)) { if (m.isStaticMethod()) { @@ -86,10 +44,11 @@ public void visitCode() { // step 1: add a field for the function id of this method System.out.println("add field: " + m.getStaticHashValueFieldName()); String fieldName = m.getStaticHashValueFieldName(); - this_.visitField(Opcodes.ACC_PUBLIC | Opcodes.ACC_STATIC, fieldName, "[B", + current.visitField(Opcodes.ACC_PUBLIC | Opcodes.ACC_STATIC, fieldName, "[B", null, null); - // step 2: rewrite current method so if MethodSwitcher returns true, returns the added function id directly + // step 2: rewrite current method so if MethodSwitcher returns true, returns the + // added function id directly // else call the original method mv.visitFieldInsn(Opcodes.GETSTATIC, className, fieldName, "[B"); mv.visitMethodInsn(Opcodes.INVOKESTATIC, "org/ray/hook/runtime/MethodSwitcher", @@ -103,10 +62,7 @@ public void visitCode() { mv.visitCode();// real work } }; - } - - // non-static - else { + } else { // non-static return super.visitMethod(access, name, desc, signature, exceptions); } } else { @@ -116,7 +72,7 @@ public void visitCode() { @Override public void visitEnd() { - if (ClinitVisitor == null) { // works fine + if (clinitVisitor == null) { // works fine // Create an empty static block and let our method // visitor modify it the same way it modifies an // existing static block @@ -163,14 +119,6 @@ public void visitEnd() { org.objectweb.asm.Type[] args = org.objectweb.asm.Type .getArgumentTypes(mid.getIdMethodDesc()); int argCount = args.length; - /* - for (int i = 0; i < argCount; ++i) { - String ldsptr = args[i].getDescriptor(); - if (!ldsptr.endsWith(";")) - ldsptr = "L" + ldsptr + ";"; - mv.visitLocalVariable("arg" + i, ldsptr, null, l0, l2, i); - } - */ mv.visitMaxs(2, argCount); mv.visitEnd(); } @@ -220,4 +168,47 @@ private String sayReturnType(String str) { return str.substring(left); } + // init the static added field in + // static { + // assign value to _hashOf_XXX + // } + class StaticBlockVisitor extends MethodVisitor { + + StaticBlockVisitor(MethodVisitor mv) { + super(Opcodes.ASM6, mv); + } + + @Override + public void visitCode() { + super.visitCode(); + + // assign value for added hash fields within + for (MethodId m : rayRemoteMethods) { + byte[] hash = m.getSha1Hash(); + insertByteArray(hash); + mv.visitFieldInsn(Opcodes.PUTSTATIC, className, m.getStaticHashValueFieldName(), "[B"); + + System.out.println("assign field: " + m.getStaticHashValueFieldName() + " = " + MethodId + .toHexHashString(hash)); + } + } + + private void insertByteArray(byte[] bytes) { + int length = bytes.length; + assert (length < Short.MAX_VALUE); + mv.visitIntInsn(Opcodes.SIPUSH, length); + mv.visitIntInsn(Opcodes.NEWARRAY, Opcodes.T_BYTE); + mv.visitInsn(Opcodes.DUP); + for (int i = 0; i < length; ++i) { + mv.visitIntInsn(Opcodes.BIPUSH, i); + mv.visitIntInsn(Opcodes.BIPUSH, bytes[i]); + mv.visitInsn(Opcodes.BASTORE); + if (i < (length - 1)) { + mv.visitInsn(Opcodes.DUP); + } + } + } + + } + } diff --git a/java/hook/src/main/java/org/ray/hook/JarRewriter.java b/java/hook/src/main/java/org/ray/hook/JarRewriter.java index 6dcce96d1557..0221f1af0ecf 100644 --- a/java/hook/src/main/java/org/ray/hook/JarRewriter.java +++ b/java/hook/src/main/java/org/ray/hook/JarRewriter.java @@ -6,7 +6,6 @@ import java.io.IOException; import java.io.PrintWriter; import java.lang.reflect.Method; -import java.security.NoSuchAlgorithmException; import java.util.ArrayList; import java.util.Collection; import java.util.Enumeration; @@ -26,7 +25,7 @@ import org.ray.util.logger.RayLog; /** - * rewrite jars to new jars with methods marked using Ray annotations + * rewrite jars to new jars with methods marked using Ray annotations. */ public class JarRewriter { @@ -51,10 +50,32 @@ public static void main(String[] args) rewrite(args[0], args[1]); } + public static LoadedFunctions load(String dir, String baseDir) + throws FileNotFoundException, SecurityException { + List functions = JarRewriter.getRewrittenFunctions(dir); + LoadedFunctions efuncs = new LoadedFunctions(); + efuncs.loader = JarLoader.loadJars(dir, false); + + for (String func : functions) { + MethodId mid = new MethodId(func, efuncs.loader); + efuncs.functions.add(mid); + } + + if (baseDir != null && !baseDir.equals("")) { + List baseFunctions = JarRewriter.getRewrittenFunctions(baseDir); + for (String func : baseFunctions) { + MethodId mid = new MethodId(func, efuncs.loader); + efuncs.functions.add(mid); + } + } + + return efuncs; + } + public static void rewrite(String fromDir, String toDir) throws IOException, DataFormatException { File fromDirFile = new File(fromDir); File toDirFileTmp = new File(toDir + ".tmp"); - File toDirFile = new File(toDir); + final File toDirFile = new File(toDir); File[] topFiles = fromDirFile.listFiles(); if (topFiles.length != 1 || !topFiles[0].isDirectory()) { @@ -103,57 +124,6 @@ public static void rewrite(String fromDir, String toDir) throws IOException, Dat FileUtils.moveDirectory(toDirFileTmp, toDirFile); } - public static LoadedFunctions load(String dir, String baseDir) - throws FileNotFoundException, SecurityException { - List functions = JarRewriter.getRewrittenFunctions(dir); - LoadedFunctions efuncs = new LoadedFunctions(); - efuncs.loader = JarLoader.loadJars(dir, false); - - for (String func : functions) { - MethodId mid = new MethodId(func, efuncs.loader); - efuncs.functions.add(mid); - } - - if (baseDir != null && !baseDir.equals("")) { - List baseFunctions = JarRewriter.getRewrittenFunctions(baseDir); - for (String func : baseFunctions) { - MethodId mid = new MethodId(func, efuncs.loader); - efuncs.functions.add(mid); - } - } - - return efuncs; - } - - public static LoadedFunctions loadBase(String baseDir) - throws FileNotFoundException, SecurityException { - List functions = JarRewriter.getRewrittenFunctions(baseDir); - LoadedFunctions efuncs = new LoadedFunctions(); - efuncs.loader = null; - - for (String func : functions) { - MethodId mid = new MethodId(func, efuncs.loader); - efuncs.functions.add(mid); - } - - return efuncs; - } - - public static List getRewrittenFunctions(String rewrittenDir) - throws FileNotFoundException { - ArrayList functions = new ArrayList<>(); - Scanner s = new Scanner(new File(rewrittenDir + "/" + FUNCTIONS_FILE)); - while (s.hasNext()) { - String f = s.next(); - if (!f.startsWith("(")) { - functions.add(f); - } - } - s.close(); - - return functions; - } - public static void rewrite(JarFile from, String to, BiConsumer consumer) throws IOException { @@ -170,7 +140,8 @@ public static void rewrite(JarFile from, String to, BiConsumer getRewrittenFunctions(String rewrittenDir) + throws FileNotFoundException { + ArrayList functions = new ArrayList<>(); + Scanner s = new Scanner(new File(rewrittenDir + "/" + FUNCTIONS_FILE)); + while (s.hasNext()) { + String f = s.next(); + if (!f.startsWith("(")) { + functions.add(f); + } + } + s.close(); + + return functions; + } + + public static LoadedFunctions loadBase(String baseDir) + throws FileNotFoundException, SecurityException { + List functions = JarRewriter.getRewrittenFunctions(baseDir); + LoadedFunctions efuncs = new LoadedFunctions(); + efuncs.loader = null; + + for (String func : functions) { + MethodId mid = new MethodId(func, efuncs.loader); + efuncs.functions.add(mid); + } + + return efuncs; + } } diff --git a/java/hook/src/main/java/org/ray/hook/MethodId.java b/java/hook/src/main/java/org/ray/hook/MethodId.java index dd8e9e2efb91..bee6e86e2fac 100644 --- a/java/hook/src/main/java/org/ray/hook/MethodId.java +++ b/java/hook/src/main/java/org/ray/hook/MethodId.java @@ -11,27 +11,26 @@ import org.ray.util.logger.RayLog; /** - * Represent a Method in a Class + * Represent a Method in a Class. */ public class MethodId { + static final String getFunctionIdPostfix = "_function_id"; String className; String methodName; String methodDesc; boolean isStatic; ClassLoader loader; - static final String getFunctionIdPostfix = "_function_id"; - - public MethodId(String cls, String method, String mdesc, boolean isstatic, ClassLoader loader_) { + public MethodId(String cls, String method, String mdesc, boolean isstatic, ClassLoader loader) { className = cls; methodName = method; methodDesc = mdesc; isStatic = isstatic; - loader = loader_; + this.loader = loader; } - public MethodId(String encodedString, ClassLoader loader_) { + public MethodId(String encodedString, ClassLoader loader) { // className + "." + methodName + "::" + methodDesc + "&&" + isStatic; int lastPos3 = encodedString.lastIndexOf("&&"); int lastPos2 = encodedString.lastIndexOf("::"); @@ -45,7 +44,24 @@ public MethodId(String encodedString, ClassLoader loader_) { methodName = encodedString.substring(lastPos1 + ".".length(), lastPos2); methodDesc = encodedString.substring(lastPos2 + "::".length(), lastPos3); isStatic = Boolean.parseBoolean(encodedString.substring(lastPos3 + "&&".length())); - loader = loader_; + this.loader = loader; + } + + public static String toHexHashString(byte[] id) { + String s = ""; + String hex = "0123456789abcdef"; + assert (id.length == 20); + for (int i = 0; i < 20; i++) { + int val = id[i] & 0xff; + s += hex.charAt(val >> 4); + s += hex.charAt(val & 0xf); + } + return s; + } + + private String toHexHashString() { + byte[] id = this.getSha1Hash(); + return toHexHashString(id); } public String getClassName() { @@ -76,23 +92,6 @@ public String getIdMethodDesc() { return "(L" + this.className + ";" + this.methodDesc.substring(1); } - public static String toHexHashString(byte[] id) { - String s = ""; - String hex = "0123456789abcdef"; - assert (id.length == 20); - for (int i = 0; i < 20; i++) { - int val = id[i] & 0xff; - s += hex.charAt(val >> 4); - s += hex.charAt(val & 0xf); - } - return s; - } - - private String toHexHashString() { - byte[] id = this.getSha1Hash(); - return toHexHashString(id); - } - public Method load() { String loadClsName = className.replace('/', '.'); Class cls; @@ -153,7 +152,8 @@ public Method load() { : "") + " vs id-hash: " + toHexHashString()); } } - } catch (NoSuchFieldException | SecurityException | IllegalArgumentException | IllegalAccessException e) { + } catch (NoSuchFieldException | SecurityException | IllegalArgumentException + | IllegalAccessException e) { RayLog.core.error("load method hash field failed for " + toString(), e); } return m; diff --git a/java/hook/src/main/java/org/ray/hook/runtime/JarLoader.java b/java/hook/src/main/java/org/ray/hook/runtime/JarLoader.java index ffbc9d1d0eed..069fdfb857f3 100644 --- a/java/hook/src/main/java/org/ray/hook/runtime/JarLoader.java +++ b/java/hook/src/main/java/org/ray/hook/runtime/JarLoader.java @@ -17,13 +17,13 @@ import org.ray.util.logger.RayLog; /** - * load and unload jars from a dir + * load and unload jars from a dir. */ public class JarLoader { - private static Method AddUrl = InitAddUrl(); + private static Method AddUrl = initAddUrl(); - private static Method InitAddUrl() { + private static Method initAddUrl() { try { Method m = URLClassLoader.class.getDeclaredMethod("addURL", URL.class); m.setAccessible(true); @@ -59,10 +59,6 @@ public static ClassLoader loadJars(String[] appJars, boolean explicitLoadForHook return loadJar(jars, explicitLoadForHook); } - public static void unloadJars(ClassLoader loader) { - // TODO: - } - private static URLClassLoader loadJar(Collection appJars, boolean explicitLoadForHook) { List jars = new ArrayList<>(); List urls = new ArrayList<>(); @@ -128,4 +124,8 @@ private static void loadJarDir(String jarDir, List jars) { jars.addAll(files); } + + public static void unloadJars(ClassLoader loader) { + // TODO: + } } diff --git a/java/hook/src/main/java/org/ray/hook/runtime/LoadedFunctions.java b/java/hook/src/main/java/org/ray/hook/runtime/LoadedFunctions.java index ec418c53dc30..0de1931935a4 100644 --- a/java/hook/src/main/java/org/ray/hook/runtime/LoadedFunctions.java +++ b/java/hook/src/main/java/org/ray/hook/runtime/LoadedFunctions.java @@ -7,6 +7,6 @@ public class LoadedFunctions { - public ClassLoader loader = null; public final Set functions = Collections.synchronizedSet(new HashSet<>()); + public ClassLoader loader = null; } diff --git a/java/hook/src/main/java/org/ray/hook/runtime/MethodHash.java b/java/hook/src/main/java/org/ray/hook/runtime/MethodHash.java index 873d1e0be30b..85c83384d83e 100644 --- a/java/hook/src/main/java/org/ray/hook/runtime/MethodHash.java +++ b/java/hook/src/main/java/org/ray/hook/runtime/MethodHash.java @@ -10,15 +10,15 @@ public MethodHash(byte[] hash) { this.hash = hash; } - public byte[] getHash() { - return hash; - } - @Override public int hashCode() { return Arrays.hashCode(getHash()); } + public byte[] getHash() { + return hash; + } + @Override public boolean equals(Object obj) { if (this == obj) { diff --git a/java/hook/src/main/java/org/ray/hook/runtime/MethodSwitcher.java b/java/hook/src/main/java/org/ray/hook/runtime/MethodSwitcher.java index 196062c77e3f..0e36f82575fb 100644 --- a/java/hook/src/main/java/org/ray/hook/runtime/MethodSwitcher.java +++ b/java/hook/src/main/java/org/ray/hook/runtime/MethodSwitcher.java @@ -1,7 +1,7 @@ package org.ray.hook.runtime; /** - * method mode switch at runtime + * method mode switch at runtime. */ public class MethodSwitcher { diff --git a/java/pom.xml b/java/pom.xml index 5515c19af173..79aa7f04fb30 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -1,122 +1,156 @@ - - 4.0.0 - pom + + 4.0.0 + pom - org.ray.parent - ray-superpom - 1.0 - - api - common - hook - runtime-common - runtime-native - runtime-dev - test - example - + org.ray.parent + ray-superpom + 1.0 + + api + common + hook + runtime-common + runtime-native + runtime-dev + test + example + - - 1.8 - UTF-8 - + + 1.8 + UTF-8 + - - - - org.apache.arrow - arrow-plasma - 0.10.0-SNAPSHOT - - - + + + + org.apache.arrow + arrow-plasma + 0.10.0-SNAPSHOT + + + - - - - - org.apache.maven.plugins - maven-compiler-plugin - 3.6.1 - - ${java.version} - ${java.version} - ${project.build.sourceEncoding} - -parameters - -parameters - - + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.6.1 + + ${java.version} + ${java.version} + ${project.build.sourceEncoding} + -parameters + -parameters + + - - org.apache.maven.plugins - maven-source-plugin - 3.0.1 - - - attach-sources - deploy - - jar - - - - + + org.apache.maven.plugins + maven-source-plugin + 3.0.1 + + + attach-sources + deploy + + jar + + + + - - org.apache.maven.plugins - maven-dependency-plugin - 2.10 - - - org.apache.maven.plugins - maven-clean-plugin - 3.0.0 - + + org.apache.maven.plugins + maven-dependency-plugin + 2.10 + + + org.apache.maven.plugins + maven-clean-plugin + 3.0.0 + - - org.mortbay.jetty - maven-jetty-plugin - 6.1.26 - + + org.mortbay.jetty + maven-jetty-plugin + 6.1.26 + - - org.apache.maven.plugins - maven-assembly-plugin - 2.2 - + + org.apache.maven.plugins + maven-assembly-plugin + 2.2 + - - org.apache.maven.plugins - maven-javadoc-plugin - 2.10.4 - - - attach-javadocs - deploy - - jar - - - - - - maven-deploy-plugin - 2.8.2 - - - deploy - deploy - - deploy - - - - - - - + + org.apache.maven.plugins + maven-javadoc-plugin + 2.10.4 + + + attach-javadocs + deploy + + jar + + + + + + maven-deploy-plugin + 2.8.2 + + + deploy + deploy + + deploy + + + + + + org.apache.maven.plugins + maven-checkstyle-plugin + 2.17 + + + com.puppycrawl.tools + checkstyle + 6.19 + + + + + validate + validate + + check + + + + + checkstyle.xml + checkstyle-suppressions.xml + UTF-8 + true + false + true + warning + xml + html + ${project.build.directory}/test/checkstyle-errors.xml + false + + + + + diff --git a/java/ray.config.ini b/java/ray.config.ini index 4712dd241173..d7e0cc8bf925 100644 --- a/java/ray.config.ini +++ b/java/ray.config.ini @@ -117,7 +117,7 @@ driver_class = [ray.java.path.jni.package] %CONFIG_FILE_DIR%/../build/src/plasma = -%CONFIG_FILE_DIR%/../build/src/local_scheduler = +%CONFIG_FILE_DIR%/../build/src/local_scheduler = [ray.java.path.jni.deploy] %CONFIG_FILE_DIR%/native/lib = @@ -130,7 +130,7 @@ store_manager = %CONFIG_FILE_DIR%/../build/src/plasma/plasma_manager local_scheduler = %CONFIG_FILE_DIR%/../build/src/local_scheduler/local_scheduler global_scheduler = %CONFIG_FILE_DIR%/../build/src/global_scheduler/global_scheduler python_dir = %CONFIG_FILE_DIR%/../build/ -java_runtime_rewritten_jars_dir = +java_runtime_rewritten_jars_dir = java_class_paths = ray.java.path.classes.source java_jnilib_paths = ray.java.path.jni.package @@ -142,7 +142,7 @@ store_manager = %CONFIG_FILE_DIR%/../build/src/plasma/plasma_manager local_scheduler = %CONFIG_FILE_DIR%/../build/src/local_scheduler/local_scheduler global_scheduler = %CONFIG_FILE_DIR%/../build/src/global_scheduler/global_scheduler python_dir = %CONFIG_FILE_DIR%/../build/ -java_runtime_rewritten_jars_dir = +java_runtime_rewritten_jars_dir = java_class_paths = ray.java.path.classes.package java_jnilib_paths = ray.java.path.jni.package @@ -154,7 +154,7 @@ store_manager = %CONFIG_FILE_DIR%/native/bin/plasma_manager local_scheduler = %CONFIG_FILE_DIR%/native/bin/local_scheduler global_scheduler = %CONFIG_FILE_DIR%/native/bin/global_scheduler python_dir = %CONFIG_FILE_DIR%/python -java_runtime_rewritten_jars_dir = %CONFIG_FILE_DIR%/java/lib/ +java_runtime_rewritten_jars_dir = %CONFIG_FILE_DIR%/java/lib/ java_class_paths = ray.java.path.classes.deploy java_jnilib_paths = ray.java.path.jni.deploy diff --git a/java/runtime-common/pom.xml b/java/runtime-common/pom.xml index f8688667ba67..01f2a3c1da56 100644 --- a/java/runtime-common/pom.xml +++ b/java/runtime-common/pom.xml @@ -1,58 +1,58 @@ - - - org.ray.parent - ray-superpom - 1.0 - - 4.0.0 - org.ray - ray-runtime-common - - runtime common - runtime common - - - jar - - - - org.ray - ray-api - 1.0 - - - org.ray - ray-hook - 1.0 - - - de.ruedigermoeller - fst - 2.47 - - - - - com.github.davidmoten - flatbuffers-java - 1.7.0.1 - - - - - redis.clients - jedis - 2.8.0 - - - org.apache.arrow - arrow-plasma - - + + + org.ray.parent + ray-superpom + 1.0 + + 4.0.0 + org.ray + ray-runtime-common + + runtime common + runtime common + + + jar + + + + org.ray + ray-api + 1.0 + + + org.ray + ray-hook + 1.0 + + + de.ruedigermoeller + fst + 2.47 + + + + + com.github.davidmoten + flatbuffers-java + 1.7.0.1 + + + + + redis.clients + jedis + 2.8.0 + + + org.apache.arrow + arrow-plasma + + diff --git a/java/runtime-common/src/main/java/org/ray/core/ArgumentsBuilder.java b/java/runtime-common/src/main/java/org/ray/core/ArgumentsBuilder.java index f1486fdb0245..6c0eab42a1d2 100644 --- a/java/runtime-common/src/main/java/org/ray/core/ArgumentsBuilder.java +++ b/java/runtime-common/src/main/java/org/ray/core/ArgumentsBuilder.java @@ -20,7 +20,7 @@ import org.ray.util.exception.TaskExecutionException; /** - * arguments wrap and unwrap + * arguments wrap and unwrap. */ public class ArgumentsBuilder { @@ -36,12 +36,10 @@ public static FunctionArg[] wrap(RayInvocation invocation) { } else if (oarg.getClass().equals(RayActor.class)) { // serialize actor unique id if (k == 0) { - RayActorID aid = new RayActorID(); - aid.Id = ((RayActor) oarg).getId(); + RayActorId aid = new RayActorId(); + aid.id = ((RayActor) oarg).getId(); fargs[k].data = Serializer.encode(aid); - } - // serialize actor handle - else { + } else { // serialize actor handle fargs[k].data = Serializer.encode(oarg); } @@ -85,17 +83,17 @@ private static boolean checkSimpleValue(Object o) { @SuppressWarnings({"rawtypes", "unchecked"}) public static Pair unwrap(TaskSpec task, Method m, ClassLoader classLoader) throws TaskExecutionException { - FunctionArg fargs[] = task.args; - Object this_ = null; - Object realArgs[]; + FunctionArg[] fargs = task.args; + Object current = null; + Object[] realArgs; int start = 0; // check actor method if (!Modifier.isStatic(m.getModifiers())) { start = 1; - RayActorID actorId = Serializer.decode(fargs[0].data, classLoader); - this_ = RayRuntime.getInstance().getLocalActor(actorId.Id); + RayActorId actorId = Serializer.decode(fargs[0].data, classLoader); + current = RayRuntime.getInstance().getLocalActor(actorId.id); realArgs = new Object[fargs.length - 1]; } else { realArgs = new Object[fargs.length]; @@ -110,22 +108,16 @@ public static Pair unwrap(TaskSpec task, Method m, ClassLoader Object obj = Serializer.decode(farg.data, classLoader); // due to remote lambda, method may be static - if (obj instanceof RayActorID) { + if (obj instanceof RayActorId) { assert (k == 0); - realArgs[raIndex] = RayRuntime.getInstance().getLocalActor(((RayActorID) obj).Id); + realArgs[raIndex] = RayRuntime.getInstance().getLocalActor(((RayActorId) obj).id); } else { realArgs[raIndex] = obj; } - } - - // only ids, big data or single object id - else if (farg.data == null) { + } else if (farg.data == null) { // only ids, big data or single object id assert (farg.ids.size() == 1); realArgs[raIndex] = RayRuntime.getInstance().get(farg.ids.get(0)); - } - - // both id and data, could be RayList or RayMap only - else { + } else { // both id and data, could be RayList or RayMap only Object idBag = Serializer.decode(farg.data, classLoader); if (idBag instanceof RayMapArg) { Map newMap = new HashMap<>(); @@ -144,7 +136,7 @@ else if (farg.data == null) { } } } - return Pair.of(this_, realArgs); + return Pair.of(current, realArgs); } //for recognition @@ -161,9 +153,9 @@ public static class RayListArg extends ArrayList { } - public static class RayActorID implements Serializable { + public static class RayActorId implements Serializable { private static final long serialVersionUID = 3993646395842605166L; - public UniqueID Id; + public UniqueID id; } } diff --git a/java/runtime-common/src/main/java/org/ray/core/InvocationExecutor.java b/java/runtime-common/src/main/java/org/ray/core/InvocationExecutor.java index 1d663c362e44..8c54f0336e35 100644 --- a/java/runtime-common/src/main/java/org/ray/core/InvocationExecutor.java +++ b/java/runtime-common/src/main/java/org/ray/core/InvocationExecutor.java @@ -14,7 +14,7 @@ import org.ray.util.logger.RayLog; /** - * how to execute a invocation + * how to execute a invocation. */ public class InvocationExecutor { @@ -67,18 +67,14 @@ public static void execute(TaskSpec task, Pair pr) } } - private static void safePut(UniqueID objectId, Object obj) { - RayRuntime.getInstance().putRaw(objectId, obj); - } - private static void executeInternal(TaskSpec task, Pair pr, - String taskdesc) + String taskdesc) throws IllegalAccessException, IllegalArgumentException, InvocationTargetException { Method m = pr.getRight().invokable; Map userRayReturnIdMap = null; Class returnType = m.getReturnType(); // TODO: not ready for multiple return etc. boolean hasMultiReturn = false; - if(task.returnIds != null && task.returnIds.length > 0) { + if (task.returnIds != null && task.returnIds.length > 0) { hasMultiReturn = UniqueIdHelper.hasMultipleReturnOrNotFromReturnObjectId(task.returnIds[0]); } @@ -94,10 +90,10 @@ private static void executeInternal(TaskSpec task, Pair if (!UniqueIdHelper.isLambdaFunction(task.functionId)) { result = m.invoke(realArgs.getLeft(), realArgs.getRight()); } else { - result = m.invoke(realArgs.getLeft(), new Object[]{realArgs.getRight()}); + result = m.invoke(realArgs.getLeft(), new Object[] {realArgs.getRight()}); } - if(task.returnIds == null || task.returnIds.length == 0) { + if (task.returnIds == null || task.returnIds.length == 0) { return; } // set result into storage @@ -147,4 +143,8 @@ private static String formatTaskExecutionExceptionMsg(TaskSpec task, String func return "Execute task " + task.taskId + " failed with function name = " + funcName; } + + private static void safePut(UniqueID objectId, Object obj) { + RayRuntime.getInstance().putRaw(objectId, obj); + } } diff --git a/java/runtime-common/src/main/java/org/ray/core/LocalFunctionManager.java b/java/runtime-common/src/main/java/org/ray/core/LocalFunctionManager.java index 6ef058856f2e..35df33267383 100644 --- a/java/runtime-common/src/main/java/org/ray/core/LocalFunctionManager.java +++ b/java/runtime-common/src/main/java/org/ray/core/LocalFunctionManager.java @@ -13,18 +13,49 @@ import org.ray.util.logger.RayLog; /** - * local function manager which pulls remote functions on demand + * local function manager which pulls remote functions on demand. */ public class LocalFunctionManager { + private final RemoteFunctionManager remoteLoader; + private final ConcurrentHashMap functionTables + = new ConcurrentHashMap<>(); + /** * initialize load function manager using remote function manager to pull remote functions on - * demand + * demand. */ public LocalFunctionManager(RemoteFunctionManager remoteLoader) { this.remoteLoader = remoteLoader; } + /** + * get local method for executing, which pulls information from remote repo on-demand, therefore + * it may block for a while if the related resources (e.g., jars) are not ready on local machine + */ + public Pair getMethod(UniqueID driverId, UniqueID methodId, + FunctionArg[] args) throws NoSuchMethodException, + SecurityException, ClassNotFoundException { + FunctionTable funcs = loadDriverFunctions(driverId); + RayMethod m; + + // hooked methods + if (!UniqueIdHelper.isLambdaFunction(methodId)) { + m = funcs.functions.get(methodId); + if (null == m) { + throw new RuntimeException( + "DriverId " + driverId + " load remote function methodId:" + methodId + " failed"); + } + } else { // remote lambda + assert args.length >= 2; + String fname = Serializer.decode(args[args.length - 2].data); + Method fm = Class.forName(fname).getMethod("execute", Object[].class); + m = new RayMethod(fm); + } + + return Pair.of(funcs.linkedFunctions.loader, m); + } + private synchronized FunctionTable loadDriverFunctions(UniqueID driverId) { FunctionTable funcs = functionTables.get(driverId); if (null == funcs) { @@ -50,9 +81,7 @@ private synchronized FunctionTable loadDriverFunctions(UniqueID driverId) { } functionTables.put(driverId, funcs); - } - // reSync automatically - else { + } else { // reSync automatically // more functions are loaded if (funcs.linkedFunctions.functions.size() > funcs.functions.size()) { for (MethodId mid : funcs.linkedFunctions.functions) { @@ -71,36 +100,7 @@ private synchronized FunctionTable loadDriverFunctions(UniqueID driverId) { } /** - * get local method for executing, which pulls information from remote repo on-demand, therefore - * it may block for a while if the related resources (e.g., jars) are not ready on local machine - */ - public Pair getMethod(UniqueID driverId, UniqueID methodId, - FunctionArg[] args) throws NoSuchMethodException, SecurityException, ClassNotFoundException { - FunctionTable funcs = loadDriverFunctions(driverId); - RayMethod m; - - // hooked methods - if (!UniqueIdHelper.isLambdaFunction(methodId)) { - m = funcs.functions.get(methodId); - if (null == m) { - throw new RuntimeException( - "DriverId " + driverId + " load remote function methodId:" + methodId + " failed"); - } - } - - // remote lambda - else { - assert args.length >= 2; - String fname = Serializer.decode(args[args.length - 2].data); - Method fm = Class.forName(fname).getMethod("execute", Object[].class); - m = new RayMethod(fm); - } - - return Pair.of(funcs.linkedFunctions.loader, m); - } - - /** - * unload the functions when the driver is declared dead + * unload the functions when the driver is declared dead. */ public synchronized void removeApp(UniqueID driverId) { FunctionTable funcs = functionTables.get(driverId); @@ -115,7 +115,4 @@ static class FunctionTable { public final ConcurrentHashMap functions = new ConcurrentHashMap<>(); public LoadedFunctions linkedFunctions; } - - private final RemoteFunctionManager remoteLoader; - private final ConcurrentHashMap functionTables = new ConcurrentHashMap<>(); } diff --git a/java/runtime-common/src/main/java/org/ray/core/RayRuntime.java b/java/runtime-common/src/main/java/org/ray/core/RayRuntime.java index 441ce83ae35c..ecac1c3a78bc 100644 --- a/java/runtime-common/src/main/java/org/ray/core/RayRuntime.java +++ b/java/runtime-common/src/main/java/org/ray/core/RayRuntime.java @@ -9,7 +9,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; - import org.apache.arrow.plasma.ObjectStoreLink; import org.apache.commons.lang3.tuple.Pair; import org.ray.api.Ray; @@ -35,19 +34,20 @@ import org.ray.util.logger.RayLog; /** - * Core functionality to implement Ray APIs + * Core functionality to implement Ray APIs. */ public abstract class RayRuntime implements RayApi { + public static ConfigReader configReader; protected static RayRuntime ins = null; - protected static RayParameters params = null; - private static boolean fromRayInit = false; - - public static ConfigReader configReader; - - public abstract void cleanUp(); + protected Worker worker; + protected LocalSchedulerProxy localSchedulerProxy; + protected ObjectStoreProxy objectStoreProxy; + protected LocalFunctionManager functions; + protected RemoteFunctionManager remoteFunctionManager; + protected PathConfig pathConfig; // app level Ray.init() // make it private so there is no direct usage but only from Ray.init @@ -65,24 +65,6 @@ private static RayRuntime init() { return ins; } - // init with command line args - // --config=ray.config.ini --overwrite=updateConfigStr - public static RayRuntime init(String[] args) throws Exception { - String config = null; - String updateConfig = null; - for (String arg : args) { - if (arg.startsWith("--config=")) { - config = arg.substring("--config=".length()); - } else if (arg.startsWith("--overwrite=")) { - updateConfig = arg.substring("--overwrite=".length()); - } else { - throw new RuntimeException("Input argument " + arg - + " is not recognized, please use --overwrite to merge it into config file"); - } - } - return init(config, updateConfig); - } - // engine level RayRuntime.init(xx, xx) // updateConfigStr is sth like section1.k1=v1;section2.k2=v2 public static RayRuntime init(String configPath, String updateConfigStr) throws Exception { @@ -114,9 +96,43 @@ public static RayRuntime init(String configPath, String updateConfigStr) throws return ins; } + // init with command line args + // --config=ray.config.ini --overwrite=updateConfigStr + public static RayRuntime init(String[] args) throws Exception { + String config = null; + String updateConfig = null; + for (String arg : args) { + if (arg.startsWith("--config=")) { + config = arg.substring("--config=".length()); + } else if (arg.startsWith("--overwrite=")) { + updateConfig = arg.substring("--overwrite=".length()); + } else { + throw new RuntimeException("Input argument " + arg + + " is not recognized, please use --overwrite to merge it into config file"); + } + } + return init(config, updateConfig); + } + + protected void init( + LocalSchedulerLink slink, + ObjectStoreLink plink, + RemoteFunctionManager remoteLoader, + PathConfig pathManager + ) { + UniqueIdHelper.setThreadRandomSeed(UniqueIdHelper.getUniqueness(params.driver_id)); + remoteFunctionManager = remoteLoader; + pathConfig = pathManager; + + functions = new LocalFunctionManager(remoteLoader); + localSchedulerProxy = new LocalSchedulerProxy(slink); + objectStoreProxy = new ObjectStoreProxy(plink); + worker = new Worker(localSchedulerProxy, functions); + } + private static RayRuntime instantiate(RayParameters params) { - String className = params.run_mode.isNativeRuntime() ? - "org.ray.core.impl.RayNativeRuntime" : "org.ray.core.impl.RayDevRuntime"; + String className = params.run_mode.isNativeRuntime() + ? "org.ray.core.impl.RayNativeRuntime" : "org.ray.core.impl.RayDevRuntime"; RayRuntime runtime; try { @@ -129,7 +145,8 @@ private static RayRuntime instantiate(RayParameters params) { runtime = (RayRuntime) cons.newInstance(); cons.setAccessible(false); } catch (InstantiationException | IllegalAccessException | IllegalArgumentException - | InvocationTargetException | SecurityException | ClassNotFoundException | NoSuchMethodException e) { + | InvocationTargetException | SecurityException | ClassNotFoundException + | NoSuchMethodException e) { RayLog.core .error("Load class " + className + " failed for run-mode " + params.run_mode.toString(), e); @@ -150,6 +167,11 @@ private static RayRuntime instantiate(RayParameters params) { return runtime; } + /** + * start runtime. + */ + public abstract void start(RayParameters params) throws Exception; + public static RayRuntime getInstance() { return ins; } @@ -158,38 +180,57 @@ public static RayParameters getParams() { return params; } - /*********** RayApi methods ***********/ + public abstract void cleanUp(); - public void putRaw(UniqueID taskId, UniqueID objectId, T obj, TM metadata) { + public void putRaw(UniqueID taskId, UniqueID objectId, T obj) { + putRaw(taskId, objectId, obj, null); + } + + /*********** + * RayApi methods. + ***********/ + + public void putRaw(UniqueID taskId, UniqueID objectId, T obj, TMT metadata) { RayLog.core.info("Task " + taskId.toString() + " Object " + objectId.toString() + " put"); localSchedulerProxy.markTaskPutDependency(taskId, objectId); objectStoreProxy.put(objectId, obj, metadata); } - public void putRaw(UniqueID taskId, UniqueID objectId, T obj) { - putRaw(taskId, objectId, obj, null); - } - public void putRaw(UniqueID objectId, T obj) { - UniqueID taskId = getCurrentTaskID(); + UniqueID taskId = getCurrentTaskId(); putRaw(taskId, objectId, obj, null); } public void putRaw(T obj) { - UniqueID taskId = getCurrentTaskID(); - UniqueID objectId = getCurrentTaskNextPutID(); + UniqueID taskId = getCurrentTaskId(); + UniqueID objectId = getCurrentTaskNextPutId(); putRaw(taskId, objectId, obj, null); } + /** + * get the task identity of the currently running task, UniqueID.Nil if not inside any + */ + public UniqueID getCurrentTaskId() { + return worker.getCurrentTaskId(); + } + + /** + * get the to-be-returned objects identities of the currently running task, empty array if not + * inside any. + */ + public UniqueID getCurrentTaskNextPutId() { + return worker.getCurrentTaskNextPutId(); + } + @Override public RayObject put(T obj) { return put(obj, null); } @Override - public RayObject put(T obj, TM metadata) { - UniqueID taskId = getCurrentTaskID(); - UniqueID objectId = getCurrentTaskNextPutID(); + public RayObject put(T obj, TMT metadata) { + UniqueID taskId = getCurrentTaskId(); + UniqueID objectId = getCurrentTaskNextPutId(); putRaw(taskId, objectId, obj, metadata); return new RayObject<>(objectId); } @@ -199,86 +240,74 @@ public T get(UniqueID objectId) throws TaskExecutionException { return doGet(objectId, false); } + @Override + public List get(List objectIds) throws TaskExecutionException { + return doGet(objectIds, false); + } + @Override public T getMeta(UniqueID objectId) throws TaskExecutionException { return doGet(objectId, true); } - private T doGet(UniqueID objectId, boolean isMetadata) throws TaskExecutionException { - - boolean wasBlocked = false; - UniqueID taskId = getCurrentTaskID(); - try { - // Do an initial fetch. - objectStoreProxy.fetch(objectId); - - // Get the object. We initially try to get the object immediately. - Pair ret = objectStoreProxy - .get(objectId, params.default_first_check_timeout_ms, isMetadata); - - wasBlocked = (ret.getRight() != GetStatus.SUCCESS); + @Override + public List getMeta(List objectIds) throws TaskExecutionException { + return doGet(objectIds, true); + } - // Try reconstructing the object. Try to get it until at least PlasmaLink.GET_TIMEOUT_MS - // milliseconds passes, then repeat. - while (ret.getRight() != GetStatus.SUCCESS) { - RayLog.core.warn( - "Task " + taskId + " Object " + objectId.toString() + " get failed, reconstruct ..."); - localSchedulerProxy.reconstructObject(objectId); + @Override + public WaitResult wait(RayList waitfor, int numReturns, int timeout) { + return objectStoreProxy.wait(waitfor, numReturns, timeout); + } - // Do another fetch - objectStoreProxy.fetch(objectId); + @Override + public RayObjects call(UniqueID taskId, Callable funcRun, int returnCount, Object... args) { + return worker.rpc(taskId, funcRun, returnCount, args); + } - ret = objectStoreProxy.get(objectId, params.default_get_check_interval_ms, - isMetadata);//check the result every 5s, but it will return once available - } - RayLog.core.debug( - "Task " + taskId + " Object " + objectId.toString() + " get" + ", the result " + ret - .getLeft()); - return ret.getLeft(); - } catch (TaskExecutionException e) { - RayLog.core - .error("Task " + taskId + " Object " + objectId.toString() + " get with Exception", e); - throw e; - } finally { - // If the object was not able to get locally, let the local scheduler - // know that we're now unblocked. - if (wasBlocked) { - localSchedulerProxy.notifyUnblocked(); - } - } + @Override + public RayObjects call(UniqueID taskId, Class funcCls, Serializable lambda, int returnCount, + Object... args) { + return worker.rpc(taskId, UniqueID.nil, funcCls, lambda, returnCount, args); + } + @Override + public RayMap callWithReturnLabels(UniqueID taskId, Callable funcRun, + Collection returnIds, + Object... args) { + return worker.rpcWithReturnLabels(taskId, funcRun, returnIds, args); } @Override - public List get(List objectIds) throws TaskExecutionException { - return doGet(objectIds, false); + public RayMap callWithReturnLabels(UniqueID taskId, Class funcCls, + Serializable lambda, Collection + returnids, + Object... args) { + return worker.rpcWithReturnLabels(taskId, funcCls, lambda, returnids, args); } @Override - public List getMeta(List objectIds) throws TaskExecutionException { - return doGet(objectIds, true); + public RayList callWithReturnIndices(UniqueID taskId, Callable funcRun, + Integer returnCount, Object... args) { + return worker.rpcWithReturnIndices(taskId, funcRun, returnCount, args); } - // We divide the fetch into smaller fetches so as to not block the manager - // for a prolonged period of time in a single call. - private void dividedFetch(List objectIds) { - int fetchSize = objectStoreProxy.getFetchSize(); + @Override + public RayList callWithReturnIndices(UniqueID taskId, Class funcCls, + Serializable lambda, Integer returnCount, Object... + args) { + return worker.rpcWithReturnIndices(taskId, funcCls, lambda, returnCount, args); + } - int numObjectIds = objectIds.size(); - for (int i = 0; i < numObjectIds; i += fetchSize) { - int endIndex = i + fetchSize; - if (endIndex < numObjectIds) { - objectStoreProxy.fetch(objectIds.subList(i, endIndex)); - } else { - objectStoreProxy.fetch(objectIds.subList(i, numObjectIds)); - } - } + @Override + public boolean isRemoteLambda() { + return params.run_mode.isRemoteLambda(); } private List doGet(List objectIds, boolean isMetadata) throws TaskExecutionException { boolean wasBlocked = false; - UniqueID taskId = getCurrentTaskID(); + UniqueID taskId = getCurrentTaskId(); try { int numObjectIds = objectIds.size(); @@ -348,53 +377,65 @@ private List doGet(List objectIds, boolean isMetadata) } - @Override - public WaitResult wait(RayList waitfor, int numReturns, int timeout) { - return objectStoreProxy.wait(waitfor, numReturns, timeout); - } + private T doGet(UniqueID objectId, boolean isMetadata) throws TaskExecutionException { - @Override - public RayObjects call(UniqueID taskId, Callable funcRun, int returnCount, Object... args) { - return worker.rpc(taskId, funcRun, returnCount, args); - } + boolean wasBlocked = false; + UniqueID taskId = getCurrentTaskId(); + try { + // Do an initial fetch. + objectStoreProxy.fetch(objectId); - @Override - public RayObjects call(UniqueID taskId, Class funcCls, Serializable lambda, int returnCount, - Object... args) { - return worker.rpc(taskId, UniqueID.nil, funcCls, lambda, returnCount, args); - } + // Get the object. We initially try to get the object immediately. + Pair ret = objectStoreProxy + .get(objectId, params.default_first_check_timeout_ms, isMetadata); - @Override - public RayMap callWithReturnLabels(UniqueID taskId, Callable funcRun, - Collection returnIds, - Object... args) { - return worker.rpcWithReturnLabels(taskId, funcRun, returnIds, args); - } + wasBlocked = (ret.getRight() != GetStatus.SUCCESS); - @Override - public RayMap callWithReturnLabels(UniqueID taskId, Class funcCls, - Serializable lambda, Collection returnids, - Object... args) { - return worker.rpcWithReturnLabels(taskId, funcCls, lambda, returnids, args); - } + // Try reconstructing the object. Try to get it until at least PlasmaLink.GET_TIMEOUT_MS + // milliseconds passes, then repeat. + while (ret.getRight() != GetStatus.SUCCESS) { + RayLog.core.warn( + "Task " + taskId + " Object " + objectId.toString() + " get failed, reconstruct ..."); + localSchedulerProxy.reconstructObject(objectId); - @Override - public RayList callWithReturnIndices(UniqueID taskId, Callable funcRun, - Integer returnCount, Object... args) { - return worker.rpcWithReturnIndices(taskId, funcRun, returnCount, args); - } + // Do another fetch + objectStoreProxy.fetch(objectId); + + ret = objectStoreProxy.get(objectId, params.default_get_check_interval_ms, + isMetadata);//check the result every 5s, but it will return once available + } + RayLog.core.debug( + "Task " + taskId + " Object " + objectId.toString() + " get" + ", the result " + ret + .getLeft()); + return ret.getLeft(); + } catch (TaskExecutionException e) { + RayLog.core + .error("Task " + taskId + " Object " + objectId.toString() + " get with Exception", e); + throw e; + } finally { + // If the object was not able to get locally, let the local scheduler + // know that we're now unblocked. + if (wasBlocked) { + localSchedulerProxy.notifyUnblocked(); + } + } - @Override - public RayList callWithReturnIndices(UniqueID taskId, Class funcCls, - Serializable lambda, Integer returnCount, Object... args) { - return worker.rpcWithReturnIndices(taskId, funcCls, lambda, returnCount, args); } - /** - * get the task identity of the currently running task, UniqueID.Nil if not inside any - */ - public UniqueID getCurrentTaskID() { - return worker.getCurrentTaskID(); + // We divide the fetch into smaller fetches so as to not block the manager + // for a prolonged period of time in a single call. + private void dividedFetch(List objectIds) { + int fetchSize = objectStoreProxy.getFetchSize(); + + int numObjectIds = objectIds.size(); + for (int i = 0; i < numObjectIds; i += fetchSize) { + int endIndex = i + fetchSize; + if (endIndex < numObjectIds) { + objectStoreProxy.fetch(objectIds.subList(i, endIndex)); + } else { + objectStoreProxy.fetch(objectIds.subList(i, numObjectIds)); + } + } } /** @@ -404,48 +445,16 @@ public UniqueID[] getCurrentTaskReturnIDs() { return worker.getCurrentTaskReturnIDs(); } - /** - * get the to-be-returned objects identities of the currently running task, empty array if not - * inside any - */ - public UniqueID getCurrentTaskNextPutID() { - return worker.getCurrentTaskNextPutID(); - } - - @Override - public boolean isRemoteLambda() { - return params.run_mode.isRemoteLambda(); - } - - protected void init( - LocalSchedulerLink slink, - ObjectStoreLink plink, - RemoteFunctionManager remoteLoader, - PathConfig pathManager - ) { - UniqueIdHelper.setThreadRandomSeed(UniqueIdHelper.getUniqueness(params.driver_id)); - remoteFunctionManager = remoteLoader; - pathConfig = pathManager; - - functions = new LocalFunctionManager(remoteLoader); - localSchedulerProxy = new LocalSchedulerProxy(slink); - objectStoreProxy = new ObjectStoreProxy(plink); - worker = new Worker(localSchedulerProxy, functions); - } - - /*********** Internal Methods ***********/ + /*********** + * Internal Methods. + ***********/ public void loop() { worker.loop(); } /** - * start runtime - */ - public abstract void start(RayParameters params) throws Exception; - - /** - * get actor with given id + * get actor with given id. */ public abstract Object getLocalActor(UniqueID id); @@ -456,11 +465,4 @@ public PathConfig getPaths() { public RemoteFunctionManager getRemoteFunctionManager() { return remoteFunctionManager; } - - protected Worker worker; - protected LocalSchedulerProxy localSchedulerProxy; - protected ObjectStoreProxy objectStoreProxy; - protected LocalFunctionManager functions; - protected RemoteFunctionManager remoteFunctionManager; - protected PathConfig pathConfig; } diff --git a/java/runtime-common/src/main/java/org/ray/core/Serializer.java b/java/runtime-common/src/main/java/org/ray/core/Serializer.java index 7a48129e5eaf..d9c293b51b06 100644 --- a/java/runtime-common/src/main/java/org/ray/core/Serializer.java +++ b/java/runtime-common/src/main/java/org/ray/core/Serializer.java @@ -14,11 +14,6 @@ public static byte[] encode(Object obj) { return conf.get().asByteArray(obj); } - @SuppressWarnings("unchecked") - public static T decode(byte[] bs) { - return (T) conf.get().asObject(bs); - } - public static byte[] encode(Object obj, ClassLoader classLoader) { byte[] result; FSTConfiguration current = conf.get(); @@ -34,6 +29,11 @@ public static byte[] encode(Object obj, ClassLoader classLoader) { return result; } + @SuppressWarnings("unchecked") + public static T decode(byte[] bs) { + return (T) conf.get().asObject(bs); + } + @SuppressWarnings("unchecked") public static T decode(byte[] bs, ClassLoader classLoader) { Object object; diff --git a/java/runtime-common/src/main/java/org/ray/core/UniqueIdHelper.java b/java/runtime-common/src/main/java/org/ray/core/UniqueIdHelper.java index 90a483ea9e36..6ca592e65776 100644 --- a/java/runtime-common/src/main/java/org/ray/core/UniqueIdHelper.java +++ b/java/runtime-common/src/main/java/org/ray/core/UniqueIdHelper.java @@ -15,16 +15,20 @@ // public class UniqueIdHelper { - public enum Type { - OBJECT, - TASK, - ACTOR, - } - private static final ThreadLocal longBuffer = ThreadLocal .withInitial(() -> ByteBuffer.allocate(Long.SIZE / Byte.SIZE)); private static final ThreadLocal rand = ThreadLocal.withInitial(Random::new); private static final ThreadLocal randSeed = new ThreadLocal<>(); + private static final int batchPos = 0; + private static final int uniquenessPos = Long.SIZE / Byte.SIZE; + private static final int typePos = 2 * Long.SIZE / Byte.SIZE; + private static final BitField typeField = new BitField(0x7); + private static final int testPos = 2 * Long.SIZE / Byte.SIZE; + private static final BitField testField = new BitField(0x1 << 3); + private static final int unionPos = 2 * Long.SIZE / Byte.SIZE; + private static final BitField multipleReturnField = new BitField(0x1 << 8); + private static final BitField isReturnIdField = new BitField(0x1 << 9); + private static final BitField withinTaskIndexField = new BitField(0xFFFFFC00); public static void setThreadRandomSeed(long seed) { if (randSeed.get() != null) { @@ -43,123 +47,80 @@ public static Long getNextCreateThreadRandomSeed() { UniqueID currentTaskId = WorkerContext.currentTask().taskId; byte[] bytes; - ByteBuffer lBuffer = longBuffer.get(); + ByteBuffer lbuffer = longBuffer.get(); // similar to task id generation (see nextTaskId below) if (!currentTaskId.isNil()) { ByteBuffer rbb = ByteBuffer.wrap(currentTaskId.getBytes()); rbb.order(ByteOrder.LITTLE_ENDIAN); long cid = rbb.getLong(uniquenessPos); - byte[] cbuffer = lBuffer.putLong(cid).array(); + byte[] cbuffer = lbuffer.putLong(cid).array(); bytes = MD5Digestor.digest(cbuffer, WorkerContext.nextCallIndex()); } else { long cid = rand.get().nextLong(); - byte[] cbuffer = lBuffer.putLong(cid).array(); + byte[] cbuffer = lbuffer.putLong(cid).array(); bytes = MD5Digestor.digest(cbuffer, rand.get().nextLong()); } - lBuffer.clear(); + lbuffer.clear(); - lBuffer.put(bytes, 0, Long.SIZE / Byte.SIZE); - long r = lBuffer.getLong(); - lBuffer.clear(); + lbuffer.put(bytes, 0, Long.SIZE / Byte.SIZE); + long r = lbuffer.getLong(); + lbuffer.clear(); return r; } - private static final int batchPos = 0; - - private static void setBatch(ByteBuffer bb, long batchId) { - bb.putLong(batchPos, batchId); - } - - private static long getBatch(ByteBuffer bb) { - return bb.getLong(batchPos); - } - - private static final int uniquenessPos = Long.SIZE / Byte.SIZE; - - private static void setUniqueness(ByteBuffer bb, long uniqueness) { - bb.putLong(uniquenessPos, uniqueness); - } - - private static void setUniqueness(ByteBuffer bb, byte[] uniqueness) { - for (int i = 0; i < Long.SIZE / Byte.SIZE; ++i) { - bb.put(uniquenessPos + i, uniqueness[i]); - } - } - - private static long getUniqueness(ByteBuffer bb) { - return bb.getLong(uniquenessPos); - } - - private static final int typePos = 2 * Long.SIZE / Byte.SIZE; - private static final BitField typeField = new BitField(0x7); - - private static void setType(ByteBuffer bb, Type type) { - byte v = bb.get(typePos); - v = (byte) typeField.setValue(v, type.ordinal()); - bb.put(typePos, v); - } - private static Type getType(ByteBuffer bb) { byte v = bb.get(typePos); return Type.values()[typeField.getValue(v)]; } - private static final int testPos = 2 * Long.SIZE / Byte.SIZE; - private static final BitField testField = new BitField(0x1 << 3); - - private static void setIsTest(ByteBuffer bb, boolean isTest) { - byte v = bb.get(testPos); - v = (byte) testField.setValue(v, isTest ? 1 : 0); - bb.put(testPos, v); - } - private static boolean getIsTest(ByteBuffer bb) { byte v = bb.get(testPos); return testField.getValue(v) == 1; } - private static final int unionPos = 2 * Long.SIZE / Byte.SIZE; - private static final BitField multipleReturnField = new BitField(0x1 << 8); - private static final BitField isReturnIdField = new BitField(0x1 << 9); - private static final BitField withinTaskIndexField = new BitField(0xFFFFFC00); - - private static void setHasMultipleReturn(ByteBuffer bb, int hasMultipleReturnOrNot) { + private static int getIsReturn(ByteBuffer bb) { int v = bb.getInt(unionPos); - v = multipleReturnField.setValue(v, hasMultipleReturnOrNot); - bb.putInt(unionPos, v); + return isReturnIdField.getValue(v); } - private static int getHasMultipleReturn(ByteBuffer bb) { + private static int getWithinTaskIndex(ByteBuffer bb) { int v = bb.getInt(unionPos); - return multipleReturnField.getValue(v); + return withinTaskIndexField.getValue(v); } - private static void setIsReturn(ByteBuffer bb, int isReturn) { - int v = bb.getInt(unionPos); - v = isReturnIdField.setValue(v, isReturn); - bb.putInt(unionPos, v); + public static void setTest(UniqueID id, boolean isTest) { + ByteBuffer bb = ByteBuffer.wrap(id.getBytes()); + setIsTest(bb, isTest); } - private static int getIsReturn(ByteBuffer bb) { - int v = bb.getInt(unionPos); - return isReturnIdField.getValue(v); + private static void setIsTest(ByteBuffer bb, boolean isTest) { + byte v = bb.get(testPos); + v = (byte) testField.setValue(v, isTest ? 1 : 0); + bb.put(testPos, v); } - private static void setWithinTaskIndex(ByteBuffer bb, int index) { - int v = bb.getInt(unionPos); - v = withinTaskIndexField.setValue(v, index); - bb.putInt(unionPos, v); + public static long getUniqueness(UniqueID id) { + ByteBuffer bb = ByteBuffer.wrap(id.getBytes()); + bb.order(ByteOrder.LITTLE_ENDIAN); + return getUniqueness(bb); } - private static int getWithinTaskIndex(ByteBuffer bb) { - int v = bb.getInt(unionPos); - return withinTaskIndexField.getValue(v); + private static long getUniqueness(ByteBuffer bb) { + return bb.getLong(uniquenessPos); + } + + public static UniqueID taskComputeReturnId( + UniqueID uid, + int returnIndex, + boolean hasMultipleReturn + ) { + return objectIdFromTaskId(uid, true, hasMultipleReturn, returnIndex); } private static UniqueID objectIdFromTaskId(UniqueID taskId, - boolean isReturn, - boolean hasMultipleReturn, - int index + boolean isReturn, + boolean hasMultipleReturn, + int index ) { UniqueID oid = newZero(); ByteBuffer rbb = ByteBuffer.wrap(taskId.getBytes()); @@ -181,23 +142,46 @@ private static UniqueID newZero() { return new UniqueID(b); } - public static void setTest(UniqueID id, boolean isTest) { - ByteBuffer bb = ByteBuffer.wrap(id.getBytes()); - setIsTest(bb, isTest); + private static void setBatch(ByteBuffer bb, long batchId) { + bb.putLong(batchPos, batchId); } - public static long getUniqueness(UniqueID id) { - ByteBuffer bb = ByteBuffer.wrap(id.getBytes()); - bb.order(ByteOrder.LITTLE_ENDIAN); - return getUniqueness(bb); + private static long getBatch(ByteBuffer bb) { + return bb.getLong(batchPos); } - public static UniqueID taskComputeReturnId( - UniqueID uid, - int returnIndex, - boolean hasMultipleReturn - ) { - return objectIdFromTaskId(uid, true, hasMultipleReturn, returnIndex); + private static void setUniqueness(ByteBuffer bb, long uniqueness) { + bb.putLong(uniquenessPos, uniqueness); + } + + private static void setUniqueness(ByteBuffer bb, byte[] uniqueness) { + for (int i = 0; i < Long.SIZE / Byte.SIZE; ++i) { + bb.put(uniquenessPos + i, uniqueness[i]); + } + } + + private static void setType(ByteBuffer bb, Type type) { + byte v = bb.get(typePos); + v = (byte) typeField.setValue(v, type.ordinal()); + bb.put(typePos, v); + } + + private static void setHasMultipleReturn(ByteBuffer bb, int hasMultipleReturnOrNot) { + int v = bb.getInt(unionPos); + v = multipleReturnField.setValue(v, hasMultipleReturnOrNot); + bb.putInt(unionPos, v); + } + + private static void setIsReturn(ByteBuffer bb, int isReturn) { + int v = bb.getInt(unionPos); + v = isReturnIdField.setValue(v, isReturn); + bb.putInt(unionPos, v); + } + + private static void setWithinTaskIndex(ByteBuffer bb, int index) { + int v = bb.getInt(unionPos); + v = withinTaskIndexField.setValue(v, index); + bb.putInt(unionPos, v); } public static UniqueID taskComputePutId(UniqueID uid, int putIndex) { @@ -210,6 +194,11 @@ public static boolean hasMultipleReturnOrNotFromReturnObjectId(UniqueID returnId return getHasMultipleReturn(bb) != 0; } + private static int getHasMultipleReturn(ByteBuffer bb) { + int v = bb.getInt(unionPos); + return multipleReturnField.getValue(v); + } + public static UniqueID taskIdFromObjectId(UniqueID objectId) { UniqueID taskId = newZero(); ByteBuffer rbb = ByteBuffer.wrap(objectId.getBytes()); @@ -242,25 +231,24 @@ public static UniqueID nextTaskId(long batchId) { // setup unique id (task id) byte[] idBytes; - ByteBuffer lBuffer = longBuffer.get(); + ByteBuffer lbuffer = longBuffer.get(); // if inside a task if (!currentTaskId.isNil()) { long cid = rbb.getLong(uniquenessPos); - byte[] cbuffer = lBuffer.putLong(cid).array(); + byte[] cbuffer = lbuffer.putLong(cid).array(); idBytes = MD5Digestor.digest(cbuffer, WorkerContext.nextCallIndex()); // if not } else { long cid = rand.get().nextLong(); - byte[] cbuffer = lBuffer.putLong(cid).array(); + byte[] cbuffer = lbuffer.putLong(cid).array(); idBytes = MD5Digestor.digest(cbuffer, rand.get().nextLong()); } setUniqueness(wbb, idBytes); - lBuffer.clear(); + lbuffer.clear(); return taskId; } - public static boolean isLambdaFunction(UniqueID functionId) { ByteBuffer wbb = ByteBuffer.wrap(functionId.getBytes()); wbb.order(ByteOrder.LITTLE_ENDIAN); @@ -285,4 +273,10 @@ public static boolean isNonLambdaCommonFunction(UniqueID functionId) { wbb.order(ByteOrder.LITTLE_ENDIAN); return getUniqueness(wbb) == 0; } + + public enum Type { + OBJECT, + TASK, + ACTOR, + } } diff --git a/java/runtime-common/src/main/java/org/ray/core/Worker.java b/java/runtime-common/src/main/java/org/ray/core/Worker.java index c8233d3797b4..d607656330c8 100644 --- a/java/runtime-common/src/main/java/org/ray/core/Worker.java +++ b/java/runtime-common/src/main/java/org/ray/core/Worker.java @@ -72,57 +72,13 @@ public static void execute(TaskSpec task, LocalFunctionManager funcs) { } - private RayObjects taskSubmit(UniqueID taskId, - byte[] fid, - int returnCount, - boolean multiReturn, - Object[] args) { - RayInvocation ri = new RayInvocation(fid, args); - return scheduler.submit(taskId, ri, returnCount, multiReturn); - } - - private RayObjects actorTaskSubmit(UniqueID taskId, - byte[] fid, - int returnCount, - boolean multiReturn, - Object[] args, - RayActor actor) { - RayInvocation ri = new RayInvocation(fid, args, actor); - RayObjects returnObjs = scheduler.submit(taskId, ri, returnCount + 1, multiReturn); - actor.setTaskCursor(returnObjs.pop().getId()); - return returnObjs; - } - - private RayObjects submit(UniqueID taskId, - byte[] fid, - int returnCount, - boolean multiReturn, - Object[] args) { - if (taskId == null) { - taskId = UniqueIdHelper.nextTaskId(-1); - } - if (args.length > 0 && args[0].getClass().equals(RayActor.class)) { - return actorTaskSubmit(taskId, fid, returnCount, multiReturn, args, (RayActor) args[0]); - } else { - return taskSubmit(taskId, fid, returnCount, multiReturn, args); - } - } - public RayObjects rpc(UniqueID taskId, Callable funcRun, int returnCount, Object[] args) { byte[] fid = fidFromHook(funcRun); return submit(taskId, fid, returnCount, false, args); } - public RayObjects rpcCreateActor(UniqueID taskId, UniqueID createActorId, Callable funcRun, - int returnCount, - Object[] args) { - byte[] fid = fidFromHook(funcRun); - RayInvocation ri = new RayInvocation(fid, new Object[]{}); - return scheduler.submit(taskId, createActorId, ri, returnCount, false); - } - public RayObjects rpc(UniqueID taskId, UniqueID functionId, Class funcCls, Serializable lambda, - int returnCount, Object[] args) { + int returnCount, Object[] args) { byte[] fid = functionId.getBytes(); Object[] ls = Arrays.copyOf(args, args.length + 2); @@ -132,38 +88,99 @@ public RayObjects rpc(UniqueID taskId, UniqueID functionId, Class funcCls, Se return submit(taskId, fid, returnCount, false, ls); } - public RayObjects rpcCreateActor(UniqueID taskId, UniqueID createActorId, UniqueID functionId, - Class funcCls, Serializable lambda, int returnCount, Object[] args) { + public RayObjects rpc(UniqueID taskId, RayActor actor, Callable funcRun, int returnCount, + Object[] args) { + byte[] fid = fidFromHook(funcRun); + return actorTaskSubmit(taskId, fid, returnCount, false, args, actor); + } + + public RayObjects rpc(UniqueID taskId, UniqueID functionId, RayActor actor, Class funcCls, + Serializable lambda, int returnCount, Object[] args) { byte[] fid = functionId.getBytes(); Object[] ls = Arrays.copyOf(args, args.length + 2); ls[args.length] = funcCls.getName(); ls[args.length + 1] = SerializationUtils.serialize(lambda); - RayInvocation ri = new RayInvocation(fid, ls); - return scheduler.submit(taskId, createActorId, ri, returnCount, false); + return actorTaskSubmit(taskId, fid, returnCount, false, ls, actor); } - public RayObjects rpc(UniqueID taskId, RayActor actor, Callable funcRun, int returnCount, - Object[] args) { + private byte[] fidFromHook(Callable funcRun) { + MethodSwitcher.IsRemoteCall.set(true); + try { + funcRun.run(); + } catch (Throwable e) { + RayLog.core.error( + "make sure you are using code rewritten using the rewrite tool, see JarRewriter for" + + " options", e); + throw new RuntimeException("make sure you are using code rewritten using the rewrite tool," + + "see JarRewriter for options"); + } + byte[] fid = MethodSwitcher.MethodId.get();//get the identity of function from hook + MethodSwitcher.IsRemoteCall.set(false); + return fid; + } + + private RayObjects submit(UniqueID taskId, + byte[] fid, + int returnCount, + boolean multiReturn, + Object[] args) { + if (taskId == null) { + taskId = UniqueIdHelper.nextTaskId(-1); + } + if (args.length > 0 && args[0].getClass().equals(RayActor.class)) { + return actorTaskSubmit(taskId, fid, returnCount, multiReturn, args, (RayActor) args[0]); + } else { + return taskSubmit(taskId, fid, returnCount, multiReturn, args); + } + } + + private RayObjects actorTaskSubmit(UniqueID taskId, + byte[] fid, + int returnCount, + boolean multiReturn, + Object[] args, + RayActor actor) { + RayInvocation ri = new RayInvocation(fid, args, actor); + RayObjects returnObjs = scheduler.submit(taskId, ri, returnCount + 1, multiReturn); + actor.setTaskCursor(returnObjs.pop().getId()); + return returnObjs; + } + + private RayObjects taskSubmit(UniqueID taskId, + byte[] fid, + int returnCount, + boolean multiReturn, + Object[] args) { + RayInvocation ri = new RayInvocation(fid, args); + return scheduler.submit(taskId, ri, returnCount, multiReturn); + } + + public RayObjects rpcCreateActor(UniqueID taskId, UniqueID createActorId, Callable funcRun, + int returnCount, + Object[] args) { byte[] fid = fidFromHook(funcRun); - return actorTaskSubmit(taskId, fid, returnCount, false, args, actor); + RayInvocation ri = new RayInvocation(fid, new Object[] {}); + return scheduler.submit(taskId, createActorId, ri, returnCount, false); } - public RayObjects rpc(UniqueID taskId, UniqueID functionId, RayActor actor, Class funcCls, - Serializable lambda, int returnCount, Object[] args) { + public RayObjects rpcCreateActor(UniqueID taskId, UniqueID createActorId, UniqueID functionId, + Class funcCls, Serializable lambda, int returnCount, + Object[] args) { byte[] fid = functionId.getBytes(); Object[] ls = Arrays.copyOf(args, args.length + 2); ls[args.length] = funcCls.getName(); ls[args.length + 1] = SerializationUtils.serialize(lambda); - return actorTaskSubmit(taskId, fid, returnCount, false, ls, actor); + RayInvocation ri = new RayInvocation(fid, ls); + return scheduler.submit(taskId, createActorId, ri, returnCount, false); } - public RayMap rpcWithReturnLabels(UniqueID taskId, Callable funcRun, - Collection returnids, - Object[] args) { + public RayMap rpcWithReturnLabels(UniqueID taskId, Callable funcRun, + Collection returnids, + Object[] args) { byte[] fid = fidFromHook(funcRun); if (taskId == null) { taskId = UniqueIdHelper.nextTaskId(-1); @@ -171,10 +188,11 @@ public RayMap rpcWithReturnLabels(UniqueID taskId, Callable fun return scheduler.submit(taskId, new RayInvocation(fid, args), returnids); } - public RayMap rpcWithReturnLabels(UniqueID taskId, Class funcCls, - Serializable lambda, Collection returnids, - Object[] args) { - byte[] fid = UniqueID.nil.getBytes(); + public RayMap rpcWithReturnLabels(UniqueID taskId, Class funcCls, + Serializable lambda, + Collection returnids, + Object[] args) { + final byte[] fid = UniqueID.nil.getBytes(); if (taskId == null) { taskId = UniqueIdHelper.nextTaskId(-1); } @@ -187,8 +205,8 @@ public RayMap rpcWithReturnLabels(UniqueID taskId, Class fun } public RayList rpcWithReturnIndices(UniqueID taskId, Callable funcRun, - Integer returnCount, - Object[] args) { + Integer returnCount, + Object[] args) { byte[] fid = fidFromHook(funcRun); RayObjects objs = submit(taskId, fid, returnCount, true, args); RayList rets = new RayList<>(); @@ -200,8 +218,8 @@ public RayList rpcWithReturnIndices(UniqueID taskId, Callable funcRun, @SuppressWarnings("unchecked") public RayList rpcWithReturnIndices(UniqueID taskId, Class funcCls, - Serializable lambda, Integer returnCount, - Object[] args) { + Serializable lambda, Integer returnCount, + Object[] args) { byte[] fid = UniqueID.nil.getBytes(); Object[] ls = Arrays.copyOf(args, args.length + 2); ls[args.length] = funcCls.getName(); @@ -216,27 +234,11 @@ public RayList rpcWithReturnIndices(UniqueID taskId, Class funcCls, return rets; } - private byte[] fidFromHook(Callable funcRun) { - MethodSwitcher.IsRemoteCall.set(true); - try { - funcRun.run(); - } catch (Throwable e) { - RayLog.core.error( - "make sure you are using code rewritten using the rewrite tool, see JarRewriter for options", - e); - throw new RuntimeException( - "make sure you are using code rewritten using the rewrite tool, see JarRewriter for options"); - } - byte[] fid = MethodSwitcher.MethodId.get();//get the identity of function from hook - MethodSwitcher.IsRemoteCall.set(false); - return fid; - } - - public UniqueID getCurrentTaskID() { + public UniqueID getCurrentTaskId() { return WorkerContext.currentTask().taskId; } - public UniqueID getCurrentTaskNextPutID() { + public UniqueID getCurrentTaskNextPutId() { return UniqueIdHelper.taskComputePutId( WorkerContext.currentTask().taskId, WorkerContext.nextPutIndex()); } diff --git a/java/runtime-common/src/main/java/org/ray/core/WorkerContext.java b/java/runtime-common/src/main/java/org/ray/core/WorkerContext.java index 84d8e5ca3f43..8f53814c30e7 100644 --- a/java/runtime-common/src/main/java/org/ray/core/WorkerContext.java +++ b/java/runtime-common/src/main/java/org/ray/core/WorkerContext.java @@ -6,34 +6,29 @@ public class WorkerContext { + private static final ThreadLocal currentWorkerCtx = + ThreadLocal.withInitial(() -> init(RayRuntime.getParams())); /** - * id of worker + * id of worker. */ - public static UniqueID workerID = UniqueID.randomID(); - + public static UniqueID workerID = UniqueID.randomId(); /** - * current doing task + * current doing task. */ private TaskSpec currentTask; - /** - * current app classloader + * current app classloader. */ private ClassLoader currentClassLoader; - /** - * how many puts done by current task + * how many puts done by current task. */ private int currentTaskPutCount; - /** - * how many calls done by current task + * how many calls done by current task. */ private int currentTaskCallCount; - private static final ThreadLocal currentWorkerCtx = - ThreadLocal.withInitial(() -> init(RayRuntime.getParams())); - public static WorkerContext init(RayParameters params) { WorkerContext ctx = new WorkerContext(); currentWorkerCtx.set(ctx); @@ -48,10 +43,6 @@ public static WorkerContext init(RayParameters params) { return ctx; } - public static WorkerContext get() { - return currentWorkerCtx.get(); - } - public static void prepare(TaskSpec task, ClassLoader classLoader) { WorkerContext wc = get(); wc.currentTask = task; @@ -60,6 +51,10 @@ public static void prepare(TaskSpec task, ClassLoader classLoader) { wc.currentClassLoader = classLoader; } + public static WorkerContext get() { + return currentWorkerCtx.get(); + } + public static TaskSpec currentTask() { return get().currentTask; } @@ -72,7 +67,7 @@ public static int nextCallIndex() { return ++get().currentTaskCallCount; } - public static UniqueID currentWorkerID() { + public static UniqueID currentWorkerId() { return WorkerContext.workerID; } diff --git a/java/runtime-common/src/main/java/org/ray/core/model/RayParameters.java b/java/runtime-common/src/main/java/org/ray/core/model/RayParameters.java index 9108af8de8b4..897b8a61ed6b 100644 --- a/java/runtime-common/src/main/java/org/ray/core/model/RayParameters.java +++ b/java/runtime-common/src/main/java/org/ray/core/model/RayParameters.java @@ -6,7 +6,7 @@ import org.ray.util.config.ConfigReader; /** - * Runtime parameters of Ray process + * Runtime parameters of Ray process. */ public class RayParameters { @@ -115,7 +115,8 @@ public class RayParameters { @AConfig(comment = "whether to disable process failover") public boolean disable_process_failover = false; - @AConfig(comment = "the max size of each file of java worker log, could be set as 10KB, 10MB, 1GB or something similar") + @AConfig(comment = "the max size of each file of java worker log, could be set as 10KB, 10MB, " + + "1GB or something similar") public String max_java_log_file_size = "500MB"; @AConfig(comment = "delay seconds under onebox before app logic for debugging") @@ -124,7 +125,8 @@ public class RayParameters { public RayParameters(ConfigReader config) { if (null != config) { String networkInterface = config.getStringValue("ray.java", "network_interface", null, - "Network interface to be specified for host ip address(e.g., en0, eth0), may use ifconfig to get options"); + "Network interface to be specified for host ip address(e.g., en0, eth0), may use " + + "ifconfig to get options"); node_ip_address = NetworkUtil.getIpAddress(networkInterface); config.readObject("ray.java.start", this, this); } diff --git a/java/runtime-common/src/main/java/org/ray/core/model/RunMode.java b/java/runtime-common/src/main/java/org/ray/core/model/RunMode.java index b230ea052357..d6017d693c0d 100644 --- a/java/runtime-common/src/main/java/org/ray/core/model/RunMode.java +++ b/java/runtime-common/src/main/java/org/ray/core/model/RunMode.java @@ -5,24 +5,21 @@ public enum RunMode { SINGLE_BOX(true, false, true, true), // remote lambda, dev path, native runtime CLUSTER(false, true, false, true); // static rewrite, deploy path, naive runtime + private final boolean remoteLambda; + private final boolean staticRewrite; + private final boolean devPathManager; + private final boolean nativeRuntime; + RunMode(boolean remoteLambda, boolean staticRewrite, boolean devPathManager, - boolean nativeRuntime) { + boolean nativeRuntime) { this.remoteLambda = remoteLambda; this.staticRewrite = staticRewrite; this.devPathManager = devPathManager; this.nativeRuntime = nativeRuntime; } - private final boolean remoteLambda; - - private final boolean staticRewrite; - - private final boolean devPathManager; - - private final boolean nativeRuntime; - /** - * Getter method for property remoteLambda + * Getter method for property remoteLambda. * * @return property value of remoteLambda */ @@ -31,7 +28,7 @@ public boolean isRemoteLambda() { } /** - * Getter method for property staticRewrite + * Getter method for property staticRewrite. * * @return property value of staticRewrite */ @@ -40,7 +37,7 @@ public boolean isStaticRewrite() { } /** - * Getter method for property devPathManager + * Getter method for property devPathManager. * * @return property value of devPathManager */ @@ -49,7 +46,7 @@ public boolean isDevPathManager() { } /** - * Getter method for property nativeRuntime + * Getter method for property nativeRuntime. * * @return property value of nativeRuntime */ diff --git a/java/runtime-common/src/main/java/org/ray/spi/LocalSchedulerProxy.java b/java/runtime-common/src/main/java/org/ray/spi/LocalSchedulerProxy.java index 1c75af967926..fa06210f2468 100644 --- a/java/runtime-common/src/main/java/org/ray/spi/LocalSchedulerProxy.java +++ b/java/runtime-common/src/main/java/org/ray/spi/LocalSchedulerProxy.java @@ -29,27 +29,27 @@ public LocalSchedulerProxy(LocalSchedulerLink scheduler) { } public RayObjects submit(UniqueID taskId, RayInvocation invocation, int returnCount, - boolean multiReturn) { + boolean multiReturn) { UniqueID[] returnIds = buildReturnIds(taskId, returnCount, multiReturn); this.doSubmit(invocation, taskId, returnIds, UniqueID.nil); return new RayObjects(returnIds); } public RayObjects submit(UniqueID taskId, UniqueID createActorId, RayInvocation invocation, - int returnCount, boolean multiReturn) { + int returnCount, boolean multiReturn) { UniqueID[] returnIds = buildReturnIds(taskId, returnCount, multiReturn); this.doSubmit(invocation, taskId, returnIds, createActorId); return new RayObjects(returnIds); } - public RayMap submit(UniqueID taskId, RayInvocation invocation, - Collection userReturnIds) { + public RayMap submit(UniqueID taskId, RayInvocation invocation, + Collection userReturnIds) { UniqueID[] returnIds = buildReturnIds(taskId, userReturnIds.size(), true); - RayMap ret = new RayMap<>(); - Map returnidmapArg = new HashMap<>(); + RayMap ret = new RayMap<>(); + Map returnidmapArg = new HashMap<>(); int index = 0; - for (RID userReturnId : userReturnIds) { + for (RIDT userReturnId : userReturnIds) { if (returnidmapArg.containsKey(userReturnId)) { RayLog.core.error("TaskId " + taskId + " userReturnId is duplicate " + userReturnId); continue; @@ -63,10 +63,10 @@ public RayMap submit(UniqueID taskId, RayInvocation invocation, System.arraycopy(returnIds, 0, newReturnIds, 0, index); returnIds = newReturnIds; } - Object args[] = invocation.getArgs(); + Object[] args = invocation.getArgs(); Object[] newargs; if (args == null) { - newargs = new Object[]{returnidmapArg}; + newargs = new Object[] {returnidmapArg}; } else { newargs = new Object[args.length + 1]; newargs[0] = returnidmapArg; @@ -77,10 +77,19 @@ public RayMap submit(UniqueID taskId, RayInvocation invocation, return ret; } + // build Object IDs of return values. + private UniqueID[] buildReturnIds(UniqueID taskId, int returnCount, boolean multiReturn) { + UniqueID[] returnIds = new UniqueID[returnCount]; + for (int k = 0; k < returnCount; k++) { + returnIds[k] = UniqueIdHelper.taskComputeReturnId(taskId, k, multiReturn); + } + return returnIds; + } + private void doSubmit(RayInvocation invocation, UniqueID taskId, - UniqueID[] returnIds, UniqueID createActorId) { + UniqueID[] returnIds, UniqueID createActorId) { - TaskSpec current = WorkerContext.currentTask(); + final TaskSpec current = WorkerContext.currentTask(); TaskSpec task = new TaskSpec(); task.actorCounter = invocation.getActor().increaseTaskCounter(); task.actorId = invocation.getActor().getId(); @@ -101,19 +110,10 @@ private void doSubmit(RayInvocation invocation, UniqueID taskId, "Task " + taskId + " submitted, functionId = " + task.functionId + " actorId = " + task.actorId + ", driverId = " + task.driverId + ", return_ids = " + Arrays .toString(returnIds) + ", currentTask " + WorkerContext.currentTask().taskId - + " cursorId = " + task.cursorId); + + " cursorId = " + task.cursorId); scheduler.submitTask(task); } - // build Object IDs of return values. - private UniqueID[] buildReturnIds(UniqueID taskId, int returnCount, boolean multiReturn) { - UniqueID[] returnIds = new UniqueID[returnCount]; - for (int k = 0; k < returnCount; k++) { - returnIds[k] = UniqueIdHelper.taskComputeReturnId(taskId, k, multiReturn); - } - return returnIds; - } - public TaskSpec getTask() { TaskSpec ts = scheduler.getTaskTodo(); RayLog.core.info("Task " + ts.taskId.toString() + " received"); diff --git a/java/runtime-common/src/main/java/org/ray/spi/NopRemoteFunctionManager.java b/java/runtime-common/src/main/java/org/ray/spi/NopRemoteFunctionManager.java index 0f848dded30c..2f8ad27f938c 100644 --- a/java/runtime-common/src/main/java/org/ray/spi/NopRemoteFunctionManager.java +++ b/java/runtime-common/src/main/java/org/ray/spi/NopRemoteFunctionManager.java @@ -7,10 +7,12 @@ import org.ray.util.logger.RayLog; /** - * mock version of remote function manager using local loaded jars + runtime hook + * mock version of remote function manager using local loaded jars + runtime hook. */ public class NopRemoteFunctionManager implements RemoteFunctionManager { + private final LoadedFunctions loadedFunctions = new LoadedFunctions(); + public NopRemoteFunctionManager(UniqueID driverId) { //onLoad(driverId, Agent.hookedMethods); //Agent.consumers.add(m -> { this.onLoad(m); }); @@ -48,17 +50,6 @@ public void unregisterApp(UniqueID driverId) { // nothing to do } - private void onLoad(UniqueID driverId, Set methods) { - //assert (startupDriverId().equals(driverId)); - for (MethodId mid : methods) { - onLoad(mid); - } - } - - private void onLoad(MethodId mid) { - loadedFunctions.functions.add(mid); - } - @Override public LoadedFunctions loadFunctions(UniqueID driverId) { //assert (startupDriverId().equals(driverId)); @@ -76,5 +67,14 @@ public void unloadFunctions(UniqueID driverId) { //assert (startupDriverId().equals(driverId)); } - private final LoadedFunctions loadedFunctions = new LoadedFunctions(); + private void onLoad(UniqueID driverId, Set methods) { + //assert (startupDriverId().equals(driverId)); + for (MethodId mid : methods) { + onLoad(mid); + } + } + + private void onLoad(MethodId mid) { + loadedFunctions.functions.add(mid); + } } diff --git a/java/runtime-common/src/main/java/org/ray/spi/ObjectStoreProxy.java b/java/runtime-common/src/main/java/org/ray/spi/ObjectStoreProxy.java index e316db0f488c..3b4b34f8e1f7 100644 --- a/java/runtime-common/src/main/java/org/ray/spi/ObjectStoreProxy.java +++ b/java/runtime-common/src/main/java/org/ray/spi/ObjectStoreProxy.java @@ -2,7 +2,6 @@ import java.util.ArrayList; import java.util.List; - import org.apache.arrow.plasma.ObjectStoreLink; import org.apache.commons.lang3.tuple.Pair; import org.ray.api.RayList; @@ -19,19 +18,21 @@ */ public class ObjectStoreProxy { - public enum GetStatus {SUCCESS, FAILED} - private final ObjectStoreLink store; - - private final int GET_TIMEOUT_MS = 1000; + private final int getTimeoutMs = 1000; public ObjectStoreProxy(ObjectStoreLink store) { this.store = store; } - public Pair get(UniqueID id, int timeout_ms, boolean isMetadata) + public Pair get(UniqueID objectId, boolean isMetadata) + throws TaskExecutionException { + return get(objectId, getTimeoutMs, isMetadata); + } + + public Pair get(UniqueID id, int timeoutMs, boolean isMetadata) throws TaskExecutionException { - byte[] obj = store.get(id.getBytes(), timeout_ms, isMetadata); + byte[] obj = store.get(id.getBytes(), timeoutMs, isMetadata); if (obj != null) { T t = Serializer.decode(obj, WorkerContext.currentClassLoader()); store.release(id.getBytes()); @@ -44,9 +45,9 @@ public Pair get(UniqueID id, int timeout_ms, boolean isMetadat } } - public Pair get(UniqueID objectId, boolean isMetadata) + public List> get(List objectIds, boolean isMetadata) throws TaskExecutionException { - return get(objectId, GET_TIMEOUT_MS, isMetadata); + return get(objectIds, getTimeoutMs, isMetadata); } public List> get(List ids, int timeoutMs, boolean isMetadata) @@ -69,9 +70,13 @@ public List> get(List ids, int timeoutMs, boole return ret; } - public List> get(List objectIds, boolean isMetadata) - throws TaskExecutionException { - return get(objectIds, GET_TIMEOUT_MS, isMetadata); + private static byte[][] getIdBytes(List objectIds) { + int size = objectIds.size(); + byte[][] ids = new byte[size][]; + for (int i = 0; i < size; i++) { + ids[i] = objectIds.get(i).getBytes(); + } + return ids; } public void put(UniqueID id, Object obj, Object metadata) { @@ -111,12 +116,7 @@ public int getFetchSize() { } - private static byte[][] getIdBytes(List objectIds) { - int size = objectIds.size(); - byte[][] ids = new byte[size][]; - for (int i = 0; i < size; i++) { - ids[i] = objectIds.get(i).getBytes(); - } - return ids; + public enum GetStatus { + SUCCESS, FAILED } } diff --git a/java/runtime-common/src/main/java/org/ray/spi/RemoteFunctionManager.java b/java/runtime-common/src/main/java/org/ray/spi/RemoteFunctionManager.java index d08c5eadd93e..bb458c3bf7fb 100644 --- a/java/runtime-common/src/main/java/org/ray/spi/RemoteFunctionManager.java +++ b/java/runtime-common/src/main/java/org/ray/spi/RemoteFunctionManager.java @@ -4,13 +4,12 @@ import org.ray.hook.runtime.LoadedFunctions; /** - * register and load functions from function table + * register and load functions from function table. */ public interface RemoteFunctionManager { - /** - * register mapping, and upload resource - * + /* + * register mapping, and upload resource. * this function is invoked by app proxy or other stand-alone tools it should detect for * duplication first though * @@ -20,7 +19,7 @@ public interface RemoteFunctionManager { UniqueID registerResource(byte[] resourceZip); /** - * download resource content + * download resource content. * * @return resource content */ @@ -28,44 +27,40 @@ public interface RemoteFunctionManager { /** * remove resource by its hash id - * - * be careful of invoking this function to make sure it is no longer used + * be careful of invoking this function to make sure it is no longer used. * * @param resourceId SHA-1 hash of the resource zip bytes */ void unregisterResource(UniqueID resourceId); - /** - * register the mapping to repo - * + /* + * register the mapping to repo, * this function is invoked by whoever initiates the driver id */ void registerApp(UniqueID driverId, UniqueID resourceId); /** - * get the resourceId of one app + * get the resourceId of one app. * * @return resourceId of the app driver */ UniqueID getAppResourceId(UniqueID driverId); - /** + /* * unregister mapping - * * this function is called when the driver exits or detected dead */ void unregisterApp(UniqueID driverId); /** * load resource and functions for this driver this function is used by the workers on demand when - * a required function is not found in {@code LocalFunctionManager} + * a required function is not found in {@code LocalFunctionManager}. */ LoadedFunctions loadFunctions(UniqueID driverId); /** * unload functions for this driver - * - * this function is used by the workers on demand when a driver is dead + * this function is used by the workers on demand when a driver is dead. */ void unloadFunctions(UniqueID driverId); } diff --git a/java/runtime-common/src/main/java/org/ray/spi/model/RayInvocation.java b/java/runtime-common/src/main/java/org/ray/spi/model/RayInvocation.java index 5d7059e2642f..01c2fa60f02a 100644 --- a/java/runtime-common/src/main/java/org/ray/spi/model/RayInvocation.java +++ b/java/runtime-common/src/main/java/org/ray/spi/model/RayInvocation.java @@ -8,21 +8,19 @@ */ public class RayInvocation { + private static final RayActor nil = new RayActor<>(UniqueID.nil, UniqueID.nil); /** - * unique id for a method + * unique id for a method. * * @see UniqueID */ private final byte[] id; + private final RayActor actor; /** - * function arguments + * function arguments. */ private Object[] args; - private final RayActor actor; - - private static final RayActor nil = new RayActor<>(UniqueID.nil, UniqueID.nil); - public RayInvocation(byte[] id, Object[] args) { this(id, args, nil); } diff --git a/java/runtime-common/src/main/java/org/ray/spi/model/RayMethod.java b/java/runtime-common/src/main/java/org/ray/spi/model/RayMethod.java index fac19e1d7e1a..200c2f79b482 100644 --- a/java/runtime-common/src/main/java/org/ray/spi/model/RayMethod.java +++ b/java/runtime-common/src/main/java/org/ray/spi/model/RayMethod.java @@ -3,7 +3,7 @@ import java.lang.reflect.Method; /** - * method info + * method info. */ public class RayMethod { @@ -11,6 +11,11 @@ public class RayMethod { public final String fullName; // TODO: other annotated information + public RayMethod(Method m) { + invokable = m; + fullName = m.getDeclaringClass().getName() + "." + m.getName(); + } + public void check() { for (Class paramCls : invokable.getParameterTypes()) { if (paramCls.isPrimitive()) { @@ -19,9 +24,4 @@ public void check() { } } } - - public RayMethod(Method m) { - invokable = m; - fullName = m.getDeclaringClass().getName() + "." + m.getName(); - } } diff --git a/java/runtime-dev/pom.xml b/java/runtime-dev/pom.xml index 7f710ed870e8..9f8a7d47e0e7 100644 --- a/java/runtime-dev/pom.xml +++ b/java/runtime-dev/pom.xml @@ -1,54 +1,54 @@ - - - org.ray.parent - ray-superpom - 1.0 - - 4.0.0 - org.ray - ray-runtime-dev - - runtime-dev - runtime for app development - - - jar - - - - org.ray - ray-api - 1.0 - - - org.ray - ray-runtime-common - 1.0 - - - de.ruedigermoeller - fst - 2.47 - - - - - com.github.davidmoten - flatbuffers-java - 1.7.0.1 - - - - - redis.clients - jedis - 2.8.0 - - + + + org.ray.parent + ray-superpom + 1.0 + + 4.0.0 + org.ray + ray-runtime-dev + + runtime-dev + runtime for app development + + + jar + + + + org.ray + ray-api + 1.0 + + + org.ray + ray-runtime-common + 1.0 + + + de.ruedigermoeller + fst + 2.47 + + + + + com.github.davidmoten + flatbuffers-java + 1.7.0.1 + + + + + redis.clients + jedis + 2.8.0 + + diff --git a/java/runtime-dev/src/main/java/org/ray/core/impl/RayDevRuntime.java b/java/runtime-dev/src/main/java/org/ray/core/impl/RayDevRuntime.java index e3fcb6715f9c..114f533b8e47 100644 --- a/java/runtime-dev/src/main/java/org/ray/core/impl/RayDevRuntime.java +++ b/java/runtime-dev/src/main/java/org/ray/core/impl/RayDevRuntime.java @@ -21,21 +21,11 @@ public class RayDevRuntime extends RayRuntime { - protected RayDevRuntime() { - } + private final ConcurrentHashMap actors = new ConcurrentHashMap<>(); - @Override - public void start(RayParameters params) { - PathConfig pathConfig = new PathConfig(configReader); - RemoteFunctionManager rfm = new NopRemoteFunctionManager(params.driver_id); - MockObjectStore store = new MockObjectStore(); - MockLocalScheduler scheduler = new MockLocalScheduler(store); - init(scheduler, store, rfm, pathConfig); - scheduler.setLocalFunctionManager(this.functions); + protected RayDevRuntime() { } - private final ConcurrentHashMap actors = new ConcurrentHashMap<>(); - @RayRemote private static byte[] createActor(String className) { return ((RayDevRuntime) RayRuntime.getInstance()).createLocalActor(className); @@ -56,7 +46,9 @@ private byte[] createLocalActor(String className) { actors.put(actorId, r); RayLog.core.info("TaskId " + taskId + ", create actor ok " + actorId); return actorId.getBytes(); - } catch (ClassNotFoundException | InstantiationException | IllegalAccessException | IllegalArgumentException | InvocationTargetException | NoSuchMethodException | SecurityException e) { + } catch (ClassNotFoundException | InstantiationException | IllegalAccessException + | IllegalArgumentException | InvocationTargetException | NoSuchMethodException + | SecurityException e) { // TODO Auto-generated catch block e.printStackTrace(); String logInfo = @@ -69,8 +61,18 @@ private byte[] createLocalActor(String className) { } @Override - public RayActor create(Class cls) { - return new RayActor<>(Ray.call(RayDevRuntime::createActor, cls.getName()).getId()); + public void start(RayParameters params) { + PathConfig pathConfig = new PathConfig(configReader); + RemoteFunctionManager rfm = new NopRemoteFunctionManager(params.driver_id); + MockObjectStore store = new MockObjectStore(); + MockLocalScheduler scheduler = new MockLocalScheduler(store); + init(scheduler, store, rfm, pathConfig); + scheduler.setLocalFunctionManager(this.functions); + } + + @Override + public void cleanUp() { + // nothing to do } @Override @@ -79,7 +81,7 @@ public Object getLocalActor(UniqueID id) { } @Override - public void cleanUp() { - // nothing to do + public RayActor create(Class cls) { + return new RayActor<>(Ray.call(RayDevRuntime::createActor, cls.getName()).getId()); } } diff --git a/java/runtime-dev/src/main/java/org/ray/spi/impl/MockLocalScheduler.java b/java/runtime-dev/src/main/java/org/ray/spi/impl/MockLocalScheduler.java index c03895d3dcd6..89bbee1b1f0b 100644 --- a/java/runtime-dev/src/main/java/org/ray/spi/impl/MockLocalScheduler.java +++ b/java/runtime-dev/src/main/java/org/ray/spi/impl/MockLocalScheduler.java @@ -15,34 +15,46 @@ */ public class MockLocalScheduler implements LocalSchedulerLink { - private final Map> waitTasks_ = new ConcurrentHashMap<>(); - private final MockObjectStore store_; - private LocalFunctionManager functions_ = null; + private final Map> waitTasks = new ConcurrentHashMap<>(); + private final MockObjectStore store; + private LocalFunctionManager functions = null; public MockLocalScheduler(MockObjectStore store) { - store_ = store; + this.store = store; store.registerScheduler(this); } public void setLocalFunctionManager(LocalFunctionManager mgr) { - functions_ = mgr; + functions = mgr; } public void onObjectPut(UniqueID id) { - Map bucket = waitTasks_.get(id); + Map bucket = waitTasks.get(id); if (bucket != null) { - waitTasks_.remove(id); + waitTasks.remove(id); for (TaskSpec ts : bucket.values()) { submitTask(ts); } } } + @Override + public void submitTask(TaskSpec task) { + UniqueID id = isTaskReady(task); + if (id == null) { + Worker.execute(task, functions); + } else { + Map bucket = waitTasks + .computeIfAbsent(id, id_ -> new ConcurrentHashMap<>()); + bucket.put(id, task); + } + } + private UniqueID isTaskReady(TaskSpec spec) { for (FunctionArg arg : spec.args) { if (arg.ids != null) { for (UniqueID id : arg.ids) { - if (!store_.isObjectReady(id)) { + if (!store.isObjectReady(id)) { return id; } } @@ -51,18 +63,6 @@ private UniqueID isTaskReady(TaskSpec spec) { return null; } - @Override - public void submitTask(TaskSpec task) { - UniqueID id = isTaskReady(task); - if (id == null) { - Worker.execute(task, functions_); - } else { - Map bucket = waitTasks_ - .computeIfAbsent(id, id_ -> new ConcurrentHashMap<>()); - bucket.put(id, task); - } - } - @Override public TaskSpec getTaskTodo() { throw new RuntimeException("invalid execution flow here"); diff --git a/java/runtime-dev/src/main/java/org/ray/spi/impl/MockObjectStore.java b/java/runtime-dev/src/main/java/org/ray/spi/impl/MockObjectStore.java index 5fe20065b6ee..96b3f0bad173 100644 --- a/java/runtime-dev/src/main/java/org/ray/spi/impl/MockObjectStore.java +++ b/java/runtime-dev/src/main/java/org/ray/spi/impl/MockObjectStore.java @@ -5,7 +5,6 @@ import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; - import org.apache.arrow.plasma.ObjectStoreLink; import org.ray.api.UniqueID; import org.ray.core.WorkerContext; @@ -16,9 +15,9 @@ */ public class MockObjectStore implements ObjectStoreLink { - private final Map data_ = new ConcurrentHashMap<>(); - private final Map metadata_ = new ConcurrentHashMap<>(); - private MockLocalScheduler scheduler_ = null; + private final Map data = new ConcurrentHashMap<>(); + private final Map metadata = new ConcurrentHashMap<>(); + private MockLocalScheduler scheduler = null; @Override public void put(byte[] objectId, byte[] value, byte[] metadataValue) { @@ -27,47 +26,33 @@ public void put(byte[] objectId, byte[] value, byte[] metadataValue) { .error(logPrefix() + "cannot put null: " + objectId + "," + Arrays.toString(value)); System.exit(-1); } - UniqueID uniqueID = new UniqueID(objectId); - data_.put(uniqueID, value); - metadata_.put(uniqueID, metadataValue); + UniqueID uniqueId = new UniqueID(objectId); + data.put(uniqueId, value); + metadata.put(uniqueId, metadataValue); - if (scheduler_ != null) { - scheduler_.onObjectPut(uniqueID); + if (scheduler != null) { + scheduler.onObjectPut(uniqueId); } } @Override public List get(byte[][] objectIds, int timeoutMs, boolean isMetadata) { - final Map dataMap = isMetadata ? metadata_ : data_; + final Map dataMap = isMetadata ? metadata : data; ArrayList rets = new ArrayList<>(objectIds.length); for (byte[] objId : objectIds) { - UniqueID uniqueID = new UniqueID(objId); - RayLog.core.info(logPrefix() + " is notified for objectid " + uniqueID); - rets.add(dataMap.get(uniqueID)); + UniqueID uniqueId = new UniqueID(objId); + RayLog.core.info(logPrefix() + " is notified for objectid " + uniqueId); + rets.add(dataMap.get(uniqueId)); } return rets; } - private String logPrefix() { - return WorkerContext.currentTask().taskId + "-" + getUserTrace() + " -> "; - } - - private String getUserTrace() { - StackTraceElement stes[] = Thread.currentThread().getStackTrace(); - int k = 1; - while (stes[k].getClassName().startsWith("org.ray") - && !stes[k].getClassName().contains("test")) { - k++; - } - return stes[k].getFileName() + ":" + stes[k].getLineNumber(); - } - @Override public List wait(byte[][] objectIds, int timeoutMs, int numReturns) { ArrayList rets = new ArrayList<>(); for (byte[] objId : objectIds) { //tod test - if (data_.containsKey(new UniqueID(objId))) { + if (data.containsKey(new UniqueID(objId))) { rets.add(objId); } } @@ -89,14 +74,6 @@ public long evict(long numBytes) { return 0; } - public boolean isObjectReady(UniqueID id) { - return data_.containsKey(id); - } - - public void registerScheduler(MockLocalScheduler s) { - scheduler_ = s; - } - @Override public void release(byte[] objectId) { return; @@ -105,6 +82,28 @@ public void release(byte[] objectId) { @Override public boolean contains(byte[] objectId) { - return data_.containsKey(new UniqueID(objectId)); + return data.containsKey(new UniqueID(objectId)); + } + + private String logPrefix() { + return WorkerContext.currentTask().taskId + "-" + getUserTrace() + " -> "; + } + + private String getUserTrace() { + StackTraceElement[] stes = Thread.currentThread().getStackTrace(); + int k = 1; + while (stes[k].getClassName().startsWith("org.ray") + && !stes[k].getClassName().contains("test")) { + k++; + } + return stes[k].getFileName() + ":" + stes[k].getLineNumber(); + } + + public boolean isObjectReady(UniqueID id) { + return data.containsKey(id); + } + + public void registerScheduler(MockLocalScheduler s) { + scheduler = s; } } diff --git a/java/runtime-native/assembly.xml b/java/runtime-native/assembly.xml index 1684a11d6f99..d787d9587c0b 100644 --- a/java/runtime-native/assembly.xml +++ b/java/runtime-native/assembly.xml @@ -1,15 +1,15 @@ - ear - - zip - - - - true - lib - - - + ear + + zip + + + + true + lib + + + - + diff --git a/java/runtime-native/pom.xml b/java/runtime-native/pom.xml index 2f7eca83e7ae..ef74c55d4cb7 100644 --- a/java/runtime-native/pom.xml +++ b/java/runtime-native/pom.xml @@ -1,82 +1,82 @@ - - - org.ray.parent - ray-superpom - 1.0 - - 4.0.0 + + + org.ray.parent + ray-superpom + 1.0 + + 4.0.0 - org.ray - ray-runtime-native + org.ray + ray-runtime-native - native runtime for ray - native runtime for ray - + native runtime for ray + native runtime for ray + - jar + jar - - - org.ray - ray-runtime-common - 1.0 - + + + org.ray + ray-runtime-common + 1.0 + - - - commons-io - commons-io - 2.5 - + + + commons-io + commons-io + 2.5 + - - net.lingala.zip4j - zip4j - 1.3.2 - + + net.lingala.zip4j + zip4j + 1.3.2 + - - commons-codec - commons-codec - 1.4 - - - com.google.guava - guava - 19.0 - - - org.apache.arrow - arrow-plasma - - - - ray-runtime-deploy - - - org.apache.maven.plugins - maven-assembly-plugin - - - assembly.xml - - - - - make-assembly - install - - single - - - - - + + commons-codec + commons-codec + 1.4 + + + com.google.guava + guava + 19.0 + + + org.apache.arrow + arrow-plasma + + + + ray-runtime-deploy + + + org.apache.maven.plugins + maven-assembly-plugin + + + assembly.xml + + + + + make-assembly + install + + single + + + + + - + diff --git a/java/runtime-native/src/main/java/org/ray/core/impl/RayNativeRuntime.java b/java/runtime-native/src/main/java/org/ray/core/impl/RayNativeRuntime.java index 96ad00d5148c..df3b55de61fe 100644 --- a/java/runtime-native/src/main/java/org/ray/core/impl/RayNativeRuntime.java +++ b/java/runtime-native/src/main/java/org/ray/core/impl/RayNativeRuntime.java @@ -5,10 +5,11 @@ import java.util.HashMap; import java.util.List; import java.util.Map; - import org.apache.arrow.plasma.ObjectStoreLink; import org.apache.arrow.plasma.PlasmaClient; -import org.ray.api.*; +import org.ray.api.RayActor; +import org.ray.api.RayRemote; +import org.ray.api.UniqueID; import org.ray.api.funcs.RayFunc_2_1; import org.ray.core.RayRuntime; import org.ray.core.UniqueIdHelper; @@ -31,7 +32,7 @@ import org.ray.util.logger.RayLog; /** - * native runtime for local box and cluster run + * native runtime for local box and cluster run. */ public class RayNativeRuntime extends RayRuntime { @@ -44,25 +45,12 @@ public class RayNativeRuntime extends RayRuntime { private StateStoreProxy stateStoreProxy; private KeyValueStoreLink kvStore = null; private RunManager manager = null; - - @Override - public void cleanUp() { - if (null != manager) { - manager.cleanup(true); - } - } + private Object actor = null; + private UniqueID actorId = UniqueID.nil; protected RayNativeRuntime() { } - private void initStateStore(String redisAddress) throws Exception { - kvStore = new RedisClient(); - kvStore.SetAddr(redisAddress); - stateStoreProxy = new StateStoreProxyImpl(kvStore); - //stateStoreProxy.setStore(kvStore); - stateStoreProxy.initializeGlobalState(); - } - @Override public void start(RayParameters params) throws Exception { boolean isWorker = (params.worker_mode == WorkerMode.WORKER); @@ -86,13 +74,14 @@ public void start(RayParameters params) throws Exception { } // initialize remote function manager - RemoteFunctionManager funcMgr = params.run_mode.isStaticRewrite() ? - new NativeRemoteFunctionManager(kvStore) : + RemoteFunctionManager funcMgr = params.run_mode.isStaticRewrite() + ? new NativeRemoteFunctionManager(kvStore) : new NopRemoteFunctionManager(params.driver_id); // initialize worker context if (params.worker_mode == WorkerMode.DRIVER) { - // TODO: The relationship between workerID, driver_id and dummy_task.driver_id should be recheck carefully + // TODO: The relationship between workerID, driver_id and dummy_task.driver_id should be + // recheck carefully WorkerContext.workerID = params.driver_id; } WorkerContext.init(params); @@ -109,20 +98,21 @@ public void start(RayParameters params) throws Exception { if (params.worker_mode != WorkerMode.NONE) { String overwrites = ""; // initialize the links - int release_delay = RayRuntime.configReader + int releaseDelay = RayRuntime.configReader .getIntegerValue("ray", "plasma_default_release_delay", 0, "how many release requests should be delayed in plasma client"); - ObjectStoreLink pLink = new PlasmaClient(params.object_store_name, params.object_store_manager_name, release_delay); + ObjectStoreLink plink = new PlasmaClient(params.object_store_name, params + .object_store_manager_name, releaseDelay); - LocalSchedulerLink sLink = new DefaultLocalSchedulerClient( + LocalSchedulerLink slink = new DefaultLocalSchedulerClient( params.local_scheduler_name, - WorkerContext.currentWorkerID(), + WorkerContext.currentWorkerId(), UniqueID.nil, isWorker, 0 ); - init(sLink, pLink, funcMgr, pathConfig); + init(slink, plink, funcMgr, pathConfig); // register registerWorker(isWorker, params.node_ip_address, params.object_store_name, @@ -136,24 +126,48 @@ public void start(RayParameters params) throws Exception { ); } + @Override + public void cleanUp() { + if (null != manager) { + manager.cleanup(true); + } + } + + @Override + public Object getLocalActor(UniqueID id) { + if (actorId.equals(id)) { + return actor; + } else { + return null; + } + } + private void startOnebox(RayParameters params, PathConfig paths) throws Exception { params.cleanup = true; manager = new RunManager(params, paths, RayRuntime.configReader); manager.startRayHead(); params.redis_address = manager.info().redisAddress; - params.object_store_name = manager.info().local_stores.get(0).storeName; - params.object_store_manager_name = manager.info().local_stores.get(0).managerName; - params.local_scheduler_name = manager.info().local_stores.get(0).schedulerName; + params.object_store_name = manager.info().localStores.get(0).storeName; + params.object_store_manager_name = manager.info().localStores.get(0).managerName; + params.local_scheduler_name = manager.info().localStores.get(0).schedulerName; //params.node_ip_address = NetworkUtil.getIpAddress(); } - private void registerWorker(boolean isWorker, String node_ip_address, String storeName, - String managerName, String schedulerName) { + private void initStateStore(String redisAddress) throws Exception { + kvStore = new RedisClient(); + kvStore.setAddr(redisAddress); + stateStoreProxy = new StateStoreProxyImpl(kvStore); + //stateStoreProxy.setStore(kvStore); + stateStoreProxy.initializeGlobalState(); + } + + private void registerWorker(boolean isWorker, String nodeIpAddress, String storeName, + String managerName, String schedulerName) { Map workerInfo = new HashMap<>(); - String workerId = new String(WorkerContext.currentWorkerID().getBytes()); + String workerId = new String(WorkerContext.currentWorkerId().getBytes()); if (!isWorker) { - workerInfo.put("node_ip_address", node_ip_address); + workerInfo.put("node_ip_address", nodeIpAddress); workerInfo.put("driver_id", workerId); workerInfo.put("start_time", String.valueOf(System.currentTimeMillis())); workerInfo.put("plasma_store_socket", storeName); @@ -161,31 +175,17 @@ private void registerWorker(boolean isWorker, String node_ip_address, String sto workerInfo.put("local_scheduler_socket", schedulerName); workerInfo.put("name", System.getProperty("user.dir")); //TODO: worker.redis_client.hmset(b"Drivers:" + worker.workerId, driver_info) - kvStore.Hmset("Drivers:" + workerId, workerInfo); + kvStore.hmset("Drivers:" + workerId, workerInfo); } else { - workerInfo.put("node_ip_address", node_ip_address); - //TODO: - /* - "stdout_file": os.path.abspath(log_stdout_file.name), - "stderr_file": os.path.abspath(log_stderr_file.name), - */ + workerInfo.put("node_ip_address", nodeIpAddress); workerInfo.put("plasma_store_socket", storeName); workerInfo.put("plasma_manager_socket", managerName); workerInfo.put("local_scheduler_socket", schedulerName); //TODO: b"Workers:" + worker.workerId, - kvStore.Hmset("Workers:" + workerId, workerInfo); + kvStore.hmset("Workers:" + workerId, workerInfo); } } - private Object _actor = null; - private UniqueID actorID = UniqueID.nil; - - @RayRemote - public static byte[] createActorInActor(byte[] actorId, String className) { - ((RayNativeRuntime) RayRuntime.getInstance()).localCreateActorInActor(actorId, className); - return actorId; - } - @SuppressWarnings("unchecked") @Override public RayActor create(Class cls) { @@ -202,7 +202,7 @@ public RayActor create(Class cls) { RayFunc_2_1.class, createActorLambda, 1, - new Object[]{actorId.getBytes(), cls.getName()} + new Object[] {actorId.getBytes(), cls.getName()} ).getObjs()[0].getId(); } else { cursorId = worker.rpcCreateActor( @@ -210,16 +210,22 @@ public RayActor create(Class cls) { actorId, () -> RayNativeRuntime.createActorInActor(null, null), 1, - new Object[]{actorId.getBytes(), cls.getName()} + new Object[] {actorId.getBytes(), cls.getName()} ).getObjs()[0].getId(); } actor.setTaskCursor(cursorId); return actor; } + @RayRemote + public static byte[] createActorInActor(byte[] actorId, String className) { + ((RayNativeRuntime) RayRuntime.getInstance()).localCreateActorInActor(actorId, className); + return actorId; + } + public Object localCreateActorInActor(byte[] actorId, String className) { try { - actorID = new UniqueID(actorId); + this.actorId = new UniqueID(actorId); Class cls = Class.forName(className, true, Thread.currentThread().getContextClassLoader()); Constructor[] cts = cls.getConstructors(); @@ -227,25 +233,18 @@ public Object localCreateActorInActor(byte[] actorId, String className) { System.err.println(ct.getName() + ", param count = " + ct.getParameterCount()); } - _actor = cls.getConstructor(new Class[0]).newInstance(); - RayLog.core.info("create actor " + actorID + " inside actor ok"); - return _actor; - } catch (ClassNotFoundException | InstantiationException | IllegalAccessException | IllegalArgumentException | InvocationTargetException | NoSuchMethodException | SecurityException e) { + actor = cls.getConstructor(new Class[0]).newInstance(); + RayLog.core.info("create actor " + this.actorId + " inside actor ok"); + return actor; + } catch (ClassNotFoundException | InstantiationException | IllegalAccessException + | IllegalArgumentException | InvocationTargetException | NoSuchMethodException + | SecurityException e) { e.printStackTrace(); - String log = "create actor " + actorID + " for " + className + " failed, ex = " + e + String log = "create actor " + this.actorId + " for " + className + " failed, ex = " + e .getMessage(); System.err.println(log); RayLog.core.error(log, e); throw new TaskExecutionException(log, e); } } - - @Override - public Object getLocalActor(UniqueID id) { - if (actorID.equals(id)) { - return _actor; - } else { - return null; - } - } } diff --git a/java/runtime-native/src/main/java/org/ray/runner/RunInfo.java b/java/runtime-native/src/main/java/org/ray/runner/RunInfo.java index 56bb28ff7e9c..bb58cff7eb97 100644 --- a/java/runtime-native/src/main/java/org/ray/runner/RunInfo.java +++ b/java/runtime-native/src/main/java/org/ray/runner/RunInfo.java @@ -6,23 +6,17 @@ import org.ray.spi.model.AddressInfo; /** - * information of kinds of processes + * information of kinds of processes. */ public class RunInfo { public String redisAddress; public List redisShards; - public List local_stores = new ArrayList<>(); + public List localStores = new ArrayList<>(); public ArrayList> allProcesses = initProcessInfoArray(); public ArrayList> toBeCleanedProcesses = initProcessArray(); public ArrayList deadProcess = new ArrayList<>(); - public enum ProcessType { - PT_WORKER, PT_LOCAL_SCHEDULER, PT_PLASMA_MANAGER, PT_PLASMA_STORE, - PT_GLOBAL_SCHEDULER, PT_REDIS_SERVER, PT_WEB_UI, - PT_DRIVER - } - private ArrayList> initProcessArray() { ArrayList> processes = new ArrayList<>(); for (ProcessType ignored : ProcessType.values()) { @@ -38,4 +32,10 @@ private ArrayList> initProcessInfoArray() { } return processes; } + + public enum ProcessType { + PT_WORKER, PT_LOCAL_SCHEDULER, PT_PLASMA_MANAGER, PT_PLASMA_STORE, + PT_GLOBAL_SCHEDULER, PT_REDIS_SERVER, PT_WEB_UI, + PT_DRIVER + } } diff --git a/java/runtime-native/src/main/java/org/ray/runner/RunManager.java b/java/runtime-native/src/main/java/org/ray/runner/RunManager.java index 5c3deb4d6796..f38638793a31 100644 --- a/java/runtime-native/src/main/java/org/ray/runner/RunManager.java +++ b/java/runtime-native/src/main/java/org/ray/runner/RunManager.java @@ -22,7 +22,7 @@ import redis.clients.jedis.Jedis; /** - * Ray service management on one box + * Ray service management on one box. */ public class RunManager { @@ -40,6 +40,21 @@ public class RunManager { private RunInfo runInfo = new RunInfo(); + public RunManager(RayParameters params, PathConfig paths, ConfigReader configReader) { + this.params = params; + this.paths = paths; + this.configReader = configReader; + } + + private static boolean killProcess(Process p) { + if (p.isAlive()) { + p.destroyForcibly(); + return true; + } else { + return false; + } + } + public RunInfo info() { return runInfo; } @@ -56,12 +71,6 @@ public String getProcStderrFileName() { return procStderrFileName; } - public RunManager(RayParameters params, PathConfig paths, ConfigReader configReader) { - this.params = params; - this.paths = paths; - this.configReader = configReader; - } - public void startRayHead() throws Exception { if (params.redis_address.length() != 0) { throw new Exception("Redis address must be empty in head node."); @@ -99,6 +108,44 @@ public void startRayNode() throws Exception { startRayProcesses(); } + public Process startDriver(String mainClass, String redisAddress, UniqueID driverId, + String workDir, String ip, + String driverClass, String additonalClassPaths, String + additionalConfigs) { + String driverConfigs = + "ray.java.start.driver_id=" + driverId + ";ray.java.start.driver_class=" + driverClass; + if (null != additionalConfigs) { + additionalConfigs += ";" + driverConfigs; + } else { + additionalConfigs = driverConfigs; + } + + return startJavaProcess( + RunInfo.ProcessType.PT_DRIVER, + mainClass, + additonalClassPaths, + additionalConfigs, + "", + workDir, + ip, + redisAddress, + true, + false, + null + ); + } + + private Process startJavaProcess(RunInfo.ProcessType pt, String mainClass, + String additonalClassPaths, String additionalConfigs, + String additionalJvmArgs, String workDir, String ip, String + redisAddr, boolean redirect, + boolean cleanup, String agentlibAddr) { + + String cmd = buildJavaProcessCommand(pt, mainClass, additonalClassPaths, additionalConfigs, + additionalJvmArgs, workDir, ip, redisAddr, agentlibAddr); + return startProcess(cmd.split(" "), null, pt, workDir, redisAddr, ip, redirect, cleanup); + } + private String buildJavaProcessCommand( RunInfo.ProcessType pt, String mainClass, String additionalClassPaths, String additionalConfigs, @@ -137,44 +184,89 @@ private String buildJavaProcessCommand( return cmd; } - private Process startJavaProcess(RunInfo.ProcessType pt, String mainClass, - String additonalClassPaths, String additionalConfigs, - String additionalJvmArgs, String workDir, String ip, String redisAddr, boolean redirect, - boolean cleanup, String agentlibAddr) { + private Process startProcess(String[] cmd, Map env, RunInfo.ProcessType type, + String workDir, + String redisAddress, String ip, boolean redirect, + boolean cleanup) { + File wdir = new File(workDir); + if (!wdir.exists()) { + wdir.mkdirs(); + } - String cmd = buildJavaProcessCommand(pt, mainClass, additonalClassPaths, additionalConfigs, - additionalJvmArgs, workDir, ip, redisAddr, agentlibAddr); - return startProcess(cmd.split(" "), null, pt, workDir, redisAddr, ip, redirect, cleanup); - } + int processIndex = runInfo.allProcesses.get(type.ordinal()).size(); + ProcessBuilder builder; + List newCmd = Arrays.stream(cmd).filter(s -> s.length() > 0) + .collect(Collectors.toList()); + builder = new ProcessBuilder(newCmd); + builder.directory(new File(workDir)); + if (redirect) { + String stdoutFile; + String stderrFile; + stdoutFile = workDir + "/" + processIndex + ".out.txt"; + stderrFile = workDir + "/" + processIndex + ".err.txt"; + builder.redirectOutput(new File(stdoutFile)); + builder.redirectError(new File(stderrFile)); + List stdFileList = new ArrayList<>(); + stdFileList.add(stdoutFile); + stdFileList.add(stderrFile); + record_log_files_in_redis(redisAddress, ip, stdFileList); + procStdoutFileName = stdoutFile; + procStderrFileName = stderrFile; + } - public Process startDriver(String mainClass, String redisAddress, UniqueID driverId, - String workDir, String ip, - String driverClass, String additonalClassPaths, String additionalConfigs) { - String driverConfigs = - "ray.java.start.driver_id=" + driverId + ";ray.java.start.driver_class=" + driverClass; - if (null != additionalConfigs) { - additionalConfigs += ";" + driverConfigs; - } else { - additionalConfigs = driverConfigs; + if (env != null && !env.isEmpty()) { + builder.environment().putAll(env); } - return startJavaProcess( - RunInfo.ProcessType.PT_DRIVER, - mainClass, - additonalClassPaths, - additionalConfigs, - "", - workDir, - ip, - redisAddress, - true, - false, - null - ); + Process p = null; + try { + p = builder.start(); + } catch (IOException e) { + RayLog.core + .error("Start process " + Arrays.toString(cmd).replace(',', ' ') + " in working dir '" + + workDir + "' failed", + e); + return null; + } + + RayLog.core.info( + "Start process " + p.hashCode() + " OK, cmd = " + Arrays.toString(cmd).replace(',', ' ') + + ", working dir = '" + workDir + "'" + (redirect ? ", redirect" : ", no redirect")); + + if (cleanup) { + runInfo.toBeCleanedProcesses.get(type.ordinal()).add(p); + } + + ProcessInfo processInfo = new ProcessInfo(); + processInfo.cmd = cmd; + processInfo.type = type; + processInfo.workDir = workDir; + processInfo.redisAddress = redisAddress; + processInfo.ip = ip; + processInfo.redirect = redirect; + processInfo.cleanup = cleanup; + processInfo.process = p; + runInfo.allProcesses.get(type.ordinal()).add(processInfo); + + return p; + } + + private void record_log_files_in_redis(String redisAddress, String nodeIpAddress, + List logfiles) { + if (redisAddress != null && !redisAddress.isEmpty() && nodeIpAddress != null + && !nodeIpAddress.isEmpty() && logfiles.size() > 0) { + String[] ipPort = redisAddress.split(":"); + Jedis jedisClient = new Jedis(ipPort[0], Integer.parseInt(ipPort[1])); + String logFileListKey = String.format("LOG_FILENAMES:{%s}", nodeIpAddress); + for (String logfile : logfiles) { + jedisClient.rpush(logFileListKey, logfile); + } + jedisClient.close(); + } } public void startRayProcesses() { - Jedis _redis_client = null; + Jedis redisClient = null; RayLog.core.info("start ray processes @ " + params.node_ip_address + " ..."); @@ -185,14 +277,14 @@ public void startRayProcesses() { params.redis_address = primaryShards.get(0); String[] args = params.redis_address.split(":"); - _redis_client = new Jedis(args[0], Integer.parseInt(args[1])); + redisClient = new Jedis(args[0], Integer.parseInt(args[1])); // Register the number of Redis shards in the primary shard, so that clients // know how many redis shards to expect under RedisShards. - _redis_client.set("NumRedisShards", Integer.toString(params.num_redis_shards)); + redisClient.set("NumRedisShards", Integer.toString(params.num_redis_shards)); } else { String[] args = params.redis_address.split(":"); - _redis_client = new Jedis(args[0], Integer.parseInt(args[1])); + redisClient = new Jedis(args[0], Integer.parseInt(args[1])); } runInfo.redisAddress = params.redis_address; @@ -206,10 +298,10 @@ public void startRayProcesses() { // Store redis shard information in the primary redis shard. for (int i = 0; i < runInfo.redisShards.size(); i++) { String addr = runInfo.redisShards.get(i); - _redis_client.rpush("RedisShards", addr); + redisClient.rpush("RedisShards", addr); } } - _redis_client.close(); + redisClient.close(); // start global scheduler if (params.include_global_scheduler) { @@ -236,12 +328,12 @@ public void startRayProcesses() { assert (params.num_gpus.length == params.num_local_schedulers); } - int[] local_num_workers = new int[params.num_local_schedulers]; + int[] localNumWorkers = new int[params.num_local_schedulers]; if (params.num_workers == 0) { - System.arraycopy(params.num_cpus, 0, local_num_workers, 0, params.num_local_schedulers); + System.arraycopy(params.num_cpus, 0, localNumWorkers, 0, params.num_local_schedulers); } else { for (int i = 0; i < params.num_local_schedulers; i++) { - local_num_workers[i] = params.num_workers; + localNumWorkers[i] = params.num_workers; } } @@ -257,7 +349,7 @@ public void startRayProcesses() { params.working_directory + "/storeManager", params.redis_address, params.node_ip_address, params.redirect, params.cleanup); - runInfo.local_stores.add(info); + runInfo.localStores.add(info); } // start local scheduler @@ -265,11 +357,11 @@ public void startRayProcesses() { int workerCount = 0; if (params.start_workers_from_local_scheduler) { - workerCount = local_num_workers[i]; - local_num_workers[i] = 0; + workerCount = localNumWorkers[i]; + localNumWorkers[i] = 0; } - startLocalScheduler(i, runInfo.local_stores.get(i), + startLocalScheduler(i, runInfo.localStores.get(i), params.num_cpus[i], params.num_gpus[i], workerCount, params.working_directory + "/localScheduler", params.redis_address, params.node_ip_address, params.redirect, params.cleanup); @@ -277,13 +369,13 @@ public void startRayProcesses() { // start local workers for (int i = 0; i < params.num_local_schedulers; i++) { - runInfo.local_stores.get(i).workerCount = local_num_workers[i]; - for (int j = 0; j < local_num_workers[i]; j++) { - startWorker(runInfo.local_stores.get(i).storeName, - runInfo.local_stores.get(i).managerName, runInfo.local_stores.get(i).schedulerName, - params.working_directory + "/worker" + i + "." + j, params.redis_address, - params.node_ip_address, UniqueID.nil, "", - params.redirect, params.cleanup); + runInfo.localStores.get(i).workerCount = localNumWorkers[i]; + for (int j = 0; j < localNumWorkers[i]; j++) { + startWorker(runInfo.localStores.get(i).storeName, + runInfo.localStores.get(i).managerName, runInfo.localStores.get(i).schedulerName, + params.working_directory + "/worker" + i + "." + j, params.redis_address, + params.node_ip_address, UniqueID.nil, "", + params.redirect, params.cleanup); } } @@ -302,10 +394,11 @@ public boolean checkAlive(HashSet excludeTypes) { } ProcessInfo p; - for (int j = 0; j < runInfo.allProcesses.get(i).size();) { + for (int j = 0; j < runInfo.allProcesses.get(i).size(); ) { p = runInfo.allProcesses.get(i).get(j); if (!p.process.isAlive()) { - RayLog.core.error("Process " + p.hashCode() + " is not alive!" + " Process Type " + types[i].name()); + RayLog.core.error("Process " + p.hashCode() + " is not alive!" + " Process Type " + + types[i].name()); runInfo.deadProcess.add(p); runInfo.allProcesses.get(i).remove(j); } else { @@ -378,96 +471,6 @@ public void cleanup(boolean killAll) { } } - private void record_log_files_in_redis(String redis_address, String node_ip_address, - List logfiles) { - if (redis_address != null && !redis_address.isEmpty() && node_ip_address != null - && !node_ip_address.isEmpty() && logfiles.size() > 0) { - String[] ip_port = redis_address.split(":"); - Jedis jedis_client = new Jedis(ip_port[0], Integer.parseInt(ip_port[1])); - String log_file_list_key = String.format("LOG_FILENAMES:{%s}", node_ip_address); - for (String logfile : logfiles) { - jedis_client.rpush(log_file_list_key, logfile); - } - jedis_client.close(); - } - } - - private Process startProcess(String[] cmd, Map env, RunInfo.ProcessType type, - String workDir, - String redisAddress, String ip, boolean redirect, - boolean cleanup) { - File wdir = new File(workDir); - if (!wdir.exists()) { - wdir.mkdirs(); - } - - int processIndex = runInfo.allProcesses.get(type.ordinal()).size(); - ProcessBuilder builder; - List newCmd = Arrays.stream(cmd).filter(s -> s.length() > 0) - .collect(Collectors.toList()); - builder = new ProcessBuilder(newCmd); - builder.directory(new File(workDir)); - if (redirect) { - String stdout_file; - String stderr_file; - stdout_file = workDir + "/" + processIndex + ".out.txt"; - stderr_file = workDir + "/" + processIndex + ".err.txt"; - builder.redirectOutput(new File(stdout_file)); - builder.redirectError(new File(stderr_file)); - List std_file_list = new ArrayList<>(); - std_file_list.add(stdout_file); - std_file_list.add(stderr_file); - record_log_files_in_redis(redisAddress, ip, std_file_list); - procStdoutFileName = stdout_file; - procStderrFileName = stderr_file; - } - - if (env != null && !env.isEmpty()) { - builder.environment().putAll(env); - } - - Process p = null; - try { - p = builder.start(); - } catch (IOException e) { - RayLog.core - .error("Start process " + Arrays.toString(cmd).replace(',', ' ') + " in working dir '" - + workDir + "' failed", - e); - return null; - } - - RayLog.core.info( - "Start process " + p.hashCode() + " OK, cmd = " + Arrays.toString(cmd).replace(',', ' ') - + ", working dir = '" + workDir + "'" + (redirect ? ", redirect" : ", no redirect")); - - if (cleanup) { - runInfo.toBeCleanedProcesses.get(type.ordinal()).add(p); - } - - ProcessInfo processInfo = new ProcessInfo(); - processInfo.cmd = cmd; - processInfo.type = type; - processInfo.workDir = workDir; - processInfo.redisAddress = redisAddress; - processInfo.ip = ip; - processInfo.redirect = redirect; - processInfo.cleanup = cleanup; - processInfo.process = p; - runInfo.allProcesses.get(type.ordinal()).add(processInfo); - - return p; - } - - private static boolean killProcess(Process p) { - if (p.isAlive()) { - p.destroyForcibly(); - return true; - } else { - return false; - } - } - // // start a redis server // @@ -481,7 +484,7 @@ private static boolean killProcess(Process p) { // @return primary redis shard address // private List startRedis(String workDir, String ip, int port, int numOfShards, - boolean redirect, boolean cleanup) { + boolean redirect, boolean cleanup) { ArrayList shards = new ArrayList<>(); String addr; for (int i = 0; i < numOfShards; i++) { @@ -509,7 +512,7 @@ private List startRedis(String workDir, String ip, int port, int numOfSh // @return redis server address // private String startRedisInstance(String workDir, String ip, int port, - boolean redirect, boolean cleanup) { + boolean redirect, boolean cleanup) { String redisFilePath = paths.redis_server; String redisModule = paths.redis_module; @@ -531,6 +534,7 @@ private String startRedisInstance(String workDir, String ip, int port, try { TimeUnit.MILLISECONDS.sleep(300); } catch (InterruptedException e) { + e.printStackTrace(); } Jedis client = new Jedis(params.node_ip_address, port); @@ -546,7 +550,7 @@ private String startRedisInstance(String workDir, String ip, int port, } private void startGlobalScheduler(String workDir, String redisAddress, String ip, - boolean redirect, boolean cleanup) { + boolean redirect, boolean cleanup) { String filePath = paths.global_scheduler; String cmd = filePath + " -r " + redisAddress + " -h " + ip; @@ -592,9 +596,9 @@ private Map retrieveEnv(String conf, Map env) { * start */ private void startLocalScheduler(int index, AddressInfo info, int numCpus, - int numGpus, int numWorkers, String workDir, - String redisAddress, String ip, boolean redirect, - boolean cleanup) { + int numGpus, int numWorkers, String workDir, + String redisAddress, String ip, boolean redirect, + boolean cleanup) { //if (numCpus <= 0) // numCpus = Runtime.getRuntime().availableProcessors(); if (numGpus <= 0) { @@ -606,7 +610,7 @@ private void startLocalScheduler(int index, AddressInfo info, int numCpus, String name = "/tmp/scheduler" + rpcPort; String rpcAddr = ""; String cmd = filePath + " -s " + name + " -p " + info.storeName + " -h " + ip + " -n " - + numWorkers + " -c " + "CPU," + INT16_MAX +",GPU,0"; + + numWorkers + " -c " + "CPU," + INT16_MAX + ",GPU,0"; assert (info.managerName.length() > 0); assert (info.storeName.length() > 0); @@ -616,7 +620,7 @@ private void startLocalScheduler(int index, AddressInfo info, int numCpus, String workerCmd = null; workerCmd = buildWorkerCommand(true, info.storeName, info.managerName, name, UniqueID.nil, - "", workDir + rpcPort, ip, redisAddress); + "", workDir + rpcPort, ip, redisAddress); cmd += " -w \"" + workerCmd + "\""; if (redisAddress.length() > 0) { @@ -627,7 +631,7 @@ private void startLocalScheduler(int index, AddressInfo info, int numCpus, } Map env = null; - String[] cmds = StringUtil.Split(cmd, " ", "\"", "\"").toArray(new String[0]); + String[] cmds = StringUtil.split(cmd, " ", "\"", "\"").toArray(new String[0]); Process p = startProcess(cmds, env, RunInfo.ProcessType.PT_LOCAL_SCHEDULER, workDir + rpcPort, redisAddress, ip, redirect, cleanup); @@ -635,6 +639,7 @@ private void startLocalScheduler(int index, AddressInfo info, int numCpus, try { TimeUnit.MILLISECONDS.sleep(100); } catch (InterruptedException e) { + e.printStackTrace(); } } @@ -649,8 +654,9 @@ private void startLocalScheduler(int index, AddressInfo info, int numCpus, } private String buildWorkerCommand(boolean isFromLocalScheduler, String storeName, - String storeManagerName, String localSchedulerName, - UniqueID actorId, String actorClass, String workDir, String ip, String redisAddress) { + String storeManagerName, String localSchedulerName, + UniqueID actorId, String actorClass, String workDir, String + ip, String redisAddress) { String workerConfigs = "ray.java.start.object_store_name=" + storeName + ";ray.java.start.object_store_manager_name=" + storeManagerName + ";ray.java.start.worker_mode=WORKER" @@ -680,9 +686,9 @@ private String buildWorkerCommand(boolean isFromLocalScheduler, String storeName } private void startObjectStore(int index, AddressInfo info, String workDir, String redisAddress, - String ip, boolean redirect, boolean cleanup) { - int occupiedMemoryMB = params.object_store_occupied_memory_MB; - long memoryBytes = occupiedMemoryMB * 1000000; + String ip, boolean redirect, boolean cleanup) { + int occupiedMemoryMb = params.object_store_occupied_memory_MB; + long memoryBytes = occupiedMemoryMb * 1000000; String filePath = paths.store; int rpcPort = params.object_store_rpc_port + index; String name = "/tmp/plasma_store" + rpcPort; @@ -697,6 +703,7 @@ private void startObjectStore(int index, AddressInfo info, String workDir, Strin try { TimeUnit.MILLISECONDS.sleep(100); } catch (InterruptedException e) { + e.printStackTrace(); } } @@ -711,8 +718,8 @@ private void startObjectStore(int index, AddressInfo info, String workDir, Strin } private AddressInfo startObjectManager(int index, AddressInfo info, String workDir, - String redisAddress, String ip, boolean redirect, - boolean cleanup) { + String redisAddress, String ip, boolean redirect, + boolean cleanup) { String filePath = paths.store_manager; int rpcPort = params.object_store_manager_rpc_port + index; String name = "/tmp/plasma_manager" + rpcPort; @@ -730,6 +737,7 @@ private AddressInfo startObjectManager(int index, AddressInfo info, String workD try { TimeUnit.MILLISECONDS.sleep(100); } catch (InterruptedException e) { + e.printStackTrace(); } } @@ -744,9 +752,9 @@ private AddressInfo startObjectManager(int index, AddressInfo info, String workD } public void startWorker(String storeName, String storeManagerName, - String localSchedulerName, String workDir, String redisAddress, - String ip, UniqueID actorId, String actorClass, - boolean redirect, boolean cleanup) { + String localSchedulerName, String workDir, String redisAddress, + String ip, UniqueID actorId, String actorClass, + boolean redirect, boolean cleanup) { String cmd = buildWorkerCommand(false, storeName, storeManagerName, localSchedulerName, actorId, actorClass, workDir, ip, redisAddress); startProcess(cmd.split(" "), null, RunInfo.ProcessType.PT_WORKER, workDir, redisAddress, ip, diff --git a/java/runtime-native/src/main/java/org/ray/runner/worker/DefaultDriver.java b/java/runtime-native/src/main/java/org/ray/runner/worker/DefaultDriver.java index 261998e55e68..d8cdde44652a 100644 --- a/java/runtime-native/src/main/java/org/ray/runner/worker/DefaultDriver.java +++ b/java/runtime-native/src/main/java/org/ray/runner/worker/DefaultDriver.java @@ -4,7 +4,7 @@ import org.ray.core.model.WorkerMode; /** - * + * The main function of DefaultDriver. */ public class DefaultDriver { @@ -22,7 +22,7 @@ public static void main(String[] args) { .getStringValue("ray.java.start", "driver_class", "", "java class which main is served as the driver in a java worker"); Class cls = Class.forName(driverClass); - cls.getMethod("main", String[].class).invoke(null, (Object) new String[]{}); + cls.getMethod("main", String[].class).invoke(null, (Object) new String[] {}); } catch (Throwable e) { e.printStackTrace(); System.exit(-1); diff --git a/java/runtime-native/src/main/java/org/ray/runner/worker/DefaultWorker.java b/java/runtime-native/src/main/java/org/ray/runner/worker/DefaultWorker.java index a14403087384..0cb2b367f0a0 100644 --- a/java/runtime-native/src/main/java/org/ray/runner/worker/DefaultWorker.java +++ b/java/runtime-native/src/main/java/org/ray/runner/worker/DefaultWorker.java @@ -4,7 +4,7 @@ import org.ray.core.model.WorkerMode; /** - * default worker implementation + * default worker implementation. */ public class DefaultWorker { diff --git a/java/runtime-native/src/main/java/org/ray/spi/FileStoreLink.java b/java/runtime-native/src/main/java/org/ray/spi/FileStoreLink.java index 13523a0d6f62..b097b4cff8bd 100644 --- a/java/runtime-native/src/main/java/org/ray/spi/FileStoreLink.java +++ b/java/runtime-native/src/main/java/org/ray/spi/FileStoreLink.java @@ -5,9 +5,6 @@ public interface FileStoreLink { - /** - * - */ boolean mkdirs(String f); /** @@ -32,11 +29,12 @@ public interface FileStoreLink { boolean isFile(String f); /** - * Delete a file. + * delete a file. * - * @param f the path to delete. + * @param f the path to delete. * @param recursive if path is a directory and set to true, the directory is deleted else throws - * an exception. In case of a file the recursive can be set to either true or false. + * an exception. In case of a file the recursive can be set to either true or + * false. * @return true if delete is successful else false. */ boolean delete(String f, boolean recursive); diff --git a/java/runtime-native/src/main/java/org/ray/spi/KeyValueStoreLink.java b/java/runtime-native/src/main/java/org/ray/spi/KeyValueStoreLink.java index 3e58762e96e1..6d21fa8e8245 100644 --- a/java/runtime-native/src/main/java/org/ray/spi/KeyValueStoreLink.java +++ b/java/runtime-native/src/main/java/org/ray/spi/KeyValueStoreLink.java @@ -5,94 +5,94 @@ import java.util.Set; /** - * Ray K/V abstraction + * Ray K/V abstraction. */ public interface KeyValueStoreLink { /** - * set address of kv store: format "ip:port" + * set address of kv store: format "ip:port". */ - void SetAddr(String addr); + void setAddr(String addr); /** - * check if the kvstore client connected + * check if the kvstore client connected. */ - void CheckConnected() throws Exception; + void checkConnected() throws Exception; /** - * Set Key-value into State Store, such as redis + * set Key-value into State Store, such as redis. * - * @param key the key to set + * @param key the key to set * @param value the value to set * @param field the field is being set when the item is a hash If it is not hash field should be - * filled with null + * filled with null * @return If the key(or field) already exists, and the StateStoreSet just produced an update of - * the value, 0 is returned, otherwise if a new key(or field) is created 1 is returned. + * the value, 0 is returned, otherwise if a new key(or field) is created 1 is returned. */ - Long Set(final String key, final String value, final String field); + Long set(final String key, final String value, final String field); - Long Set(final byte[] key, final byte[] value, final byte[] field); + Long set(final byte[] key, final byte[] value, final byte[] field); /** - * multi hash value set + * multi hash value set. * - * @param key the key in kvStore + * @param key the key in kvStore * @param hash the multi hash value to be set * @return Return OK or Exception if hash is empty */ - String Hmset(final String key, final Map hash); + String hmset(final String key, final Map hash); - String Hmset(final byte[] key, final Map hash); + String hmset(final byte[] key, final Map hash); /** - * multi hash value get + * multi hash value get. * - * @param key the key in kvStore + * @param key the key in kvStore * @param fields the fields to be get * @return Multi Bulk Reply specifically a list of all the values associated with the specified - * fields, in the same order of the request. + * fields, in the same order of the request. */ - List Hmget(final String key, final String... fields); + List hmget(final String key, final String... fields); - List Hmget(final byte[] key, final byte[]... fields); + List hmget(final byte[] key, final byte[]... fields); /** - * Get the value of the specified key from State Store + * get the value of the specified key from State Store. * - * @param key the key to get + * @param key the key to get * @param field the field is being got when the item is a hash If it is not hash field should be - * filled with null + * filled with null * @return Bulk reply If the key does not exist null is returned. */ - String Get(final String key, final String field); + String get(final String key, final String field); - byte[] Get(final byte[] key, final byte[] field); + byte[] get(final byte[] key, final byte[] field); /** - * Delete the key(or the specified field of the key) from State Store + * delete the key(or the specified field of the key) from State Store. * - * @param key the key to delete + * @param key the key to delete * @param field the field is to delete when the item is a hash If it is not hash field should be - * filled with null + * filled with null * @return Integer reply, specifically: an integer greater than 0 if the key(or the field) was - * removed 0 if none of the specified key existed + * removed 0 if none of the specified key existed */ - Long Delete(final String key, final String field); + Long delete(final String key, final String field); - Long Delete(final byte[] key, final byte[] field); + Long delete(final byte[] key, final byte[] field); /** - * get all keys which fit the pattern + * get all keys which fit the pattern. */ - Set Keys(final byte[] pattern); + Set keys(final byte[] pattern); /** - * get all keys which fit the pattern + * get all keys which fit the pattern. */ - Set Keys(String pattern); + Set keys(String pattern); /** - * get all hash of the key + * get all hash of the key. */ Map hgetAll(final byte[] key); @@ -101,25 +101,26 @@ public interface KeyValueStoreLink { * * @return Multi bulk reply, specifically a list of elements in the specified range. */ - List Lrange(final String key, final long start, final long end); + List lrange(final String key, final long start, final long end); /** + * Rpush. * @return Integer reply, specifically, the number of elements inside the list after the push - * operation. + * operation. */ - Long Rpush(final String key, final String... strings); + Long rpush(final String key, final String... strings); - Long Rpush(final byte[] key, final byte[]... strings); + Long rpush(final byte[] key, final byte[]... strings); /** - * + * Publish. * @param channel To which channel the message will be published * @param message What to publish * @return the number of clients that received the message */ - Long Publish(final String channel, final String message); + Long publish(final String channel, final String message); - Long Publish(byte[] channel, byte[] message); + Long publish(byte[] channel, byte[] message); - Object GetImpl(); + Object getImpl(); } diff --git a/java/runtime-native/src/main/java/org/ray/spi/StateStoreProxy.java b/java/runtime-native/src/main/java/org/ray/spi/StateStoreProxy.java index 4d2c2fa33e8c..2ecaadc3ae11 100644 --- a/java/runtime-native/src/main/java/org/ray/spi/StateStoreProxy.java +++ b/java/runtime-native/src/main/java/org/ray/spi/StateStoreProxy.java @@ -5,28 +5,31 @@ import org.ray.spi.model.AddressInfo; /** - * Proxy client for state store, for instance redis + * Proxy client for state store, for instance redis. */ public interface StateStoreProxy { /** + * setStore. * @param rayKvStore the underlying kv store used to store states */ void setStore(KeyValueStoreLink rayKvStore); /** - * initialize the store + * initialize the store. */ void initializeGlobalState() throws Exception; /** + * keys. * @param pattern filter which keys you are interested in. */ Set keys(final String pattern); /** + * getAddressInfo. * @return list of address information */ - List getAddressInfo(final String node_ip_address, int num_retries); + List getAddressInfo(final String nodeIpAddress, int numRetries); } diff --git a/java/runtime-native/src/main/java/org/ray/spi/impl/Arg.java b/java/runtime-native/src/main/java/org/ray/spi/impl/Arg.java index 1b6c78c622f8..9f6a20215988 100644 --- a/java/runtime-native/src/main/java/org/ray/spi/impl/Arg.java +++ b/java/runtime-native/src/main/java/org/ray/spi/impl/Arg.java @@ -2,43 +2,91 @@ // automatically generated by the FlatBuffers compiler, do not modify -import java.nio.*; -import java.lang.*; -import java.util.*; -import com.google.flatbuffers.*; +import com.google.flatbuffers.FlatBufferBuilder; +import com.google.flatbuffers.Table; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; @SuppressWarnings("unused") public final class Arg extends Table { - public static Arg getRootAsArg(ByteBuffer _bb) { return getRootAsArg(_bb, new Arg()); } - public static Arg getRootAsArg(ByteBuffer _bb, Arg obj) { _bb.order(ByteOrder.LITTLE_ENDIAN); return (obj.__assign(_bb.getInt(_bb.position()) + _bb.position(), _bb)); } - public void __init(int _i, ByteBuffer _bb) { bb_pos = _i; bb = _bb; } - public Arg __assign(int _i, ByteBuffer _bb) { __init(_i, _bb); return this; } + public static Arg getRootAsArg(ByteBuffer _bb) { + return getRootAsArg(_bb, new Arg()); + } + + public static Arg getRootAsArg(ByteBuffer _bb, Arg obj) { + _bb.order(ByteOrder.LITTLE_ENDIAN); + return (obj.__assign(_bb.getInt(_bb.position()) + _bb.position(), _bb)); + } - public String objectIds(int j) { int o = __offset(4); return o != 0 ? __string(__vector(o) + j * 4) : null; } - public int objectIdsLength() { int o = __offset(4); return o != 0 ? __vector_len(o) : 0; } - public String data() { int o = __offset(6); return o != 0 ? __string(o + bb_pos) : null; } - public ByteBuffer dataAsByteBuffer() { return __vector_as_bytebuffer(6, 1); } + public Arg __assign(int _i, ByteBuffer _bb) { + __init(_i, _bb); + return this; + } + + public void __init(int _i, ByteBuffer _bb) { + bb_pos = _i; + bb = _bb; + } public static int createArg(FlatBufferBuilder builder, - int object_idsOffset, - int dataOffset) { + int object_idsOffset, + int dataOffset) { builder.startObject(2); Arg.addData(builder, dataOffset); Arg.addObjectIds(builder, object_idsOffset); return Arg.endArg(builder); } - public static void startArg(FlatBufferBuilder builder) { builder.startObject(2); } - public static void addObjectIds(FlatBufferBuilder builder, int objectIdsOffset) { builder.addOffset(0, objectIdsOffset, 0); } - public static int createObjectIdsVector(FlatBufferBuilder builder, int[] data) { builder.startVector(4, data.length, 4); for (int i = data.length - 1; i >= 0; i--) builder.addOffset(data[i]); return builder.endVector(); } - public static void startObjectIdsVector(FlatBufferBuilder builder, int numElems) { builder.startVector(4, numElems, 4); } - public static void addData(FlatBufferBuilder builder, int dataOffset) { builder.addOffset(1, dataOffset, 0); } + public static void addData(FlatBufferBuilder builder, int dataOffset) { + builder.addOffset(1, dataOffset, 0); + } + + public static void addObjectIds(FlatBufferBuilder builder, int objectIdsOffset) { + builder.addOffset(0, objectIdsOffset, 0); + } + public static int endArg(FlatBufferBuilder builder) { int o = builder.endObject(); return o; } - //this is manually added to avoid encoding/decoding cost as our object id is a byte array instead of a string + public static void startArg(FlatBufferBuilder builder) { + builder.startObject(2); + } + + public static int createObjectIdsVector(FlatBufferBuilder builder, int[] data) { + builder.startVector(4, data.length, 4); + for (int i = data.length - 1; i >= 0; i--) { + builder.addOffset(data[i]); + } + return builder.endVector(); + } + + public static void startObjectIdsVector(FlatBufferBuilder builder, int numElems) { + builder.startVector(4, numElems, 4); + } + + public String objectIds(int j) { + int o = __offset(4); + return o != 0 ? __string(__vector(o) + j * 4) : null; + } + + public int objectIdsLength() { + int o = __offset(4); + return o != 0 ? __vector_len(o) : 0; + } + + public String data() { + int o = __offset(6); + return o != 0 ? __string(o + bb_pos) : null; + } + + public ByteBuffer dataAsByteBuffer() { + return __vector_as_bytebuffer(6, 1); + } + + //this is manually added to avoid encoding/decoding cost as our object id is a byte array + // instead of a string public ByteBuffer objectIdAsByteBuffer(int j) { int o = __offset(4); if (o == 0) { diff --git a/java/runtime-native/src/main/java/org/ray/spi/impl/DefaultLocalSchedulerClient.java b/java/runtime-native/src/main/java/org/ray/spi/impl/DefaultLocalSchedulerClient.java index d3f67649a413..db88a6f7666f 100644 --- a/java/runtime-native/src/main/java/org/ray/spi/impl/DefaultLocalSchedulerClient.java +++ b/java/runtime-native/src/main/java/org/ray/spi/impl/DefaultLocalSchedulerClient.java @@ -13,145 +13,74 @@ import org.ray.util.logger.RayLog; /** - * JNI-based local scheduler link provider + * JNI-based local scheduler link provider. */ public class DefaultLocalSchedulerClient implements LocalSchedulerLink { + private static ThreadLocal _taskBuffer = ThreadLocal.withInitial(() -> { + ByteBuffer bb = ByteBuffer + .allocateDirect(RayRuntime.getParams().max_submit_task_buffer_size_bytes); + bb.order(ByteOrder.LITTLE_ENDIAN); + return bb; + }); + private long client = 0; + public DefaultLocalSchedulerClient(String schedulerSockName, UniqueID clientId, UniqueID actorId, - boolean isWorker, long numGpus) { - _client = _init(schedulerSockName, clientId.getBytes(), actorId.getBytes(), isWorker, + boolean isWorker, long numGpus) { + client = _init(schedulerSockName, clientId.getBytes(), actorId.getBytes(), isWorker, numGpus); } + private static native long _init(String localSchedulerSocket, byte[] workerId, byte[] actorId, + boolean isWorker, long numGpus); + + private static native byte[] _computePutId(long client, byte[] taskId, int putIndex); + + private static native void _task_done(long client); @Override public void submitTask(TaskSpec task) { - ByteBuffer info = TaskSpec2Info(task); + ByteBuffer info = taskSpec2Info(task); byte[] a = null; - if(!task.actorId.isNil()) { + if (!task.actorId.isNil()) { a = task.cursorId.getBytes(); } - _submitTask(_client, a, info, info.position(), info.remaining()); + _submitTask(client, a, info, info.position(), info.remaining()); } @Override public TaskSpec getTaskTodo() { - byte[] bytes = _getTaskTodo(_client); + byte[] bytes = _getTaskTodo(client); assert (null != bytes); ByteBuffer bb = ByteBuffer.wrap(bytes); - return TaskInfo2Spec(bb); - } - - public void destroy() { - _destroy(_client); + return taskInfo2Spec(bb); } @Override - public void notifyUnblocked() { - _notify_unblocked(_client); + public void markTaskPutDependency(UniqueID taskId, UniqueID objectId) { + _put_object(client, taskId.getBytes(), objectId.getBytes()); } @Override public void reconstructObject(UniqueID objectId) { - _reconstruct_object(_client, objectId.getBytes()); + _reconstruct_object(client, objectId.getBytes()); } @Override - public void markTaskPutDependency(UniqueID taskId, UniqueID objectId) { - _put_object(_client, taskId.getBytes(), objectId.getBytes()); + public void notifyUnblocked() { + _notify_unblocked(client); } - private long _client = 0; - - private static ThreadLocal _taskBuffer = ThreadLocal.withInitial(() -> { - ByteBuffer bb = ByteBuffer - .allocateDirect(RayRuntime.getParams().max_submit_task_buffer_size_bytes); - bb.order(ByteOrder.LITTLE_ENDIAN); - return bb; - }); - - public static ByteBuffer TaskSpec2Info(TaskSpec task) { - ByteBuffer bb = _taskBuffer.get(); - bb.clear(); - - FlatBufferBuilder fbb = new FlatBufferBuilder(bb); - - int driver_idOffset = fbb.createString(task.driverId.ToByteBuffer()); - int task_idOffset = fbb.createString(task.taskId.ToByteBuffer()); - int parent_task_idOffset = fbb.createString(task.parentTaskId.ToByteBuffer()); - int parent_counter = task.parentCounter; - int actorCreate_idOffset = fbb.createString(task.createActorId.ToByteBuffer()); - int actorCreateDummy_idOffset = fbb.createString(UniqueID.nil.ToByteBuffer()); - int actor_idOffset = fbb.createString(task.actorId.ToByteBuffer()); - int actor_handle_idOffset = fbb.createString(task.actorHandleId.ToByteBuffer()); - int actor_counter = task.actorCounter; - int function_idOffset = fbb.createString(task.functionId.ToByteBuffer()); - - // serialize args - int[] argsOffsets = new int[task.args.length]; - for (int i = 0; i < argsOffsets.length; i++) { - - int object_idOffset = 0; - int dataOffset = 0; - if (task.args[i].ids != null) { - int id_count = task.args[i].ids.size(); - int[] idOffsets = new int[id_count]; - for (int k = 0; k < id_count; k++) { - idOffsets[k] = fbb.createString(task.args[i].ids.get(k).ToByteBuffer()); - } - object_idOffset = fbb.createVectorOfTables(idOffsets); - } - if (task.args[i].data != null) { - dataOffset = fbb.createString(ByteBuffer.wrap(task.args[i].data)); - } - - argsOffsets[i] = Arg.createArg(fbb, object_idOffset, dataOffset); - } - int argsOffset = fbb.createVectorOfTables(argsOffsets); - - // serialize returns - int return_count = task.returnIds.length; - int[] returnsOffsets = new int[return_count]; - for (int k = 0; k < return_count; k++) { - returnsOffsets[k] = fbb.createString(task.returnIds[k].ToByteBuffer()); - } - int returnsOffset = fbb.createVectorOfTables(returnsOffsets); - - // serialize required resources - // The required_resources vector indicates the quantities of the different - // resources required by this task. The index in this vector corresponds to - // the resource type defined in the ResourceIndex enum. For example, - - int[] required_resourcesOffsets = new int[1]; - for (int i = 0; i < required_resourcesOffsets.length; i++) { - int keyOffset = 0; - keyOffset = fbb.createString(ByteBuffer.wrap("CPU".getBytes())); - required_resourcesOffsets[i] = ResourcePair.createResourcePair(fbb,keyOffset,0.0); - } - int requiredResourcesOffset = fbb.createVectorOfTables(required_resourcesOffsets); - - int root = TaskInfo.createTaskInfo( - fbb, driver_idOffset, task_idOffset, - parent_task_idOffset, parent_counter, - actorCreate_idOffset, actorCreateDummy_idOffset, - actor_idOffset, actor_handle_idOffset, actor_counter, - false, function_idOffset, - argsOffset, returnsOffset, requiredResourcesOffset); + private static native void _notify_unblocked(long client); - fbb.finish(root); - ByteBuffer buffer = fbb.dataBuffer(); + private static native void _reconstruct_object(long client, byte[] objectId); - if (buffer.remaining() > RayRuntime.getParams().max_submit_task_buffer_size_bytes) { - RayLog.core.error( - "Allocated buffer is not enough to transfer the task specification: " + RayRuntime - .getParams().max_submit_task_buffer_size_bytes + " vs " + buffer.remaining()); - assert (false); - } + private static native void _put_object(long client, byte[] taskId, byte[] objectId); - return buffer; - } + // return TaskInfo (in FlatBuffer) + private static native byte[] _getTaskTodo(long client); - public static TaskSpec TaskInfo2Spec(ByteBuffer bb) { + public static TaskSpec taskInfo2Spec(ByteBuffer bb) { bb.order(ByteOrder.LITTLE_ENDIAN); TaskInfo info = TaskInfo.getRootAsTaskInfo(bb); @@ -171,10 +100,10 @@ public static TaskSpec TaskInfo2Spec(ByteBuffer bb) { FunctionArg darg = new FunctionArg(); Arg sarg = info.args(i); - int id_count = sarg.objectIdsLength(); - if (id_count > 0) { + int idCount = sarg.objectIdsLength(); + if (idCount > 0) { darg.ids = new ArrayList<>(); - for (int j = 0; j < id_count; j++) { + for (int j = 0; j < idCount; j++) { ByteBuffer lbb = sarg.objectIdAsByteBuffer(j); assert (lbb != null && lbb.remaining() > 0); darg.ids.add(new UniqueID(lbb)); @@ -203,24 +132,94 @@ public static TaskSpec TaskInfo2Spec(ByteBuffer bb) { return spec; } - native private static long _init(String localSchedulerSocket, byte[] workerId, byte[] actorId, - boolean isWorker, long numGpus); + public static ByteBuffer taskSpec2Info(TaskSpec task) { + ByteBuffer bb = _taskBuffer.get(); + bb.clear(); - // task -> TaskInfo (with FlatBuffer) - native private static void _submitTask(long client, byte[] cursorId, /*Direct*/ByteBuffer task, int pos, int sz); + FlatBufferBuilder fbb = new FlatBufferBuilder(bb); - // return TaskInfo (in FlatBuffer) - native private static byte[] _getTaskTodo(long client); + final int driverIdOffset = fbb.createString(task.driverId.toByteBuffer()); + final int taskIdOffset = fbb.createString(task.taskId.toByteBuffer()); + final int parentTaskIdOffset = fbb.createString(task.parentTaskId.toByteBuffer()); + final int parentCounter = task.parentCounter; + final int actorCreateIdOffset = fbb.createString(task.createActorId.toByteBuffer()); + final int actorCreateDummyIdOffset = fbb.createString(UniqueID.nil.toByteBuffer()); + final int actorIdOffset = fbb.createString(task.actorId.toByteBuffer()); + final int actorHandleIdOffset = fbb.createString(task.actorHandleId.toByteBuffer()); + final int actorCounter = task.actorCounter; + final int functionIdOffset = fbb.createString(task.functionId.toByteBuffer()); + + // serialize args + int[] argsOffsets = new int[task.args.length]; + for (int i = 0; i < argsOffsets.length; i++) { + + int objectIdOffset = 0; + int dataOffset = 0; + if (task.args[i].ids != null) { + int idCount = task.args[i].ids.size(); + int[] idOffsets = new int[idCount]; + for (int k = 0; k < idCount; k++) { + idOffsets[k] = fbb.createString(task.args[i].ids.get(k).toByteBuffer()); + } + objectIdOffset = fbb.createVectorOfTables(idOffsets); + } + if (task.args[i].data != null) { + dataOffset = fbb.createString(ByteBuffer.wrap(task.args[i].data)); + } + + argsOffsets[i] = Arg.createArg(fbb, objectIdOffset, dataOffset); + } + int argsOffset = fbb.createVectorOfTables(argsOffsets); - native private static byte[] _computePutId(long client, byte[] taskId, int putIndex); + // serialize returns + int returnCount = task.returnIds.length; + int[] returnsOffsets = new int[returnCount]; + for (int k = 0; k < returnCount; k++) { + returnsOffsets[k] = fbb.createString(task.returnIds[k].toByteBuffer()); + } + int returnsOffset = fbb.createVectorOfTables(returnsOffsets); - native private static void _destroy(long client); + // serialize required resources + // The required_resources vector indicates the quantities of the different + // resources required by this task. The index in this vector corresponds to + // the resource type defined in the ResourceIndex enum. For example, - native private static void _task_done(long client); + int[]requiredResourcesOffsets = new int[1]; + for (int i = 0; i < requiredResourcesOffsets.length; i++) { + int keyOffset = 0; + keyOffset = fbb.createString(ByteBuffer.wrap("CPU".getBytes())); + requiredResourcesOffsets[i] = ResourcePair.createResourcePair(fbb, keyOffset, 0.0); + } + int requiredResourcesOffset = fbb.createVectorOfTables(requiredResourcesOffsets); - native private static void _reconstruct_object(long client, byte[] objectId); + int root = TaskInfo.createTaskInfo( + fbb, driverIdOffset, taskIdOffset, + parentTaskIdOffset, parentCounter, + actorCreateIdOffset, actorCreateDummyIdOffset, + actorIdOffset, actorHandleIdOffset, actorCounter, + false, functionIdOffset, + argsOffset, returnsOffset, requiredResourcesOffset); + + fbb.finish(root); + ByteBuffer buffer = fbb.dataBuffer(); - native private static void _notify_unblocked(long client); + if (buffer.remaining() > RayRuntime.getParams().max_submit_task_buffer_size_bytes) { + RayLog.core.error( + "Allocated buffer is not enough to transfer the task specification: " + RayRuntime + .getParams().max_submit_task_buffer_size_bytes + " vs " + buffer.remaining()); + assert (false); + } + + return buffer; + } + + // task -> TaskInfo (with FlatBuffer) + private static native void _submitTask(long client, byte[] cursorId, /*Direct*/ByteBuffer task, + int pos, int sz); + + public void destroy() { + _destroy(client); + } - native private static void _put_object(long client, byte[] taskId, byte[] objectId); + private static native void _destroy(long client); } diff --git a/java/runtime-native/src/main/java/org/ray/spi/impl/NativeRemoteFunctionManager.java b/java/runtime-native/src/main/java/org/ray/spi/impl/NativeRemoteFunctionManager.java index d11232e8e809..9c8da488fd59 100644 --- a/java/runtime-native/src/main/java/org/ray/spi/impl/NativeRemoteFunctionManager.java +++ b/java/runtime-native/src/main/java/org/ray/spi/impl/NativeRemoteFunctionManager.java @@ -17,10 +17,15 @@ import org.ray.util.logger.RayLog; /** - * native implementation of remote function manager + * native implementation of remote function manager. */ public class NativeRemoteFunctionManager implements RemoteFunctionManager { + private ConcurrentHashMap loadedApps = new ConcurrentHashMap<>(); + private MessageDigest md; + private String appDir = System.getProperty("user.dir") + "/apps"; + private KeyValueStoreLink kvStore; + public NativeRemoteFunctionManager(KeyValueStoreLink kvStore) throws NoSuchAlgorithmException { this.kvStore = kvStore; md = MessageDigest.getInstance("SHA-1"); @@ -42,36 +47,36 @@ public UniqueID registerResource(byte[] resourceZip) { // instead of cache //if (!Ray.exist(resourceId)) { //Ray.put(resourceId, resourceZip); - kvStore.Set(resourceId.getBytes(), resourceZip, null); + kvStore.set(resourceId.getBytes(), resourceZip, null); //} return resourceId; } @Override public byte[] getResource(UniqueID resourceId) { - return kvStore.Get(resourceId.getBytes(), null); + return kvStore.get(resourceId.getBytes(), null); //return (byte[])Ray.get(resourceId); } @Override public void unregisterResource(UniqueID resourceId) { - kvStore.Delete(resourceId.getBytes(), null); + kvStore.delete(resourceId.getBytes(), null); } @Override public void registerApp(UniqueID driverId, UniqueID resourceId) { //Ray.put(driverId, resourceId); - kvStore.Set("App2ResMap", resourceId.toString(), driverId.toString()); + kvStore.set("App2ResMap", resourceId.toString(), driverId.toString()); } @Override public UniqueID getAppResourceId(UniqueID driverId) { - return new UniqueID(kvStore.Get("App2ResMap", driverId.toString())); + return new UniqueID(kvStore.get("App2ResMap", driverId.toString())); } @Override public void unregisterApp(UniqueID driverId) { - kvStore.Delete("App2ResMap", driverId.toString()); + kvStore.delete("App2ResMap", driverId.toString()); } @Override @@ -88,7 +93,7 @@ private synchronized LoadedFunctions initLoadedApps(UniqueID driverId) { RayLog.core.info("initLoadedApps" + driverId.toString()); LoadedFunctions rf = loadedApps.get(driverId); if (rf == null) { - UniqueID resId = new UniqueID(kvStore.Get("App2ResMap", driverId.toString())); + UniqueID resId = new UniqueID(kvStore.get("App2ResMap", driverId.toString())); //UniqueID resId = Ray.get(driverId); byte[] res = getResource(resId); @@ -127,9 +132,4 @@ public synchronized void unloadFunctions(UniqueID driverId) { RayLog.rapp.error("unload function for " + driverId + " failed, ex = " + e.getMessage(), e); } } - - private ConcurrentHashMap loadedApps = new ConcurrentHashMap<>(); - private MessageDigest md; - private String appDir = System.getProperty("user.dir") + "/apps"; - private KeyValueStoreLink kvStore; } diff --git a/java/runtime-native/src/main/java/org/ray/spi/impl/PlasmaObjectExistsException.java b/java/runtime-native/src/main/java/org/ray/spi/impl/PlasmaObjectExistsException.java index 9f2a38c05e35..35601f35a3b1 100644 --- a/java/runtime-native/src/main/java/org/ray/spi/impl/PlasmaObjectExistsException.java +++ b/java/runtime-native/src/main/java/org/ray/spi/impl/PlasmaObjectExistsException.java @@ -6,9 +6,7 @@ */ public class PlasmaObjectExistsException extends Exception { - /** - * - */ + private static final long serialVersionUID = 9128880292504270291L; public PlasmaObjectExistsException() { @@ -16,7 +14,7 @@ public PlasmaObjectExistsException() { } public PlasmaObjectExistsException(String message, Throwable cause, boolean enableSuppression, - boolean writableStackTrace) { + boolean writableStackTrace) { super(message, cause, enableSuppression, writableStackTrace); } diff --git a/java/runtime-native/src/main/java/org/ray/spi/impl/PlasmaOutOfMemoryException.java b/java/runtime-native/src/main/java/org/ray/spi/impl/PlasmaOutOfMemoryException.java index 18ea8624d138..67499e4fc765 100644 --- a/java/runtime-native/src/main/java/org/ray/spi/impl/PlasmaOutOfMemoryException.java +++ b/java/runtime-native/src/main/java/org/ray/spi/impl/PlasmaOutOfMemoryException.java @@ -6,9 +6,6 @@ */ public class PlasmaOutOfMemoryException extends Exception { - /** - * - */ private static final long serialVersionUID = -2786069077559520659L; public PlasmaOutOfMemoryException() { @@ -16,7 +13,7 @@ public PlasmaOutOfMemoryException() { } public PlasmaOutOfMemoryException(String message, Throwable cause, boolean enableSuppression, - boolean writableStackTrace) { + boolean writableStackTrace) { super(message, cause, enableSuppression, writableStackTrace); } diff --git a/java/runtime-native/src/main/java/org/ray/spi/impl/RedisClient.java b/java/runtime-native/src/main/java/org/ray/spi/impl/RedisClient.java index a49549e9966e..97f744f7dec9 100644 --- a/java/runtime-native/src/main/java/org/ray/spi/impl/RedisClient.java +++ b/java/runtime-native/src/main/java/org/ray/spi/impl/RedisClient.java @@ -18,11 +18,11 @@ public RedisClient() { } public RedisClient(String addr) { - SetAddr(addr); + setAddr(addr); } @Override - public synchronized void SetAddr(String addr) { + public synchronized void setAddr(String addr) { if (StringUtils.isEmpty(redisAddress)) { redisAddress = addr; String[] ipPort = addr.split(":"); @@ -34,14 +34,14 @@ public synchronized void SetAddr(String addr) { } @Override - public void CheckConnected() throws Exception { + public void checkConnected() throws Exception { if (jedisPool == null) { throw new Exception("the GlobalState API can't be used before ray init."); } } @Override - public Long Set(final String key, final String value, final String field) { + public Long set(final String key, final String value, final String field) { try (Jedis jedis = jedisPool.getResource()) { if (field == null) { jedis.set(key, value); @@ -54,7 +54,7 @@ public Long Set(final String key, final String value, final String field) { } @Override - public Long Set(byte[] key, byte[] value, byte[] field) { + public Long set(byte[] key, byte[] value, byte[] field) { try (Jedis jedis = jedisPool.getResource()) { if (field == null) { jedis.set(key, value); @@ -67,7 +67,36 @@ public Long Set(byte[] key, byte[] value, byte[] field) { } @Override - public String Get(final String key, final String field) { + public String hmset(String key, Map hash) { + try (Jedis jedis = jedisPool.getResource()) { + return jedis.hmset(key, hash); + } + + } + + @Override + public String hmset(byte[] key, Map hash) { + try (Jedis jedis = jedisPool.getResource()) { + return jedis.hmset(key, hash); + } + } + + @Override + public List hmget(String key, String... fields) { + try (Jedis jedis = jedisPool.getResource()) { + return jedis.hmget(key, fields); + } + } + + @Override + public List hmget(byte[] key, byte[]... fields) { + try (Jedis jedis = jedisPool.getResource()) { + return jedis.hmget(key, fields); + } + } + + @Override + public String get(final String key, final String field) { try (Jedis jedis = jedisPool.getResource()) { if (field == null) { return jedis.get(key); @@ -79,7 +108,7 @@ public String Get(final String key, final String field) { } @Override - public byte[] Get(byte[] key, byte[] field) { + public byte[] get(byte[] key, byte[] field) { try (Jedis jedis = jedisPool.getResource()) { if (field == null) { return jedis.get(key); @@ -91,7 +120,7 @@ public byte[] Get(byte[] key, byte[] field) { } @Override - public Long Delete(final String key, final String field) { + public Long delete(final String key, final String field) { try (Jedis jedis = jedisPool.getResource()) { if (field == null) { return jedis.del(key); @@ -103,7 +132,7 @@ public Long Delete(final String key, final String field) { } @Override - public Long Delete(byte[] key, byte[] field) { + public Long delete(byte[] key, byte[] field) { try (Jedis jedis = jedisPool.getResource()) { if (field == null) { return jedis.del(key); @@ -115,43 +144,14 @@ public Long Delete(byte[] key, byte[] field) { } @Override - public String Hmset(String key, Map hash) { - try (Jedis jedis = jedisPool.getResource()) { - return jedis.hmset(key, hash); - } - - } - - @Override - public String Hmset(byte[] key, Map hash) { - try (Jedis jedis = jedisPool.getResource()) { - return jedis.hmset(key, hash); - } - } - - @Override - public List Hmget(String key, String... fields) { - try (Jedis jedis = jedisPool.getResource()) { - return jedis.hmget(key, fields); - } - } - - @Override - public List Hmget(byte[] key, byte[]... fields) { - try (Jedis jedis = jedisPool.getResource()) { - return jedis.hmget(key, fields); - } - } - - @Override - public Set Keys(byte[] pattern) { + public Set keys(byte[] pattern) { try (Jedis jedis = jedisPool.getResource()) { return jedis.keys(pattern); } } @Override - public Set Keys(String pattern) { + public Set keys(String pattern) { try (Jedis jedis = jedisPool.getResource()) { return jedis.keys(pattern); } @@ -165,42 +165,42 @@ public Map hgetAll(byte[] key) { } @Override - public List Lrange(String key, long start, long end) { + public List lrange(String key, long start, long end) { try (Jedis jedis = jedisPool.getResource()) { return jedis.lrange(key, start, end); } } @Override - public Long Rpush(String key, String... strings) { + public Long rpush(String key, String... strings) { try (Jedis jedis = jedisPool.getResource()) { return jedis.rpush(key, strings); } } @Override - public Long Rpush(byte[] key, byte[]... strings) { + public Long rpush(byte[] key, byte[]... strings) { try (Jedis jedis = jedisPool.getResource()) { return jedis.rpush(key, strings); } } @Override - public Long Publish(String channel, String message) { + public Long publish(String channel, String message) { try (Jedis jedis = jedisPool.getResource()) { return jedis.publish(channel, message); } } @Override - public Long Publish(byte[] channel, byte[] message) { + public Long publish(byte[] channel, byte[] message) { try (Jedis jedis = jedisPool.getResource()) { return jedis.publish(channel, message); } } @Override - public Object GetImpl() { + public Object getImpl() { return jedisPool; } } diff --git a/java/runtime-native/src/main/java/org/ray/spi/impl/ResourcePair.java b/java/runtime-native/src/main/java/org/ray/spi/impl/ResourcePair.java index b002607a4ba0..d2b2b67b76ef 100644 --- a/java/runtime-native/src/main/java/org/ray/spi/impl/ResourcePair.java +++ b/java/runtime-native/src/main/java/org/ray/spi/impl/ResourcePair.java @@ -2,37 +2,70 @@ // automatically generated by the FlatBuffers compiler, do not modify -import java.nio.*; -import java.lang.*; -import java.util.*; -import com.google.flatbuffers.*; +import com.google.flatbuffers.FlatBufferBuilder; +import com.google.flatbuffers.Table; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; @SuppressWarnings("unused") public final class ResourcePair extends Table { - public static ResourcePair getRootAsResourcePair(ByteBuffer _bb) { return getRootAsResourcePair(_bb, new ResourcePair()); } - public static ResourcePair getRootAsResourcePair(ByteBuffer _bb, ResourcePair obj) { _bb.order(ByteOrder.LITTLE_ENDIAN); return (obj.__assign(_bb.getInt(_bb.position()) + _bb.position(), _bb)); } - public void __init(int _i, ByteBuffer _bb) { bb_pos = _i; bb = _bb; } - public ResourcePair __assign(int _i, ByteBuffer _bb) { __init(_i, _bb); return this; } + public static ResourcePair getRootAsResourcePair(ByteBuffer _bb) { + return getRootAsResourcePair(_bb, new ResourcePair()); + } + + public static ResourcePair getRootAsResourcePair(ByteBuffer _bb, ResourcePair obj) { + _bb.order(ByteOrder.LITTLE_ENDIAN); + return (obj.__assign(_bb.getInt(_bb.position()) + _bb.position(), _bb)); + } + + public ResourcePair __assign(int _i, ByteBuffer _bb) { + __init(_i, _bb); + return this; + } - public String key() { int o = __offset(4); return o != 0 ? __string(o + bb_pos) : null; } - public ByteBuffer keyAsByteBuffer() { return __vector_as_bytebuffer(4, 1); } - public double value() { int o = __offset(6); return o != 0 ? bb.getDouble(o + bb_pos) : 0.0; } + public void __init(int _i, ByteBuffer _bb) { + bb_pos = _i; + bb = _bb; + } public static int createResourcePair(FlatBufferBuilder builder, - int keyOffset, - double value) { + int keyOffset, + double value) { builder.startObject(2); ResourcePair.addValue(builder, value); ResourcePair.addKey(builder, keyOffset); return ResourcePair.endResourcePair(builder); } - public static void startResourcePair(FlatBufferBuilder builder) { builder.startObject(2); } - public static void addKey(FlatBufferBuilder builder, int keyOffset) { builder.addOffset(0, keyOffset, 0); } - public static void addValue(FlatBufferBuilder builder, double value) { builder.addDouble(1, value, 0.0); } + public static void addValue(FlatBufferBuilder builder, double value) { + builder.addDouble(1, value, 0.0); + } + + public static void addKey(FlatBufferBuilder builder, int keyOffset) { + builder.addOffset(0, keyOffset, 0); + } + public static int endResourcePair(FlatBufferBuilder builder) { int o = builder.endObject(); return o; } + + public static void startResourcePair(FlatBufferBuilder builder) { + builder.startObject(2); + } + + public String key() { + int o = __offset(4); + return o != 0 ? __string(o + bb_pos) : null; + } + + public ByteBuffer keyAsByteBuffer() { + return __vector_as_bytebuffer(4, 1); + } + + public double value() { + int o = __offset(6); + return o != 0 ? bb.getDouble(o + bb_pos) : 0.0; + } } diff --git a/java/runtime-native/src/main/java/org/ray/spi/impl/StateStoreProxyImpl.java b/java/runtime-native/src/main/java/org/ray/spi/impl/StateStoreProxyImpl.java index caad0efd9dd9..907d3d0593c1 100644 --- a/java/runtime-native/src/main/java/org/ray/spi/impl/StateStoreProxyImpl.java +++ b/java/runtime-native/src/main/java/org/ray/spi/impl/StateStoreProxyImpl.java @@ -13,7 +13,7 @@ import org.ray.util.logger.RayLog; /** - * A class used to interface with the Ray control state + * A class used to interface with the Ray control state. */ public class StateStoreProxyImpl implements StateStoreProxy { @@ -28,17 +28,13 @@ public void setStore(KeyValueStoreLink rayKvStore) { this.rayKvStore = rayKvStore; } - public void checkConnected() throws Exception { - rayKvStore.CheckConnected(); - } - public synchronized void initializeGlobalState() throws Exception { String es; checkConnected(); - String s = rayKvStore.Get("NumRedisShards", null); + String s = rayKvStore.get("NumRedisShards", null); if (s == null) { throw new Exception("NumRedisShards not found in redis."); } @@ -47,7 +43,7 @@ public synchronized void initializeGlobalState() throws Exception { es = String.format("Expected at least one Redis shard, found %d", numRedisShards); throw new Exception(es); } - List ipAddressPorts = rayKvStore.Lrange("RedisShards", 0, -1); + List ipAddressPorts = rayKvStore.lrange("RedisShards", 0, -1); if (ipAddressPorts.size() != numRedisShards) { es = String.format("Expected %d Redis shard addresses, found %d.", numRedisShards, ipAddressPorts.size()); @@ -61,32 +57,45 @@ public synchronized void initializeGlobalState() throws Exception { } + public void checkConnected() throws Exception { + rayKvStore.checkConnected(); + } + public synchronized Set keys(final String pattern) { Set allKeys = new HashSet<>(); Set tmpKey; - for (KeyValueStoreLink a_shardStoreList : shardStoreList) { - tmpKey = a_shardStoreList.Keys(pattern); + for (KeyValueStoreLink ashardStoreList : shardStoreList) { + tmpKey = ashardStoreList.keys(pattern); allKeys.addAll(tmpKey); } return allKeys; - } - private byte[] CharsetEncode(String str, String Charset) throws UnsupportedEncodingException { - if (str != null) { - return str.getBytes(Charset); - } - return null; } - private String CharsetDecode(byte[] bs, String Charset) throws UnsupportedEncodingException { - return new String(bs, Charset); + public List getAddressInfo(final String nodeIpAddress, int numRetries) { + int count = 0; + while (count < numRetries) { + try { + return getAddressInfoHelper(nodeIpAddress); + } catch (Exception e) { + try { + TimeUnit.MILLISECONDS.sleep(1000); + } catch (InterruptedException ie) { + RayLog.core.error("error at StateStoreProxyImpl getAddressInfo", e); + throw new RuntimeException(e); + } + } + count++; + } + throw new RuntimeException("cannot get address info from state store"); } /* * get address info of one node from primary redis * @param: node ip address, usually local ip address - * @return: a list of SchedulerInfo which contains storeName, managerName, managerPort and schedulerName + * @return: a list of SchedulerInfo which contains storeName, managerName, managerPort and + * schedulerName * @note：Redis data key is "CL:*", redis data value is a hash. * The hash contains the following： * "deleted" : 0/1 @@ -103,7 +112,7 @@ public List getAddressInfoHelper(final String nodeIpAddress) throws } List schedulerInfo = new ArrayList<>(); - Set cks = rayKvStore.Keys("CL:*".getBytes()); + Set cks = rayKvStore.keys("CL:*".getBytes()); byte[] key; List> plasmaManager = new ArrayList<>(); List> localScheduler = new ArrayList<>(); @@ -111,7 +120,7 @@ public List getAddressInfoHelper(final String nodeIpAddress) throws key = ck; Map info = rayKvStore.hgetAll(key); - String deleted = CharsetDecode(info.get("deleted".getBytes()), "US-ASCII"); + String deleted = charsetDecode(info.get("deleted".getBytes()), "US-ASCII"); if (deleted != null) { if (Boolean.getBoolean(deleted)) { continue; @@ -126,9 +135,9 @@ public List getAddressInfoHelper(final String nodeIpAddress) throws throw new Exception("no client_type in any client"); } - if (CharsetDecode(info.get("nodeIpAddress".getBytes()), "US-ASCII") + if (charsetDecode(info.get("nodeIpAddress".getBytes()), "US-ASCII") .equals(nodeIpAddress)) { - String clientType = CharsetDecode(info.get("client_type".getBytes()), "US-ASCII"); + String clientType = charsetDecode(info.get("client_type".getBytes()), "US-ASCII"); if (clientType.equals("plasmaManager")) { plasmaManager.add(info); } else if (clientType.equals("localScheduler")) { @@ -145,30 +154,30 @@ public List getAddressInfoHelper(final String nodeIpAddress) throws for (int i = 0; i < plasmaManager.size(); i++) { AddressInfo si = new AddressInfo(); - si.storeName = CharsetDecode(plasmaManager.get(i).get("store_socket_name".getBytes()), + si.storeName = charsetDecode(plasmaManager.get(i).get("store_socket_name".getBytes()), "US-ASCII"); - si.managerName = CharsetDecode(plasmaManager.get(i).get("manager_socket_name".getBytes()), + si.managerName = charsetDecode(plasmaManager.get(i).get("manager_socket_name".getBytes()), "US-ASCII"); byte[] rpc = plasmaManager.get(i).get("manager_rpc_name".getBytes()); if (rpc != null) { - si.managerRpcAddr = CharsetDecode(rpc, "US-ASCII"); + si.managerRpcAddr = charsetDecode(rpc, "US-ASCII"); } rpc = plasmaManager.get(i).get("store_rpc_name".getBytes()); if (rpc != null) { - si.storeRpcAddr = CharsetDecode(rpc, "US-ASCII"); + si.storeRpcAddr = charsetDecode(rpc, "US-ASCII"); } - String managerAddr = CharsetDecode(plasmaManager.get(i).get("manager_address".getBytes()), + String managerAddr = charsetDecode(plasmaManager.get(i).get("manager_address".getBytes()), "US-ASCII"); si.managerPort = Integer.parseInt(managerAddr.split(":")[1]); - si.schedulerName = CharsetDecode( + si.schedulerName = charsetDecode( localScheduler.get(i).get("local_scheduler_socket_name".getBytes()), "US-ASCII"); rpc = localScheduler.get(i).get("local_scheduler_rpc_name".getBytes()); if (rpc != null) { - si.schedulerRpcAddr = CharsetDecode(rpc, "US-ASCII"); + si.schedulerRpcAddr = charsetDecode(rpc, "US-ASCII"); } schedulerInfo.add(si); @@ -177,21 +186,14 @@ public List getAddressInfoHelper(final String nodeIpAddress) throws return schedulerInfo; } - public List getAddressInfo(final String node_ip_address, int numRetries) { - int count = 0; - while (count < numRetries) { - try { - return getAddressInfoHelper(node_ip_address); - } catch (Exception e) { - try { - TimeUnit.MILLISECONDS.sleep(1000); - } catch (InterruptedException ie) { - RayLog.core.error("error at StateStoreProxyImpl getAddressInfo", e); - throw new RuntimeException(e); - } - } - count++; + private String charsetDecode(byte[] bs, String charset) throws UnsupportedEncodingException { + return new String(bs, charset); + } + + private byte[] charsetEncode(String str, String charset) throws UnsupportedEncodingException { + if (str != null) { + return str.getBytes(charset); } - throw new RuntimeException("cannot get address info from state store"); + return null; } } diff --git a/java/runtime-native/src/main/java/org/ray/spi/impl/TaskInfo.java b/java/runtime-native/src/main/java/org/ray/spi/impl/TaskInfo.java index aacd585a4a27..523a6a986c70 100644 --- a/java/runtime-native/src/main/java/org/ray/spi/impl/TaskInfo.java +++ b/java/runtime-native/src/main/java/org/ray/spi/impl/TaskInfo.java @@ -1,61 +1,47 @@ package org.ray.spi.impl; // automatically generated by the FlatBuffers compiler, do not modify -import java.nio.*; -import java.lang.*; -import java.util.*; -import com.google.flatbuffers.*; +import com.google.flatbuffers.FlatBufferBuilder; +import com.google.flatbuffers.Table; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; @SuppressWarnings("unused") public final class TaskInfo extends Table { - public static TaskInfo getRootAsTaskInfo(ByteBuffer _bb) { return getRootAsTaskInfo(_bb, new TaskInfo()); } - public static TaskInfo getRootAsTaskInfo(ByteBuffer _bb, TaskInfo obj) { _bb.order(ByteOrder.LITTLE_ENDIAN); return (obj.__assign(_bb.getInt(_bb.position()) + _bb.position(), _bb)); } - public void __init(int _i, ByteBuffer _bb) { bb_pos = _i; bb = _bb; } - public TaskInfo __assign(int _i, ByteBuffer _bb) { __init(_i, _bb); return this; } - - public String driverId() { int o = __offset(4); return o != 0 ? __string(o + bb_pos) : null; } - public ByteBuffer driverIdAsByteBuffer() { return __vector_as_bytebuffer(4, 1); } - public String taskId() { int o = __offset(6); return o != 0 ? __string(o + bb_pos) : null; } - public ByteBuffer taskIdAsByteBuffer() { return __vector_as_bytebuffer(6, 1); } - public String parentTaskId() { int o = __offset(8); return o != 0 ? __string(o + bb_pos) : null; } - public ByteBuffer parentTaskIdAsByteBuffer() { return __vector_as_bytebuffer(8, 1); } - public int parentCounter() { int o = __offset(10); return o != 0 ? bb.getInt(o + bb_pos) : 0; } - public String actorCreationId() { int o = __offset(12); return o != 0 ? __string(o + bb_pos) : null; } - public ByteBuffer actorCreationIdAsByteBuffer() { return __vector_as_bytebuffer(12, 1); } - public String actorCreationDummyObjectId() { int o = __offset(14); return o != 0 ? __string(o + bb_pos) : null; } - public ByteBuffer actorCreationDummyObjectIdAsByteBuffer() { return __vector_as_bytebuffer(14, 1); } - public String actorId() { int o = __offset(16); return o != 0 ? __string(o + bb_pos) : null; } - public ByteBuffer actorIdAsByteBuffer() { return __vector_as_bytebuffer(16, 1); } - public String actorHandleId() { int o = __offset(18); return o != 0 ? __string(o + bb_pos) : null; } - public ByteBuffer actorHandleIdAsByteBuffer() { return __vector_as_bytebuffer(18, 1); } - public int actorCounter() { int o = __offset(20); return o != 0 ? bb.getInt(o + bb_pos) : 0; } - public boolean isActorCheckpointMethod() { int o = __offset(22); return o != 0 ? 0!=bb.get(o + bb_pos) : false; } - public String functionId() { int o = __offset(24); return o != 0 ? __string(o + bb_pos) : null; } - public ByteBuffer functionIdAsByteBuffer() { return __vector_as_bytebuffer(24, 1); } - public Arg args(int j) { return args(new Arg(), j); } - public Arg args(Arg obj, int j) { int o = __offset(26); return o != 0 ? obj.__assign(__indirect(__vector(o) + j * 4), bb) : null; } - public int argsLength() { int o = __offset(26); return o != 0 ? __vector_len(o) : 0; } - public String returns(int j) { int o = __offset(28); return o != 0 ? __string(__vector(o) + j * 4) : null; } - public int returnsLength() { int o = __offset(28); return o != 0 ? __vector_len(o) : 0; } - public ResourcePair requiredResources(int j) { return requiredResources(new ResourcePair(), j); } - public ResourcePair requiredResources(ResourcePair obj, int j) { int o = __offset(30); return o != 0 ? obj.__assign(__indirect(__vector(o) + j * 4), bb) : null; } - public int requiredResourcesLength() { int o = __offset(30); return o != 0 ? __vector_len(o) : 0; } + public static TaskInfo getRootAsTaskInfo(ByteBuffer _bb) { + return getRootAsTaskInfo(_bb, new TaskInfo()); + } + + public static TaskInfo getRootAsTaskInfo(ByteBuffer _bb, TaskInfo obj) { + _bb.order(ByteOrder.LITTLE_ENDIAN); + return (obj.__assign(_bb.getInt(_bb.position()) + _bb.position(), _bb)); + } + + public TaskInfo __assign(int _i, ByteBuffer _bb) { + __init(_i, _bb); + return this; + } + + public void __init(int _i, ByteBuffer _bb) { + bb_pos = _i; + bb = _bb; + } public static int createTaskInfo(FlatBufferBuilder builder, - int driver_idOffset, - int task_idOffset, - int parent_task_idOffset, - int parent_counter, - int actor_creation_idOffset, - int actor_creation_dummy_object_idOffset, - int actor_idOffset, - int actor_handle_idOffset, - int actor_counter, - boolean is_actor_checkpoint_method, - int function_idOffset, - int argsOffset, - int returnsOffset, - int required_resourcesOffset) { + int driver_idOffset, + int task_idOffset, + int parent_task_idOffset, + int parent_counter, + int actor_creation_idOffset, + int actor_creation_dummy_object_idOffset, + int actor_idOffset, + int actor_handle_idOffset, + int actor_counter, + boolean is_actor_checkpoint_method, + int function_idOffset, + int argsOffset, + int returnsOffset, + int required_resourcesOffset) { builder.startObject(14); TaskInfo.addRequiredResources(builder, required_resourcesOffset); TaskInfo.addReturns(builder, returnsOffset); @@ -74,33 +60,236 @@ public static int createTaskInfo(FlatBufferBuilder builder, return TaskInfo.endTaskInfo(builder); } - public static void startTaskInfo(FlatBufferBuilder builder) { builder.startObject(14); } - public static void addDriverId(FlatBufferBuilder builder, int driverIdOffset) { builder.addOffset(0, driverIdOffset, 0); } - public static void addTaskId(FlatBufferBuilder builder, int taskIdOffset) { builder.addOffset(1, taskIdOffset, 0); } - public static void addParentTaskId(FlatBufferBuilder builder, int parentTaskIdOffset) { builder.addOffset(2, parentTaskIdOffset, 0); } - public static void addParentCounter(FlatBufferBuilder builder, int parentCounter) { builder.addInt(3, parentCounter, 0); } - public static void addActorCreationId(FlatBufferBuilder builder, int actorCreationIdOffset) { builder.addOffset(4, actorCreationIdOffset, 0); } - public static void addActorCreationDummyObjectId(FlatBufferBuilder builder, int actorCreationDummyObjectIdOffset) { builder.addOffset(5, actorCreationDummyObjectIdOffset, 0); } - public static void addActorId(FlatBufferBuilder builder, int actorIdOffset) { builder.addOffset(6, actorIdOffset, 0); } - public static void addActorHandleId(FlatBufferBuilder builder, int actorHandleIdOffset) { builder.addOffset(7, actorHandleIdOffset, 0); } - public static void addActorCounter(FlatBufferBuilder builder, int actorCounter) { builder.addInt(8, actorCounter, 0); } - public static void addIsActorCheckpointMethod(FlatBufferBuilder builder, boolean isActorCheckpointMethod) { builder.addBoolean(9, isActorCheckpointMethod, false); } - public static void addFunctionId(FlatBufferBuilder builder, int functionIdOffset) { builder.addOffset(10, functionIdOffset, 0); } - public static void addArgs(FlatBufferBuilder builder, int argsOffset) { builder.addOffset(11, argsOffset, 0); } - public static int createArgsVector(FlatBufferBuilder builder, int[] data) { builder.startVector(4, data.length, 4); for (int i = data.length - 1; i >= 0; i--) builder.addOffset(data[i]); return builder.endVector(); } - public static void startArgsVector(FlatBufferBuilder builder, int numElems) { builder.startVector(4, numElems, 4); } - public static void addReturns(FlatBufferBuilder builder, int returnsOffset) { builder.addOffset(12, returnsOffset, 0); } - public static int createReturnsVector(FlatBufferBuilder builder, int[] data) { builder.startVector(4, data.length, 4); for (int i = data.length - 1; i >= 0; i--) builder.addOffset(data[i]); return builder.endVector(); } - public static void startReturnsVector(FlatBufferBuilder builder, int numElems) { builder.startVector(4, numElems, 4); } - public static void addRequiredResources(FlatBufferBuilder builder, int requiredResourcesOffset) { builder.addOffset(13, requiredResourcesOffset, 0); } - public static int createRequiredResourcesVector(FlatBufferBuilder builder, int[] data) { builder.startVector(4, data.length, 4); for (int i = data.length - 1; i >= 0; i--) builder.addOffset(data[i]); return builder.endVector(); } - public static void startRequiredResourcesVector(FlatBufferBuilder builder, int numElems) { builder.startVector(4, numElems, 4); } + public static void addRequiredResources(FlatBufferBuilder builder, int requiredResourcesOffset) { + builder.addOffset(13, requiredResourcesOffset, 0); + } + + public static void addReturns(FlatBufferBuilder builder, int returnsOffset) { + builder.addOffset(12, returnsOffset, 0); + } + + public static void addArgs(FlatBufferBuilder builder, int argsOffset) { + builder.addOffset(11, argsOffset, 0); + } + + public static void addFunctionId(FlatBufferBuilder builder, int functionIdOffset) { + builder.addOffset(10, functionIdOffset, 0); + } + + public static void addActorCounter(FlatBufferBuilder builder, int actorCounter) { + builder.addInt(8, actorCounter, 0); + } + + public static void addActorHandleId(FlatBufferBuilder builder, int actorHandleIdOffset) { + builder.addOffset(7, actorHandleIdOffset, 0); + } + + public static void addActorId(FlatBufferBuilder builder, int actorIdOffset) { + builder.addOffset(6, actorIdOffset, 0); + } + + public static void addActorCreationDummyObjectId(FlatBufferBuilder builder, int + actorCreationDummyObjectIdOffset) { + builder.addOffset(5, actorCreationDummyObjectIdOffset, 0); + } + + public static void addActorCreationId(FlatBufferBuilder builder, int actorCreationIdOffset) { + builder.addOffset(4, actorCreationIdOffset, 0); + } + + public static void addParentCounter(FlatBufferBuilder builder, int parentCounter) { + builder.addInt(3, parentCounter, 0); + } + + public static void addParentTaskId(FlatBufferBuilder builder, int parentTaskIdOffset) { + builder.addOffset(2, parentTaskIdOffset, 0); + } + + public static void addTaskId(FlatBufferBuilder builder, int taskIdOffset) { + builder.addOffset(1, taskIdOffset, 0); + } + + public static void addDriverId(FlatBufferBuilder builder, int driverIdOffset) { + builder.addOffset(0, driverIdOffset, 0); + } + + public static void addIsActorCheckpointMethod(FlatBufferBuilder builder, boolean + isActorCheckpointMethod) { + builder.addBoolean(9, isActorCheckpointMethod, false); + } + public static int endTaskInfo(FlatBufferBuilder builder) { int o = builder.endObject(); return o; } - //this is manually added to avoid encoding/decoding cost as our object id is a byte array instead of a string + public static void startTaskInfo(FlatBufferBuilder builder) { + builder.startObject(14); + } + + public static int createArgsVector(FlatBufferBuilder builder, int[] data) { + builder.startVector(4, data.length, 4); + for (int i = data.length - 1; i >= 0; i--) { + builder.addOffset(data[i]); + } + return builder.endVector(); + } + + public static void startArgsVector(FlatBufferBuilder builder, int numElems) { + builder.startVector(4, numElems, 4); + } + + public static int createReturnsVector(FlatBufferBuilder builder, int[] data) { + builder.startVector(4, data.length, 4); + for (int i = data.length - 1; i >= 0; i--) { + builder.addOffset(data[i]); + } + return builder.endVector(); + } + + public static void startReturnsVector(FlatBufferBuilder builder, int numElems) { + builder.startVector(4, numElems, 4); + } + + public static int createRequiredResourcesVector(FlatBufferBuilder builder, int[] data) { + builder.startVector(4, data.length, 4); + for (int i = data.length - 1; i >= 0; i--) { + builder.addOffset(data[i]); + } + return builder.endVector(); + } + + public static void startRequiredResourcesVector(FlatBufferBuilder builder, int numElems) { + builder.startVector(4, numElems, 4); + } + + public String driverId() { + int o = __offset(4); + return o != 0 ? __string(o + bb_pos) : null; + } + + public ByteBuffer driverIdAsByteBuffer() { + return __vector_as_bytebuffer(4, 1); + } + + public String taskId() { + int o = __offset(6); + return o != 0 ? __string(o + bb_pos) : null; + } + + public ByteBuffer taskIdAsByteBuffer() { + return __vector_as_bytebuffer(6, 1); + } + + public String parentTaskId() { + int o = __offset(8); + return o != 0 ? __string(o + bb_pos) : null; + } + + public ByteBuffer parentTaskIdAsByteBuffer() { + return __vector_as_bytebuffer(8, 1); + } + + public int parentCounter() { + int o = __offset(10); + return o != 0 ? bb.getInt(o + bb_pos) : 0; + } + + public String actorCreationId() { + int o = __offset(12); + return o != 0 ? __string(o + bb_pos) : null; + } + + public ByteBuffer actorCreationIdAsByteBuffer() { + return __vector_as_bytebuffer(12, 1); + } + + public String actorCreationDummyObjectId() { + int o = __offset(14); + return o != 0 ? __string(o + bb_pos) : null; + } + + public ByteBuffer actorCreationDummyObjectIdAsByteBuffer() { + return __vector_as_bytebuffer(14, 1); + } + + public String actorId() { + int o = __offset(16); + return o != 0 ? __string(o + bb_pos) : null; + } + + public ByteBuffer actorIdAsByteBuffer() { + return __vector_as_bytebuffer(16, 1); + } + + public String actorHandleId() { + int o = __offset(18); + return o != 0 ? __string(o + bb_pos) : null; + } + + public ByteBuffer actorHandleIdAsByteBuffer() { + return __vector_as_bytebuffer(18, 1); + } + + public int actorCounter() { + int o = __offset(20); + return o != 0 ? bb.getInt(o + bb_pos) : 0; + } + + public boolean isActorCheckpointMethod() { + int o = __offset(22); + return o != 0 ? 0 != bb.get(o + bb_pos) : false; + } + + public String functionId() { + int o = __offset(24); + return o != 0 ? __string(o + bb_pos) : null; + } + + public ByteBuffer functionIdAsByteBuffer() { + return __vector_as_bytebuffer(24, 1); + } + + public Arg args(int j) { + return args(new Arg(), j); + } + + public Arg args(Arg obj, int j) { + int o = __offset(26); + return o != 0 ? obj.__assign(__indirect(__vector(o) + j * 4), bb) : null; + } + + public int argsLength() { + int o = __offset(26); + return o != 0 ? __vector_len(o) : 0; + } + + public String returns(int j) { + int o = __offset(28); + return o != 0 ? __string(__vector(o) + j * 4) : null; + } + + public int returnsLength() { + int o = __offset(28); + return o != 0 ? __vector_len(o) : 0; + } + + public ResourcePair requiredResources(int j) { + return requiredResources(new ResourcePair(), j); + } + + public ResourcePair requiredResources(ResourcePair obj, int j) { + int o = __offset(30); + return o != 0 ? obj.__assign(__indirect(__vector(o) + j * 4), bb) : null; + } + + public int requiredResourcesLength() { + int o = __offset(30); + return o != 0 ? __vector_len(o) : 0; + } + + //this is manually added to avoid encoding/decoding cost as our object + //id is a byte array instead of a string public ByteBuffer returnsAsByteBuffer(int j) { int o = __offset(28); if (o == 0) { diff --git a/java/test.sh b/java/test.sh index 54674d251ef4..d277473c2b1c 100755 --- a/java/test.sh +++ b/java/test.sh @@ -7,5 +7,10 @@ mvn clean install -pl plasma -am -Dmaven.test.skip popd pushd $ROOT_DIR/../java mvn clean install -Dmaven.test.skip -mvn test +check_style=$(mvn checkstyle:check) +echo "${check_style}" +[[ ${check_style} =~ "BUILD FAILURE" ]] && exit 1 +mvn_test=$(mvn test) +echo "${mvn_test}" +[[ ${mvn_test} =~ "BUILD SUCCESS" ]] || exit 1 popd \ No newline at end of file diff --git a/java/test/pom.xml b/java/test/pom.xml index cf4bb94e1a85..8197c0fe853f 100644 --- a/java/test/pom.xml +++ b/java/test/pom.xml @@ -1,128 +1,128 @@ - - - org.ray.parent - ray-superpom - 1.0 - - 4.0.0 - - org.ray - ray-test - - java test cases for ray - java test cases for ray - - - jar - - - - org.ray - ray-api - 1.0 - - - - org.ray - ray-runtime-common - 1.0 - - - - org.ray - ray-runtime-native - 1.0 - - - - org.ray - ray-runtime-dev - 1.0 - - - - org.ray - ray-common - 1.0 - - - - org.ray - ray-hook - 1.0 - - - - junit - junit - 4.11 - - - - - - commons-collections - commons-collections - 3.2.2 - - - - com.google.guava - guava - 19.0 - - - - - - org.apache.maven.plugins - maven-surefire-plugin - 2.21.0 - - - - ${basedir}/../ray.config.ini - - -ea - -Djava.library.path=${basedir}/../../build/src/plasma:${basedir}/../../build/src/local_scheduler - -noverify - -DlogOutput=console - - ${basedir}/src/main/java/ - ${project.build.directory}/classes/ - - - - - org.apache.maven.plugins - maven-dependency-plugin - - - copy-dependencies - package - - copy-dependencies - - - ${basedir}/lib - false - false - true - - - - - - - + + + org.ray.parent + ray-superpom + 1.0 + + 4.0.0 + + org.ray + ray-test + + java test cases for ray + java test cases for ray + + + jar + + + + org.ray + ray-api + 1.0 + + + + org.ray + ray-runtime-common + 1.0 + + + + org.ray + ray-runtime-native + 1.0 + + + + org.ray + ray-runtime-dev + 1.0 + + + + org.ray + ray-common + 1.0 + + + + org.ray + ray-hook + 1.0 + + + + junit + junit + 4.11 + + + + + + commons-collections + commons-collections + 3.2.2 + + + + com.google.guava + guava + 19.0 + + + + + + org.apache.maven.plugins + maven-surefire-plugin + 2.21.0 + + + + ${basedir}/../ray.config.ini + + -ea + -Djava.library.path=${basedir}/../../build/src/plasma:${basedir}/../../build/src/local_scheduler + -noverify + -DlogOutput=console + + ${basedir}/src/main/java/ + ${project.build.directory}/classes/ + + + + + org.apache.maven.plugins + maven-dependency-plugin + + + copy-dependencies + package + + copy-dependencies + + + ${basedir}/lib + false + false + true + + + + + + + diff --git a/java/test/src/main/java/org/ray/api/benchmark/ActorPressTest.java b/java/test/src/main/java/org/ray/api/benchmark/ActorPressTest.java index c622698e1c58..d1148b1f29ea 100644 --- a/java/test/src/main/java/org/ray/api/benchmark/ActorPressTest.java +++ b/java/test/src/main/java/org/ray/api/benchmark/ActorPressTest.java @@ -46,27 +46,27 @@ public void rateLimiterTest() { super.rateLimiterPressureTest(pressureTestParameter); } + @Override + public RayObject> rayCall(RayActor rayActor) { + return Ray.call(Adder::add, (RayActor) rayActor, 10); + } + + @Override + public boolean checkResult(Object o) { + return true; + } + @RayRemote public static class Adder { + private Integer sum = 0; + public RemoteResult add(Integer n) { RemoteResult remoteResult = new RemoteResult<>(); remoteResult.setResult(sum += n); remoteResult.setFinishTime(System.nanoTime()); return remoteResult; } - - private Integer sum = 0; - } - - @Override - public RayObject> rayCall(RayActor rayActor) { - return Ray.call(Adder::add, (RayActor) rayActor, 10); - } - - @Override - public boolean checkResult(Object o) { - return true; } } diff --git a/java/test/src/main/java/org/ray/api/benchmark/MaxPressureTest.java b/java/test/src/main/java/org/ray/api/benchmark/MaxPressureTest.java index b98ed54638f7..ed8e5a1160b9 100644 --- a/java/test/src/main/java/org/ray/api/benchmark/MaxPressureTest.java +++ b/java/test/src/main/java/org/ray/api/benchmark/MaxPressureTest.java @@ -11,20 +11,9 @@ @RunWith(MyRunner.class) public class MaxPressureTest extends RayBenchmarkTest { - private static final long serialVersionUID = -1684518885171395952L; - public static final int clientNum = 2; - public static final int totalNum = 10; - - @Test - public void Test() { - PressureTestParameter pressureTestParameter = new PressureTestParameter(); - pressureTestParameter.setClientNum(clientNum); - pressureTestParameter.setTotalNum(totalNum); - pressureTestParameter.setRayBenchmarkTest(this); - super.maxPressureTest(pressureTestParameter); - } + private static final long serialVersionUID = -1684518885171395952L; @RayRemote public static RemoteResult currentTime() { @@ -34,9 +23,13 @@ public static RemoteResult currentTime() { return remoteResult; } - @Override - public boolean checkResult(Object o) { - return (int) o == 0; + @Test + public void test() { + PressureTestParameter pressureTestParameter = new PressureTestParameter(); + pressureTestParameter.setClientNum(clientNum); + pressureTestParameter.setTotalNum(totalNum); + pressureTestParameter.setRayBenchmarkTest(this); + super.maxPressureTest(pressureTestParameter); } @Override @@ -45,4 +38,9 @@ public RayObject> rayCall(RayActor rayActor) { return Ray.call(MaxPressureTest::currentTime); } + @Override + public boolean checkResult(Object o) { + return (int) o == 0; + } + } diff --git a/java/test/src/main/java/org/ray/api/benchmark/RateLimiterPressureTest.java b/java/test/src/main/java/org/ray/api/benchmark/RateLimiterPressureTest.java index 6dbb1bac77a3..94e55b33c455 100644 --- a/java/test/src/main/java/org/ray/api/benchmark/RateLimiterPressureTest.java +++ b/java/test/src/main/java/org/ray/api/benchmark/RateLimiterPressureTest.java @@ -11,23 +11,10 @@ @RunWith(MyRunner.class) public class RateLimiterPressureTest extends RayBenchmarkTest { - private static final long serialVersionUID = 6616958120966144235L; - public static final int clientNum = 2; - public static final int totalQps = 2; - public static final int duration = 10; - - @Test - public void Test() { - PressureTestParameter pressureTestParameter = new PressureTestParameter(); - pressureTestParameter.setClientNum(clientNum); - pressureTestParameter.setTotalQps(totalQps); - pressureTestParameter.setDuration(duration); - pressureTestParameter.setRayBenchmarkTest(this); - super.rateLimiterPressureTest(pressureTestParameter); - } + private static final long serialVersionUID = 6616958120966144235L; @RayRemote public static RemoteResult currentTime() { @@ -37,9 +24,14 @@ public static RemoteResult currentTime() { return remoteResult; } - @Override - public boolean checkResult(Object o) { - return (int) o == 0; + @Test + public void test() { + PressureTestParameter pressureTestParameter = new PressureTestParameter(); + pressureTestParameter.setClientNum(clientNum); + pressureTestParameter.setTotalQps(totalQps); + pressureTestParameter.setDuration(duration); + pressureTestParameter.setRayBenchmarkTest(this); + super.rateLimiterPressureTest(pressureTestParameter); } @Override @@ -47,4 +39,9 @@ public RayObject> rayCall(RayActor rayActor) { return Ray.call(RateLimiterPressureTest::currentTime); } + + @Override + public boolean checkResult(Object o) { + return (int) o == 0; + } } diff --git a/java/test/src/main/java/org/ray/api/benchmark/RayBenchmarkTest.java b/java/test/src/main/java/org/ray/api/benchmark/RayBenchmarkTest.java index c45e2786cb9c..7f3775a085b5 100644 --- a/java/test/src/main/java/org/ray/api/benchmark/RayBenchmarkTest.java +++ b/java/test/src/main/java/org/ray/api/benchmark/RayBenchmarkTest.java @@ -15,78 +15,9 @@ public abstract class RayBenchmarkTest implements Serializable { - private static final long serialVersionUID = 416045641835782523L; - //not thread safe ,but we only have one thread here public static final DecimalFormat df = new DecimalFormat("00.00"); - - private static void printList(List list) { - int len = list.size(); - int middle = len / 2; - int almostHundred = (int) (len * 0.9999); - int ninetyNine = (int) (len * 0.99); - int ninetyFive = (int) (len * 0.95); - int ninety = (int) (len * 0.9); - int fifty = (int) (len * 0.5); - - RayLog.core.error("Final result of rt as below:"); - RayLog.core.error("max: " + list.get(len - 1) + "μs"); - RayLog.core.error("min: " + list.get(0) + "μs"); - RayLog.core.error("median: " + list.get(middle) + "μs"); - RayLog.core.error("99.99% data smaller than: " + list.get(almostHundred) + "μs"); - RayLog.core.error("99% data smaller than: " + list.get(ninetyNine) + "μs"); - RayLog.core.error("95% data smaller than: " + list.get(ninetyFive) + "μs"); - RayLog.core.error("90% data smaller than: " + list.get(ninety) + "μs"); - RayLog.core.error("50% data smaller than: " + list.get(fifty) + "μs"); - } - - public void singleLatencyTest(int times, RayActor rayActor) { - - List counterList = new ArrayList<>(); - for (int i = 0; i < times; i++) { - long startTime = System.nanoTime(); - RayObject> rayObject = rayCall(rayActor); - RemoteResult remoteResult = rayObject.get(); - T t = remoteResult.getResult(); - long endTime = System.nanoTime(); - long costTime = endTime - startTime; - counterList.add(costTime / 1000); - RayLog.core.warn("SINGLE_LATENCY_cost_time: " + costTime + " us"); - Assert.assertTrue(checkResult(t)); - } - Collections.sort(counterList); - printList(counterList); - } - - public void rateLimiterPressureTest(PressureTestParameter pressureTestParameter) { - - pressureTestParameter.setPressureTestType(PressureTestType.RATE_LIMITER); - notSinglePressTest(pressureTestParameter); - } - - public void maxPressureTest(PressureTestParameter pressureTestParameter) { - - pressureTestParameter.setPressureTestType(PressureTestType.MAX); - notSinglePressTest(pressureTestParameter); - } - - private void notSinglePressTest(PressureTestParameter pressureTestParameter) { - - List counterList = new ArrayList<>(); - int clientNum = pressureTestParameter.getClientNum(); - RayObject>[] rayObjects = new RayObject[clientNum]; - - for (int i = 0; i < clientNum; i++) { - rayObjects[i] = Ray.call(RayBenchmarkTest::singleClient, pressureTestParameter); - } - for (int i = 0; i < clientNum; i++) { - List subCounterList = rayObjects[i].get(); - Assert.assertNotNull(subCounterList); - counterList.addAll(subCounterList); - } - Collections.sort(counterList); - printList(counterList); - } + private static final long serialVersionUID = 416045641835782523L; @RayRemote private static List singleClient(PressureTestParameter pressureTestParameter) { @@ -117,10 +48,6 @@ private static List singleClient(PressureTestParameter pressureTestParamet if (rateLimiter != null) { rateLimiter.acquire(); } -// Date currentTime = new Date(); -// String dateString = formatter.format(currentTime); -// RayLog.core.info(logPrefix + "_startTime: " + dateString); - RemoteResultWrapper temp = new RemoteResultWrapper(); temp.setStartTime(System.nanoTime()); temp.setRayObject(rayBenchmarkTest.rayCall(pressureTestParameter.getRayActor())); @@ -145,8 +72,76 @@ private static List singleClient(PressureTestParameter pressureTestParamet } } - abstract public RayObject> rayCall(RayActor rayActor); + public void singleLatencyTest(int times, RayActor rayActor) { - abstract public boolean checkResult(T t); + List counterList = new ArrayList<>(); + for (int i = 0; i < times; i++) { + long startTime = System.nanoTime(); + RayObject> rayObject = rayCall(rayActor); + RemoteResult remoteResult = rayObject.get(); + T t = remoteResult.getResult(); + long endTime = System.nanoTime(); + long costTime = endTime - startTime; + counterList.add(costTime / 1000); + RayLog.core.warn("SINGLE_LATENCY_cost_time: " + costTime + " us"); + Assert.assertTrue(checkResult(t)); + } + Collections.sort(counterList); + printList(counterList); + } + + public abstract RayObject> rayCall(RayActor rayActor); + + public abstract boolean checkResult(T t); + + private void printList(List list) { + int len = list.size(); + int middle = len / 2; + int almostHundred = (int) (len * 0.9999); + int ninetyNine = (int) (len * 0.99); + int ninetyFive = (int) (len * 0.95); + int ninety = (int) (len * 0.9); + int fifty = (int) (len * 0.5); + + RayLog.core.error("Final result of rt as below:"); + RayLog.core.error("max: " + list.get(len - 1) + "μs"); + RayLog.core.error("min: " + list.get(0) + "μs"); + RayLog.core.error("median: " + list.get(middle) + "μs"); + RayLog.core.error("99.99% data smaller than: " + list.get(almostHundred) + "μs"); + RayLog.core.error("99% data smaller than: " + list.get(ninetyNine) + "μs"); + RayLog.core.error("95% data smaller than: " + list.get(ninetyFive) + "μs"); + RayLog.core.error("90% data smaller than: " + list.get(ninety) + "μs"); + RayLog.core.error("50% data smaller than: " + list.get(fifty) + "μs"); + } + + public void rateLimiterPressureTest(PressureTestParameter pressureTestParameter) { + + pressureTestParameter.setPressureTestType(PressureTestType.RATE_LIMITER); + notSinglePressTest(pressureTestParameter); + } + + private void notSinglePressTest(PressureTestParameter pressureTestParameter) { + + List counterList = new ArrayList<>(); + int clientNum = pressureTestParameter.getClientNum(); + RayObject>[] rayObjects = new RayObject[clientNum]; + + for (int i = 0; i < clientNum; i++) { + rayObjects[i] = Ray.call(RayBenchmarkTest::singleClient, pressureTestParameter); + } + for (int i = 0; i < clientNum; i++) { + List subCounterList = rayObjects[i].get(); + Assert.assertNotNull(subCounterList); + counterList.addAll(subCounterList); + } + Collections.sort(counterList); + printList(counterList); + } + + public void maxPressureTest(PressureTestParameter pressureTestParameter) { + + pressureTestParameter.setPressureTestType(PressureTestType.MAX); + notSinglePressTest(pressureTestParameter); + } } diff --git a/java/test/src/main/java/org/ray/api/benchmark/SingleLatencyTest.java b/java/test/src/main/java/org/ray/api/benchmark/SingleLatencyTest.java index 540d1877fa2d..7eee020d7b25 100644 --- a/java/test/src/main/java/org/ray/api/benchmark/SingleLatencyTest.java +++ b/java/test/src/main/java/org/ray/api/benchmark/SingleLatencyTest.java @@ -11,14 +11,8 @@ @RunWith(MyRunner.class) public class SingleLatencyTest extends RayBenchmarkTest { - private static final long serialVersionUID = 3559601273941694468L; - public static final int totalNum = 10; - - @Test - public void Test() { - super.singleLatencyTest(totalNum, null); - } + private static final long serialVersionUID = 3559601273941694468L; @RayRemote public static RemoteResult doFunc() { @@ -27,6 +21,11 @@ public static RemoteResult doFunc() { return remoteResult; } + @Test + public void test() { + super.singleLatencyTest(totalNum, null); + } + @Override public RayObject> rayCall(RayActor rayActor) { return Ray.call(SingleLatencyTest::doFunc); diff --git a/java/test/src/main/java/org/ray/api/experiment/mr/MemoryMapReduce.java b/java/test/src/main/java/org/ray/api/experiment/mr/MemoryMapReduce.java index 59d7f2985f7c..7e1b7f83059e 100644 --- a/java/test/src/main/java/org/ray/api/experiment/mr/MemoryMapReduce.java +++ b/java/test/src/main/java/org/ray/api/experiment/mr/MemoryMapReduce.java @@ -13,15 +13,15 @@ import org.ray.api.RayRemote; /** - * mimic the MapReduce interface atop of Ray API (in memory version) + * mimic the MapReduce interface atop of Ray API (in memory version). */ -public class MemoryMapReduce { +public class MemoryMapReduce { - public List> Map(TInput input) throws Exception { + public List> map(TInputT input) throws Exception { throw new Exception("not implemented"); } - public TReduceValue Reduce(TMapKey k, List values) throws Exception { + public TReduceValueT reduce(TMapKeyT k, List values) throws Exception { throw new Exception("not implemented"); } @@ -31,10 +31,10 @@ public TReduceValue Reduce(TMapKey k, List values) throws Exception { // @param inputs - given input file segments each containing List // @return output file segments each containing SortedMap // - public SortedMap Run(List> inputs, int mapperCount, - Integer reducerCount) { + public SortedMap run(List> inputs, int mapperCount, + Integer reducerCount) { // start all mappers - ArrayList>>> mappers = new ArrayList<>(); + ArrayList>>> mappers = new ArrayList<>(); int inputCountPerMap = inputs.size() / mapperCount; int index = 0; @@ -43,12 +43,12 @@ public SortedMap Run(List> inputs, int mappe break; } - List> perMapInputs = new ArrayList<>(); + List> perMapInputs = new ArrayList<>(); for (int j = 0; j < inputCountPerMap && index < inputs.size(); j++) { perMapInputs.add(inputs.get(index++)); } - mappers.add(Ray.call_n(MemoryMapReduce::InternalMap, reducerCount, reducerCount, perMapInputs, + mappers.add(Ray.call_n(MemoryMapReduce::internalMap, reducerCount, reducerCount, perMapInputs, this.getClass().getName())); } @@ -57,18 +57,18 @@ public SortedMap Run(List> inputs, int mappe // Ray.wait(mappers.toArray(new RayObject[0]), mappers.size(), 0); // start all reducers - ArrayList>> reducers = new ArrayList<>(); + ArrayList>> reducers = new ArrayList<>(); for (int i = 0; i < reducerCount; i++) { // collect states from mappers for this reducer - RayList>> fromMappers = new RayList(); + RayList>> fromMappers = new RayList(); for (int j = 0; j < mapperCount; j++) { assert (mappers.get(j).size() == reducerCount); fromMappers.add(mappers.get(j).Get(i)); } // start this reducer with given input - reducers.add(Ray.call(MemoryMapReduce::InternalReduce, - (List>>) fromMappers, this.getClass().getName())); + reducers.add(Ray.call(MemoryMapReduce::internalReduce, + (List>>) fromMappers, this.getClass().getName())); } // BSP barrier for all reducers to be finished @@ -76,8 +76,8 @@ public SortedMap Run(List> inputs, int mappe // Ray.wait(reducers.toArray(new RayObject[0]), reducers.size(), 0); // collect outputs - TreeMap outputs = new TreeMap<>(); - for (RayObject> r : reducers) { + TreeMap outputs = new TreeMap<>(); + for (RayObject> r : reducers) { r.get().forEach(outputs::put); } return outputs; @@ -91,13 +91,14 @@ public SortedMap Run(List> inputs, int mappe // for each output file, it contains SortedMap> // @RayRemote - public static List>> InternalMap( + public static List>> internalMap( Integer reducerCount, - List> inputs, + List> inputs, String mrClassName) { - MemoryMapReduce mr; + MemoryMapReduce mr; try { - mr = (MemoryMapReduce) Class + mr = (MemoryMapReduce) Class .forName(mrClassName).getConstructors()[0].newInstance(); } catch (InstantiationException | IllegalAccessException | IllegalArgumentException | InvocationTargetException | SecurityException | ClassNotFoundException e1) { @@ -106,16 +107,16 @@ public static List>> out = new ArrayList<>(); + ArrayList>> out = new ArrayList<>(); for (int i = 0; i < reducerCount; i++) { out.add(new TreeMap<>()); } - for (List inputSeg : inputs) { - for (TInput input : inputSeg) { + for (List inputSeg : inputs) { + for (TInputT input : inputSeg) { try { - List> result = mr.Map(input); - for (Pair pr : result) { + List> result = mr.map(input); + for (Pair pr : result) { int reducerIndex = Math.abs(pr.getKey().hashCode()) % reducerCount; out.get(reducerIndex).computeIfAbsent(pr.getKey(), k -> new ArrayList<>()); out.get(reducerIndex).get(pr.getKey()).add(pr.getValue()); @@ -138,12 +139,13 @@ public static List // @RayRemote - public static SortedMap InternalReduce( - List>> inputs, + public static SortedMap + internalReduce( + List>> inputs, String mrClassName) { - MemoryMapReduce mr; + MemoryMapReduce mr; try { - mr = (MemoryMapReduce) Class + mr = (MemoryMapReduce) Class .forName(mrClassName).getConstructors()[0].newInstance(); } catch (InstantiationException | IllegalAccessException | IllegalArgumentException | InvocationTargetException | SecurityException | ClassNotFoundException e1) { @@ -153,9 +155,9 @@ public static SortedMap> minputs = new TreeMap<>(); - for (SortedMap> input : inputs) { - for (Map.Entry> entry : input.entrySet()) { + TreeMap> minputs = new TreeMap<>(); + for (SortedMap> input : inputs) { + for (Map.Entry> entry : input.entrySet()) { if (!minputs.containsKey(entry.getKey())) { minputs.put(entry.getKey(), new ArrayList<>()); } @@ -164,10 +166,10 @@ public static SortedMap out = new TreeMap<>(); - for (Map.Entry> entry : minputs.entrySet()) { + TreeMap out = new TreeMap<>(); + for (Map.Entry> entry : minputs.entrySet()) { try { - out.put(entry.getKey(), mr.Reduce(entry.getKey(), entry.getValue())); + out.put(entry.getKey(), mr.reduce(entry.getKey(), entry.getValue())); } catch (Exception e) { // TODO Auto-generated catch block e.printStackTrace(); diff --git a/java/test/src/main/java/org/ray/api/test/ActorTest.java b/java/test/src/main/java/org/ray/api/test/ActorTest.java index f46267a5c725..0fe26d369aed 100644 --- a/java/test/src/main/java/org/ray/api/test/ActorTest.java +++ b/java/test/src/main/java/org/ray/api/test/ActorTest.java @@ -14,8 +14,14 @@ @RunWith(MyRunner.class) public class ActorTest { + @RayRemote + public static Integer sayWorld(Integer n, RayActor adder) { + RayObject result = Ray.call(ActorTest.Adder::add, adder, 1); + return result.get() + n; + } + @Test - public void Test() { + public void test() { RayActor adder = Ray.create(ActorTest.Adder.class); Ray.call(Adder::set, adder, 10); @@ -55,6 +61,11 @@ public void Test() { public static class Adder { private List> objectList; + private Integer sum = 0; + + public static Integer add2(Integer n) { + return n + 1; + } public Integer set(Integer n) { sum = n; @@ -69,10 +80,6 @@ public Integer add(Integer n) { return (sum += n); } - public static Integer add2(Integer n) { - return n + 1; - } - public Integer setObjectList(List> objectList) { this.objectList = objectList; return 1; @@ -81,14 +88,6 @@ public Integer setObjectList(List> objectList) { public Integer testObjectList() { return ((RayObject) objectList.get(0)).get(); } - - private Integer sum = 0; - } - - @RayRemote - public static Integer sayWorld(Integer n, RayActor adder) { - RayObject result = Ray.call(ActorTest.Adder::add, adder, 1); - return result.get() + n; } @RayRemote @@ -99,6 +98,11 @@ public static class Adder2 { private List> adderList; private UniqueID id; + private Integer sum = 0; + + public static Integer add2(Adder a, Integer n) { + return n + 1; + } public Integer set(Integer n) { sum = n; @@ -121,10 +125,6 @@ public Integer add(Integer n) { return (sum += n); } - public static Integer add2(Adder a, Integer n) { - return n + 1; - } - public RayActor getAdder() { return adder; } @@ -148,7 +148,5 @@ public Integer setAdderList(List> adderList) { this.adderList = adderList; return 0; } - - private Integer sum = 0; } } diff --git a/java/test/src/main/java/org/ray/api/test/EchoTest.java b/java/test/src/main/java/org/ray/api/test/EchoTest.java index ce56205c8ab7..a8eb52268b0e 100644 --- a/java/test/src/main/java/org/ray/api/test/EchoTest.java +++ b/java/test/src/main/java/org/ray/api/test/EchoTest.java @@ -8,9 +8,25 @@ @RunWith(MyRunner.class) public class EchoTest { + @RayRemote + public static String hi() { + return "hi"; + } + + @RayRemote + public static String who(String who) { + return who; + } + + @RayRemote + public static String recho(String pre, String who) { + return pre + ", " + who + "!"; + } + @Test public void test() { - long startTime, endTime; + long startTime = 0; + long endTime = 0; for (int i = 0; i < 100; i++) { startTime = System.nanoTime(); String ret = echo("Ray++" + i); @@ -19,28 +35,12 @@ public void test() { } } - public String echo(String who) { return Ray.call( EchoTest::recho, - Ray.call(EchoTest::Hi), - Ray.call(EchoTest::Who, who) + Ray.call(EchoTest::hi), + Ray.call(EchoTest::who, who) ).get(); } - - @RayRemote - public static String Hi() { - return "Hi"; - } - - @RayRemote - public static String Who(String who) { - return who; - } - - @RayRemote - public static String recho(String pre, String who) { - return pre + ", " + who + "!"; - } } diff --git a/java/test/src/main/java/org/ray/api/test/HelloWorldTest.java b/java/test/src/main/java/org/ray/api/test/HelloWorldTest.java index eb88d945cefc..db2f79865a0e 100644 --- a/java/test/src/main/java/org/ray/api/test/HelloWorldTest.java +++ b/java/test/src/main/java/org/ray/api/test/HelloWorldTest.java @@ -9,25 +9,11 @@ import org.ray.util.logger.RayLog; /** - * Hello world + * Hello world. */ @RunWith(MyRunner.class) public class HelloWorldTest { - @Test - public void test() { - String helloWorld = sayHelloWorld(); - RayLog.rapp.info(helloWorld); - Assert.assertEquals("hello,world!", helloWorld); - Assert.assertTrue(Ray.call(TypesTest::sayBool).get()); - } - - public String sayHelloWorld() { - RayObject hello = Ray.call(HelloWorldTest::sayHello); - RayObject world = Ray.call(HelloWorldTest::sayWorld); - return Ray.call(HelloWorldTest::merge, hello, world).get(); - } - @RayRemote public static String sayHello() { String ret = "he"; @@ -48,4 +34,18 @@ public static String sayWorld() { public static String merge(String hello, String world) { return hello + "," + world; } + + @Test + public void test() { + String helloWorld = sayHelloWorld(); + RayLog.rapp.info(helloWorld); + Assert.assertEquals("hello,world!", helloWorld); + Assert.assertTrue(Ray.call(TypesTest::sayBool).get()); + } + + public String sayHelloWorld() { + RayObject hello = Ray.call(HelloWorldTest::sayHello); + RayObject world = Ray.call(HelloWorldTest::sayWorld); + return Ray.call(HelloWorldTest::merge, hello, world).get(); + } } diff --git a/java/test/src/main/java/org/ray/api/test/MemoryWordCountTest.java b/java/test/src/main/java/org/ray/api/test/MemoryWordCountTest.java index a95686d3b3c0..ce0eafe2f45d 100644 --- a/java/test/src/main/java/org/ray/api/test/MemoryWordCountTest.java +++ b/java/test/src/main/java/org/ray/api/test/MemoryWordCountTest.java @@ -10,26 +10,11 @@ import org.ray.api.experiment.mr.MemoryMapReduce; /** - * test the MapReduce interface + * test the MapReduce interface. */ @RunWith(MyRunner.class) public class MemoryWordCountTest { - public static class MemoryWordCount extends MemoryMapReduce { - - public List> Map(String input) { - ArrayList> counts = new ArrayList<>(); - for (String s : input.split(" ")) { - counts.add(Pair.of(s, 1)); - } - return counts; - } - - public Integer Reduce(String k, List values) { - return values.size(); - } - } - @Test public void test() { List> iinputs = new ArrayList<>(); @@ -55,7 +40,7 @@ public void test() { iinputs.add(inputs); MemoryWordCount wc = new MemoryWordCount(); - SortedMap result = wc.Run(iinputs, 2, 2); + SortedMap result = wc.run(iinputs, 2, 2); Assert.assertEquals(6, (int) result.get("0")); Assert.assertEquals(6, (int) result.get("1")); @@ -68,4 +53,19 @@ public void test() { Assert.assertEquals(6, (int) result.get("8")); Assert.assertEquals(6, (int) result.get("9")); } + + public static class MemoryWordCount extends MemoryMapReduce { + + public List> map(String input) { + ArrayList> counts = new ArrayList<>(); + for (String s : input.split(" ")) { + counts.add(Pair.of(s, 1)); + } + return counts; + } + + public Integer reduce(String k, List values) { + return values.size(); + } + } } diff --git a/java/test/src/main/java/org/ray/api/test/RayMethodsTest.java b/java/test/src/main/java/org/ray/api/test/RayMethodsTest.java index 39f4233264b6..ca2ae1be099d 100644 --- a/java/test/src/main/java/org/ray/api/test/RayMethodsTest.java +++ b/java/test/src/main/java/org/ray/api/test/RayMethodsTest.java @@ -24,29 +24,27 @@ public void test() { RayObject s2Id = Ray.put(String.valueOf("World!")); RayObject n1Id = Ray.put(null); - RayList wIds = new RayList<>(); - wIds.add(s1Id); - wIds.add(s2Id); - WaitResult readys = Ray.wait(wIds, 2); + RayList waitIds = new RayList<>(); + waitIds.add(s1Id); + waitIds.add(s2Id); + WaitResult readys = Ray.wait(waitIds, 2); List ss = readys.getReadyOnes().get(); int i1 = i1Id.get(); double f1 = f1Id.get(); Object n1 = n1Id.get(); - RayLog.rapp.info("Strings: " + ss.get(0) + ss.get(1) + - " int: " + i1 + - " double: " + f1 + - " null: " + n1); + RayLog.rapp.info("Strings: " + ss.get(0) + ss.get(1) + " int: " + i1 + " double: " + f1 + + " null: " + n1); Assert.assertEquals("Hello World!", ss.get(0) + ss.get(1)); Assert.assertEquals(1, i1); Assert.assertEquals(3.14, f1, Double.MIN_NORMAL); Assert.assertNull(n1); // metadata test - RayObject vId = Ray.put(643, "test metadata"); - Integer v = vId.get(); - String m = vId.getMeta(); + RayObject vid = Ray.put(643, "test metadata"); + Integer v = vid.get(); + String m = vid.getMeta(); Assert.assertEquals(643L, v.longValue()); Assert.assertEquals("test metadata", m); diff --git a/java/test/src/main/java/org/ray/api/test/RemoteLambdaTest.java b/java/test/src/main/java/org/ray/api/test/RemoteLambdaTest.java index 09fa1eb7c446..ec363973f91d 100644 --- a/java/test/src/main/java/org/ray/api/test/RemoteLambdaTest.java +++ b/java/test/src/main/java/org/ray/api/test/RemoteLambdaTest.java @@ -9,20 +9,19 @@ @RunWith(MyRunner.class) public class RemoteLambdaTest { - public static String RemoteToString(T o) { + public static String remoteToString(T o) { return o.toString(); } @Test public void test() { - RemoteFunction f0 = RemoteLambdaTest::RemoteToString; + RemoteFunction f0 = RemoteLambdaTest::remoteToString; byte[] bytes = SerializationUtils.serialize(f0); //System.out.println(new String(bytes)); //Object m = SerializationUtils.deserialize(bytes); - RemoteFunction f = x -> - { + RemoteFunction f = x -> { System.out.println("remote function " + x); return x + 1; }; @@ -31,8 +30,7 @@ public void test() { Assert.assertEquals(101, (int) f2.apply(100)); Integer y = 100; - RemoteFunction f3 = x -> - { + RemoteFunction f3 = x -> { System.out.println("remote function " + x); return x + y + 1; }; diff --git a/java/test/src/main/java/org/ray/api/test/RpcTest.java b/java/test/src/main/java/org/ray/api/test/RpcTest.java index aca61e68ed75..0c8165b74f96 100644 --- a/java/test/src/main/java/org/ray/api/test/RpcTest.java +++ b/java/test/src/main/java/org/ray/api/test/RpcTest.java @@ -9,14 +9,6 @@ @RunWith(MyRunner.class) public class RpcTest { - @Test - public void test() { - Assert.assertEquals(0, (int) Ray.call(RpcTest::with0Params).get()); - Assert.assertEquals(1, (int) Ray.call(RpcTest::with1Params, 1).get()); - Assert.assertEquals(3, (int) Ray.call(RpcTest::with2Params, 1, 2).get()); - Assert.assertEquals(6, (int) Ray.call(RpcTest::with3Params, 1, 2, 3).get()); - } - @RayRemote public static Integer with0Params() { return 0; @@ -36,4 +28,12 @@ public static Integer with2Params(Integer x, Integer y) { public static Integer with3Params(Integer x, Integer y, Integer z) { return x + y + z; } + + @Test + public void test() { + Assert.assertEquals(0, (int) Ray.call(RpcTest::with0Params).get()); + Assert.assertEquals(1, (int) Ray.call(RpcTest::with1Params, 1).get()); + Assert.assertEquals(3, (int) Ray.call(RpcTest::with2Params, 1, 2).get()); + Assert.assertEquals(6, (int) Ray.call(RpcTest::with3Params, 1, 2, 3).get()); + } } diff --git a/java/test/src/main/java/org/ray/api/test/TwoClassTest.java b/java/test/src/main/java/org/ray/api/test/TwoClassTest.java index fa0083507926..aa3a7375d327 100644 --- a/java/test/src/main/java/org/ray/api/test/TwoClassTest.java +++ b/java/test/src/main/java/org/ray/api/test/TwoClassTest.java @@ -6,7 +6,7 @@ import org.ray.api.Ray; /** - * local test in IDE, for class lazy load + * local test in IDE, for class lazy load. */ @RunWith(MyRunner.class) public class TwoClassTest { diff --git a/java/test/src/main/java/org/ray/api/test/TypesTest.java b/java/test/src/main/java/org/ray/api/test/TypesTest.java index 9fad470d3793..264f613223a2 100644 --- a/java/test/src/main/java/org/ray/api/test/TypesTest.java +++ b/java/test/src/main/java/org/ray/api/test/TypesTest.java @@ -19,84 +19,11 @@ import org.ray.api.returns.RayObjects2; /** - * types test + * types test. */ @RunWith(MyRunner.class) public class TypesTest { - @Test - public void test() { - sayTypes(); - } - - public void sayTypes() { - - Assert.assertEquals(1, (int) Ray.call(TypesTest::sayInt).get()); - Assert.assertEquals(1, (byte) Ray.call(TypesTest::sayByte).get()); - Assert.assertEquals(1, (short) Ray.call(TypesTest::sayShort).get()); - Assert.assertEquals(1, (long) Ray.call(TypesTest::sayLong).get()); - Assert.assertEquals(1.0, Ray.call(TypesTest::sayDouble).get(), 0.0); - Assert.assertEquals(1.0f, Ray.call(TypesTest::sayFloat).get(), 0.0); - Assert.assertEquals(true, Ray.call(TypesTest::sayBool).get()); - Assert.assertEquals("object", Ray.call(TypesTest::sayReference).get()); - - RayObjects2 refs = Ray.call_2(TypesTest::sayReferences); - Assert.assertEquals(123, (int) refs.r0().get()); - Assert.assertEquals("123", refs.r1().get()); - - RayMap futureRs = Ray.call_n(TypesTest::sayReferencesN, - Arrays.asList(1, 2, 4, 3), "n_refs_", "_suffix"); - for (Entry> fne : futureRs.EntrySet()) { - Assert.assertEquals(fne.getValue().get(), "n_refs_" + fne.getKey() + "_suffix"); - } - - RayMap futureRs2 = Ray.call_n(TypesTest::sayReferencesN, - Arrays.asList(1), "n_refs_", "_suffix"); - for (Entry> fne : futureRs2.EntrySet()) { - Assert.assertEquals(fne.getValue().get(), "n_refs_" + fne.getKey() + "_suffix"); - } - - RayObject future = Ray.call(TypesTest::sayRayFuture); - Assert.assertEquals(123, (int) future.get()); - RayObjects2 futures = Ray.call_2(TypesTest::sayRayFutures); - Assert.assertEquals(123, (int) futures.r0().get()); - Assert.assertEquals("123", futures.r1().get()); - RayMap futureNs = Ray.call_n(TypesTest::sayRayFuturesN, - Arrays.asList(1, 2, 4, 3), "n_futures_"); - for (Entry> fne : futureNs.EntrySet()) { - Assert.assertEquals(fne.getValue().get(), "n_futures_" + fne.getKey()); - } - - RayList ns = Ray.call_n(TypesTest::sayArray, 10, 10); - for (int i = 0; i < 10; i++) { - Assert.assertEquals(i, (int) ns.Get(i).get()); - } - - RayList ns2 = Ray.call_n(TypesTest::sayArray, 1, 1); - Assert.assertEquals(0, (int) ns2.Get(0).get()); - - RayObject> ns3 = Ray.call(TypesTest::sayArray, 1); - Assert.assertEquals(0, (int) ns3.get().get(0)); - - RayList ints = new RayList<>(); - ints.add(Ray.call(TypesTest::sayInt)); - ints.add(Ray.call(TypesTest::sayInt)); - ints.add(Ray.call(TypesTest::sayInt)); - // TODO: when RayParameters.use_remote_lambda is on, we have to explicitly - // cast RayList and RayMap to List and Map explicitly, so that the parameter - // types of the lambdas can be correctly deducted. - RayObject collection = Ray.call(TypesTest::sayReadRayList, (List) ints); - Assert.assertEquals(3, (int) collection.get()); - - RayMap namedInts = new RayMap(); - namedInts.put("a", Ray.call(TypesTest::sayInt)); - namedInts.put("b", Ray.call(TypesTest::sayInt)); - namedInts.put("c", Ray.call(TypesTest::sayInt)); - RayObject collection2 = Ray - .call(TypesTest::sayReadRayMap, (Map) namedInts); - Assert.assertEquals(3, (int) collection2.get()); - } - @RayRemote public static int sayInt() { return 1; @@ -144,7 +71,7 @@ public static MultipleReturns2 sayReferences() { @RayRemote public static Map sayReferencesN(Collection userReturnIds, - String prefix, String suffix) { + String prefix, String suffix) { Map ret = new HashMap<>(); for (Integer returnid : userReturnIds) { ret.put(returnid, prefix + returnid + suffix); @@ -161,17 +88,17 @@ public static List sayArray(Integer returnCount) { return rets; } - @RayRemote(externalIO = true) + @RayRemote(externalIo = true) public static Integer sayRayFuture() { return 123; } - @RayRemote(externalIO = true) + @RayRemote(externalIo = true) public static MultipleReturns2 sayRayFutures() { return new MultipleReturns2<>(123, "123"); } - @RayRemote(externalIO = true) + @RayRemote(externalIo = true) public static Map sayRayFuturesN( Collection userReturnIds, String prefix) { @@ -199,4 +126,77 @@ public static int sayReadRayMap(Map ints) { } return sum; } + + @Test + public void test() { + sayTypes(); + } + + public void sayTypes() { + + Assert.assertEquals(1, (int) Ray.call(TypesTest::sayInt).get()); + Assert.assertEquals(1, (byte) Ray.call(TypesTest::sayByte).get()); + Assert.assertEquals(1, (short) Ray.call(TypesTest::sayShort).get()); + Assert.assertEquals(1, (long) Ray.call(TypesTest::sayLong).get()); + Assert.assertEquals(1.0, Ray.call(TypesTest::sayDouble).get(), 0.0); + Assert.assertEquals(1.0f, Ray.call(TypesTest::sayFloat).get(), 0.0); + Assert.assertEquals(true, Ray.call(TypesTest::sayBool).get()); + Assert.assertEquals("object", Ray.call(TypesTest::sayReference).get()); + + RayObjects2 refs = Ray.call_2(TypesTest::sayReferences); + Assert.assertEquals(123, (int) refs.r0().get()); + Assert.assertEquals("123", refs.r1().get()); + + RayMap futureRs = Ray.call_n(TypesTest::sayReferencesN, + Arrays.asList(1, 2, 4, 3), "n_refs_", "_suffix"); + for (Entry> fne : futureRs.EntrySet()) { + Assert.assertEquals(fne.getValue().get(), "n_refs_" + fne.getKey() + "_suffix"); + } + + RayMap futureRs2 = Ray.call_n(TypesTest::sayReferencesN, + Arrays.asList(1), "n_refs_", "_suffix"); + for (Entry> fne : futureRs2.EntrySet()) { + Assert.assertEquals(fne.getValue().get(), "n_refs_" + fne.getKey() + "_suffix"); + } + + RayObject future = Ray.call(TypesTest::sayRayFuture); + Assert.assertEquals(123, (int) future.get()); + RayObjects2 futures = Ray.call_2(TypesTest::sayRayFutures); + Assert.assertEquals(123, (int) futures.r0().get()); + Assert.assertEquals("123", futures.r1().get()); + RayMap futureNs = Ray.call_n(TypesTest::sayRayFuturesN, + Arrays.asList(1, 2, 4, 3), "n_futures_"); + for (Entry> fne : futureNs.EntrySet()) { + Assert.assertEquals(fne.getValue().get(), "n_futures_" + fne.getKey()); + } + + RayList ns = Ray.call_n(TypesTest::sayArray, 10, 10); + for (int i = 0; i < 10; i++) { + Assert.assertEquals(i, (int) ns.Get(i).get()); + } + + RayList ns2 = Ray.call_n(TypesTest::sayArray, 1, 1); + Assert.assertEquals(0, (int) ns2.Get(0).get()); + + RayObject> ns3 = Ray.call(TypesTest::sayArray, 1); + Assert.assertEquals(0, (int) ns3.get().get(0)); + + RayList ints = new RayList<>(); + ints.add(Ray.call(TypesTest::sayInt)); + ints.add(Ray.call(TypesTest::sayInt)); + ints.add(Ray.call(TypesTest::sayInt)); + // TODO: when RayParameters.use_remote_lambda is on, we have to explicitly + // cast RayList and RayMap to List and map explicitly, so that the parameter + // types of the lambdas can be correctly deducted. + RayObject collection = Ray.call(TypesTest::sayReadRayList, (List) ints); + Assert.assertEquals(3, (int) collection.get()); + + RayMap namedInts = new RayMap(); + namedInts.put("a", Ray.call(TypesTest::sayInt)); + namedInts.put("b", Ray.call(TypesTest::sayInt)); + namedInts.put("c", Ray.call(TypesTest::sayInt)); + RayObject collection2 = Ray + .call(TypesTest::sayReadRayMap, (Map) namedInts); + Assert.assertEquals(3, (int) collection2.get()); + } } diff --git a/java/test/src/main/java/org/ray/api/test/UniqueIDTest.java b/java/test/src/main/java/org/ray/api/test/UIdTest.java similarity index 91% rename from java/test/src/main/java/org/ray/api/test/UniqueIDTest.java rename to java/test/src/main/java/org/ray/api/test/UIdTest.java index d28f9e7d900b..dd93e16bddbb 100644 --- a/java/test/src/main/java/org/ray/api/test/UniqueIDTest.java +++ b/java/test/src/main/java/org/ray/api/test/UIdTest.java @@ -11,14 +11,20 @@ import org.ray.core.UniqueIdHelper; @RunWith(MyRunner.class) -public class UniqueIDTest { +public class UIdTest { + + + @RayRemote + public static String hi(Integer i) { + return "hi" + i; + } @Test public void test() { UniqueID tid = UniqueIdHelper.nextTaskId(0xdeadbeefL); UniqueIdHelper.setTest(tid, true); System.out.println("Tested task id = " + tid); - RayFunc_1_1 f = UniqueIDTest::hi; + RayFunc_1_1 f = UIdTest::hi; RayObject result = new RayObject<>( RayRuntime.getInstance().call( tid, @@ -30,9 +36,4 @@ public void test() { System.out.println("Tested task return object id = " + result.getId()); Assert.assertEquals("hi1", result.get()); } - - @RayRemote - public static String hi(Integer i) { - return "hi" + i; - } } diff --git a/java/test/src/main/java/org/ray/api/test/WordCountTest.java b/java/test/src/main/java/org/ray/api/test/WordCountTest.java index 0e73bd2cb9f3..92c6763d1004 100644 --- a/java/test/src/main/java/org/ray/api/test/WordCountTest.java +++ b/java/test/src/main/java/org/ray/api/test/WordCountTest.java @@ -11,30 +11,10 @@ /** * given a directory of document files on each "machine", we would like to count the appearance of - * some word + * some word. */ public class WordCountTest { - //@Test - public void test() { - int sum = mapReduce(); - Assert.assertEquals(sum, 143); - } - - public int mapReduce() { - RayObject> machines = Ray.call(WordCountTest::getMachineList); - RayObject total = null; - for (String machine : machines.get()) { - RayObject wordcount = Ray.call(WordCountTest::countWord, machine, "ray"); - if (total == null) { - total = wordcount; - } else { - total = Ray.call(WordCountTest::sum, total, wordcount); - } - } - return total.get(); - } - @RayRemote public static List getMachineList() { return Arrays.asList("A", "B", "C"); @@ -71,4 +51,24 @@ public static Integer countWord(String machine, String word) { public static Integer sum(Integer a, Integer/*TODO modify int to Integer in ASM hook*/ b) { return a + b; } + + //@Test + public void test() { + int sum = mapReduce(); + Assert.assertEquals(sum, 143); + } + + public int mapReduce() { + RayObject> machines = Ray.call(WordCountTest::getMachineList); + RayObject total = null; + for (String machine : machines.get()) { + RayObject wordcount = Ray.call(WordCountTest::countWord, machine, "ray"); + if (total == null) { + total = wordcount; + } else { + total = Ray.call(WordCountTest::sum, total, wordcount); + } + } + return total.get(); + } }