Update worker.py and services.py to use plasma and the local scheduler. (#19)

* Update worker code and services code to use plasma and the local scheduler.

* Cleanups.

* Fix bug in which threads were started before the worker mode was set. This caused remote functions to be defined on workers before the worker knew it was in WORKER_MODE.

* Fix bug in install-dependencies.sh.

* Lengthen timeout in failure_test.py.

* Cleanups.

* Cleanup services.start_ray_local.

* Clean up random name generation.

* Cleanups.

Author: robertnishihara

.travis.yml

@@ -69,3 +69,8 @@ script:
   - python src/common/test/test.py
   - python src/plasma/test/test.py
   - python src/photon/test/test.py
+
+  - python test/runtest.py
+  - python test/array_test.py
+  - python test/failure_test.py
+  - python test/microbenchmarks.py

CMakeLists.txt

@@ -31,14 +31,14 @@ if [[ $platform == "linux" ]]; then
   # These commands must be kept in sync with the installation instructions.
   sudo apt-get update
   sudo apt-get install -y git cmake build-essential autoconf curl libtool python-dev python-numpy python-pip libboost-all-dev unzip graphviz
-  sudo pip install ipython funcsigs subprocess32 protobuf colorama graphviz
+  sudo pip install ipython funcsigs subprocess32 protobuf colorama graphviz redis
   sudo pip install --upgrade git+git://github.com/cloudpipe/cloudpickle.git@0d225a4695f1f65ae1cbb2e0bbc145e10167cce4  # We use the latest version of cloudpickle because it can serialize named tuples.
 elif [[ $platform == "macosx" ]]; then
   # These commands must be kept in sync with the installation instructions.
   brew install git cmake automake autoconf libtool boost graphviz
   sudo easy_install pip
   sudo pip install ipython --user
-  sudo pip install numpy funcsigs subprocess32 protobuf colorama graphviz --ignore-installed six
+  sudo pip install numpy funcsigs subprocess32 protobuf colorama graphviz redis --ignore-installed six
   sudo pip install --upgrade git+git://github.com/cloudpipe/cloudpickle.git@0d225a4695f1f65ae1cbb2e0bbc145e10167cce4  # We use the latest version of cloudpickle because it can serialize named tuples.
 fi
 

data/README.md

@@ -11,8 +11,6 @@
 
 import config
 import serialization
-from worker import scheduler_info, register_class, visualize_computation_graph, task_info, init, connect, disconnect, get, put, wait, remote, kill_workers, restart_workers_local
+from worker import register_class, error_info, init, connect, disconnect, get, put, wait, remote
 from worker import Reusable, reusables
-from libraylib import SCRIPT_MODE, WORKER_MODE, PYTHON_MODE, SILENT_MODE
-from libraylib import ObjectID
-import internal
+from worker import SCRIPT_MODE, WORKER_MODE, PYTHON_MODE, SILENT_MODE

data/mini.tar

@@ -0,0 +1,25 @@
+from __future__ import print_function
+
+import sys
+import argparse
+import numpy as np
+
+import ray
+
+parser = argparse.ArgumentParser(description="Parse addresses for the worker to connect to.")
+parser.add_argument("--node-ip-address", required=True, type=str, help="the ip address of the worker's node")
+parser.add_argument("--redis-port", required=True, type=int, help="the port to use for Redis")
+parser.add_argument("--object-store-name", type=str, help="the object store's name")
+parser.add_argument("--object-store-manager-name", type=str, help="the object store manager's name")
+parser.add_argument("--local-scheduler-name", type=str, help="the local scheduler's name")
+
+if __name__ == "__main__":
+  args = parser.parse_args()
+  address_info = {"node_ip_address": args.node_ip_address,
+                  "redis_port": args.redis_port,
+                  "object_store_name": args.object_store_name,
+                  "object_store_manager_name": args.object_store_manager_name,
+                  "local_scheduler_name": args.local_scheduler_name}
+  ray.worker.connect(address_info, ray.WORKER_MODE)
+
+  ray.worker.main_loop()

install-dependencies.sh

@@ -1,89 +1,9 @@
+from __future__ import print_function
+
 import numpy as np
 import pickling
-import libraylib as raylib
 import numbuf
 
-def is_argument_serializable(value):
-  """Checks if value is a composition of primitive types.
-
-  This will return True if the argument is one of the following:
-    - An int
-    - A float
-    - A bool
-    - None
-    - A list of length at most 100 whose elements are serializable
-    - A tuple of length at most 100 whose elements are serializable
-    - A dict of length at most 100 whose keys and values are serializable
-    - A string of length at most 100.
-    - A unicode string of length at most 100.
-
-  Args:
-    value: A Python object.
-
-  Returns:
-    True if the object can be serialized as a composition of primitive types and
-      False otherwise.
-  """
-  t = type(value)
-  if t is int or t is float or t is long or t is bool or value is None:
-    return True
-  if t is list:
-    if len(value) <= 100:
-      for element in value:
-        if not is_argument_serializable(element):
-          return False
-      return True
-    else:
-      return False
-  if t is tuple:
-    if len(value) <= 100:
-      for element in value:
-        if not is_argument_serializable(element):
-          return False
-      return True
-    else:
-      return False
-  if t is dict:
-    if len(value) <= 100:
-      for k, v in value.iteritems():
-        if not is_argument_serializable(k) or not is_argument_serializable(v):
-          return False
-      return True
-    else:
-      return False
-  if t is str:
-    return len(value) <= 100
-  if t is unicode:
-    return len(value) <= 100
-  return False
-
-def serialize_argument_if_possible(value):
-  """This method serializes arguments that are passed by value.
-
-  The result will be deserialized by deserialize_argument.
-
-  Returns:
-    None if value cannot be efficiently serialized or is too big, and otherwise
-      this returns the serialized value as a string.
-  """
-  if not is_argument_serializable(value):
-    # The argument is not obviously serializable using __repr__, so we will not
-    # serialize it.
-    return None
-  serialized_value = value.__repr__()
-  if len(serialized_value) > 1000:
-    # The argument is too big, so we will not pass it by value.
-    return None
-  # Return the serialized argument.
-  return serialized_value
-
-def deserialize_argument(serialized_value):
-  """This method deserializes arguments that are passed by value.
-
-  The argument will have been serialized by serialize_argument.
-  """
-  return eval(serialized_value)
-
 def check_serializable(cls):
   """Throws an exception if Ray cannot serialize this class efficiently.