Merge pull request #64 from autolab/feature_getImages

Adding getImages in VMMS

clients/tango-rest.py

@@ -35,17 +35,17 @@
 parser.add_argument('-o', '--open', action='store_true', help=open_help)
 upload_help = 'Uploads a file. Must specify key with -k, courselab with -l, and filename with --filename.'
 parser.add_argument('-u', '--upload', action='store_true', help=upload_help)
-addJob_help = 'Submit a job. Must specify key with -k, courselab with -l, and input files with --infiles. Modify defaults with --image (rhel.img), --outputFile (result.out), --jobname (test_job), --maxsize(0), --timeout (0).'
+addJob_help = 'Submit a job. Must specify key with -k, courselab with -l, and input files with --infiles. Modify defaults with --image (rhel), --outputFile (result.out), --jobname (test_job), --maxsize(0), --timeout (0).'
 parser.add_argument('-a', '--addJob', action='store_true', help=addJob_help)
 poll_help = 'Poll a given output file. Must specify key with -k, courselab with -l. Modify defaults with --outputFile (result.out).'
 parser.add_argument('-p', '--poll', action='store_true', help=poll_help)
 info_help = 'Obtain basic stats about the service such as uptime, number of jobs, number of threads etc. Must specify key with -k.'
 parser.add_argument('-i', '--info', action='store_true', help=info_help)
 jobs_help = 'Obtain information of live jobs (deadJobs == 0) or dead jobs (deadJobs == 1). Must specify key with -k. Modify defaults with --deadJobs (0).'
 parser.add_argument('-j', '--jobs', action='store_true', help=jobs_help)
-pool_help = 'Obtain information about a pool of VMs spawned from a specific image. Must specify key with -k. Modify defaults with --image (rhel.img).'
+pool_help = 'Obtain information about a pool of VMs spawned from a specific image. Must specify key with -k. Modify defaults with --image (rhel).'
 parser.add_argument('--pool', action='store_true', help=pool_help)
-prealloc_help = 'Create a pool of instances spawned from a specific image. Must specify key with -k. Modify defaults with --image (rhel.img), --num (2), --vmms (tashiSSH), --cores (1), and --memory (512).'
+prealloc_help = 'Create a pool of instances spawned from a specific image. Must specify key with -k. Modify defaults with --image (rhel), --num (2), --vmms (tashiSSH), --cores (1), and --memory (512).'
 parser.add_argument('--prealloc', action='store_true', help=prealloc_help)
 
 parser.add_argument('--runJob', help='Run a job from a specific directory')
@@ -54,8 +54,8 @@
 
 parser.add_argument('--vmms', default='tashiSSH',
                     help='Choose vmms between localSSH, ec2SSH, tashiSSH')
-parser.add_argument('--image', default='rhel.img',
-                    help='VM image name (default "rhel.img")')
+parser.add_argument('--image', default='rhel',
+                    help='VM image name (default "rhel")')
 parser.add_argument(
     '--infiles',
     nargs='+',

config.template.py

@@ -97,9 +97,6 @@ class Config:
     # Default vm pool size
     POOL_SIZE = 2
 
-    # Path for tashi images
-    TASHI_IMAGE_PATH = ''
-
     # Optionally log finer-grained timing information
     LOG_TIMING = False
 

preallocator.py

@@ -40,10 +40,10 @@ def update(self, vm, num):
         """ update - Updates the number of machines of a certain type
         to be preallocated.
 
-        This function is called via the TangoServer HTTP interface, and
-        therefore should do as little as possible before returning.
-        It will update the machine list, and then spawn child threads
-        to do the creation and destruction of machines as necessary.
+        This function is called via the TangoServer HTTP interface.
+        It will validate the request,update the machine list, and 
+        then spawn child threads to do the creation and destruction 
+        of machines as necessary.
         """
         self.lock.acquire()
         if vm.name not in self.machines.keys():

restful-tango/tangoREST.py

@@ -43,6 +43,7 @@ def __init__(self):
         self.wrong_courselab = self.create(-1, "Courselab not found")
         self.out_not_found = self.create(-1, "Output file not found")
         self.invalid_image = self.create(-1, "Invalid image name")
+        self.invalid_prealloc_size = self.create(-1, "Invalid prealloc size")
         self.pool_not_found = self.create(-1, "Pool not found")
         self.prealloc_failed = self.create(-1, "Preallocate VM failed")
 
@@ -414,13 +415,9 @@ def pool(self, key, image):
         """
         self.log.debug("Received pool request(%s, %s)" % (key, image))
         if self.validateKey(key):
-            if not image or image == "" or not image.endswith(".img"):
-                self.log.info("Invalid image name")
-                return self.status.invalid_image
-            image = image[:-4]
             info = self.preallocator.getPool(image)
             if len(info["pool"]) == 0:
-                self.log.info("Pool image not found: %s" % image)
+                self.log.info("Pool not found: %s" % image)
                 return self.status.pool_not_found
             self.log.info("Pool image found: %s" % image)
             result = self.status.obtained_pool
@@ -437,18 +434,23 @@ def prealloc(self, key, image, num, vmStr):
         self.log.debug("Received prealloc request(%s, %s, %s)" %
                        (key, image, num))
         if self.validateKey(key):
-            if not image or image == "" or not image.endswith(".img"):
-                self.log.info("Invalid image name")
-                return self.status.invalid_image
             if vmStr != "":
                 vmObj = json.loads(vmStr)
                 vm = self.createTangoMachine(image, vmObj=vmObj)
             else:
                 vm = self.createTangoMachine(image)
-            success = self.tango.preallocVM(vm, int(num))
-            if (success == -1):
-                self.log.info("Failed to preallocated VMs")
+
+            ret = self.tango.preallocVM(vm, int(num))
+
+            if ret == -1:
+                self.log.error("Prealloc failed")
                 return self.status.prealloc_failed
+            if ret == -2:
+                self.log.error("Invalid prealloc size")
+                return self.status.invalid_prealloc_size
+            if ret == -3:
+                self.log.error("Invalid image name")
+                return self.status.invalid_image
             self.log.info("Successfully preallocated VMs")
             return self.status.preallocated
         else:

tangod.py

@@ -43,6 +43,7 @@
 
 from config import Config
 from tangoObjects import TangoJob
+from datetime import datetime
 
 
 class TangoServer:
@@ -110,10 +111,14 @@ def preallocVM(self, vm, num):
         self.log.debug("Received preallocVM(%s,%d)request"
                        % (vm.name, num))
         try:
+            vmms = self.vmms[vm.vmms]
             if not vm or num < 0:
-                return -1
-            (base, ext) = os.path.splitext(vm.image)
-            vm.name = base
+                return -2
+            if vm.image not in vmms.getImages():
+                self.log.error("Invalid image name")
+                return -3
+            (name, ext) = os.path.splitext(vm.image)
+            vm.name = name
             self.preallocator.update(vm, num)
             return 0
         except Exception as err:
@@ -247,25 +252,17 @@ def validateJob(job, vmms):
                             (datetime.utcnow().ctime()))
             errors += 1
         else:
-            if job.vm.vmms == "tashiSSH":
-                # Check if VM name exists in Tashi directory
-                imgList = os.listdir(Config.TASHI_IMAGE_PATH)
-                imgPath = Config.TASHI_IMAGE_PATH + job.vm.image
-                if job.vm.image not in imgList:
-                    log.error("validateJob: Image not found: %s" %
-                              job.vm.image)
-                    job.appendTrace("%s|validateJob: Image not found: %s" %
-                                    (datetime.utcnow().ctime(), job.vm.image))
-                    errors += 1
-                # Check if image has read permissions
-                elif not (os.stat(imgPath).st_mode & stat.S_IRUSR):
-                    log.error("validateJob: Not readable: %s" % job.vm.image)
-                    job.appendTrace("%s|validateJob: Not readable: %s" %
-                                    (datetime.utcnow().ctime(), job.vm.image))
-                    errors += 1
-                else:
-                    (base, ext) = os.path.splitext(job.vm.image)
-                    job.vm.name = base
+            vobj = vmms[Config.VMMS_NAME]
+            imgList = vobj.getImages()
+            if job.vm.image not in imgList:
+                log.error("validateJob: Image not found: %s" %
+                          job.vm.image)
+                job.appendTrace("%s|validateJob: Image not found: %s" %
+                                (datetime.utcnow().ctime(), job.vm.image))
+                errors += 1
+            else:
+                (name, ext) = os.path.splitext(job.vm.image)
+                job.vm.name = name
 
         if not job.vm.vmms:
             log.error("validateJob: Missing job.vm.vmms")
@@ -323,7 +320,7 @@ def validateJob(job, vmms):
     # Any problems, return an error status
     if errors > 0:
         log.error("validateJob: Job rejected: %d errors" % errors)
-        job.timerace.append("%s|validateJob: Job rejected: %d errors" %
+        job.appendTrace("%s|validateJob: Job rejected: %d errors" %
                             (datetime.utcnow().ctime(), errors))
         return -1
     else:

vmms/Dockerfile

@@ -7,6 +7,7 @@ RUN apt-get update
 RUN apt-get install -y gcc
 RUN apt-get install -y make
 RUN apt-get install -y build-essential
+RUN apt-get install -y wget
 
 # Install autodriver
 WORKDIR /home
@@ -23,12 +24,25 @@ RUN make clean && make
 RUN cp autodriver /usr/bin/autodriver
 RUN chmod +s /usr/bin/autodriver
 
+# Install C0
+WORKDIR /home
+RUN wget http://c0.typesafety.net/dist/cc0-v0440-linux3.18.1-64bit-bin.tgz
+RUN tar -xvzf cc0-*
+WORKDIR /home/cc0
+RUN bin/cc0 -d doc/src/exp.c0 doc/src/exp-test.c0
+RUN ./a.out
+RUN cp bin/cc0 /usr/bin/cc0
+
 # Clean up
 WORKDIR /home
 RUN apt-get remove -y git
+RUN apt-get remove -y wget
 RUN apt-get -y autoremove
 RUN rm -rf Tango/
+RUN rm -f cc0-*
+RUN rm -rf cc0/
 
 # Check installation
 RUN ls -l /home
-RUN which autodriver
+RUN which autodriver
+RUN which cc0

vmms/localDocker.py

@@ -193,7 +193,8 @@ def safeDestroyVM(self, vm):
         return
 
     def getVMs(self):
-        """ getVMs - Executes and parses `docker ps`
+        """ getVMs - Executes and parses `docker ps`. This function
+        is a lot of parsing and can break easily.
         """
         # Get all volumes of docker containers
         machines = []
@@ -217,3 +218,21 @@ def existsVM(self, vm):
         ret = timeout(['docker', 'inspect', instanceName])
         return (ret is 0)
 
+    def getImages(self):
+        """ getImages - Executes `docker images` and returns a list of
+        images that can be used to boot a docker container with. This 
+        function is a lot of parsing and so can break easily.
+        """
+        result = set()
+        cmd = "docker images"
+        o = subprocess.check_output("docker images", shell=True)
+        o_l = o.split('\n')
+        o_l.pop()
+        o_l.reverse()
+        o_l.pop()
+        for row in o_l:
+            row_l = row.split(' ')
+            result.add(row_l[0])
+        return list(result)
+
+

vmms/tashiSSH.py

@@ -85,8 +85,6 @@ def timeoutWithReturnStatus(command, time_out, returnValue=0):
 # User defined exceptions
 #
 # tashiCall() exception
-
-
 class tashiCallError(Exception):
     pass
 
@@ -96,6 +94,8 @@ class TashiSSH:
                   "-o", "StrictHostKeyChecking=no",
                   "-o", "GSSAPIAuthentication=no"]
 
+    TASHI_IMAGE_PATH = '/raid/tashi/images'
+
     def __init__(self):
         self.config = getConfig(["Client"])[0]
         self.client = createClient(self.config)
@@ -404,3 +404,9 @@ def existsVM(self, vm):
             if vm.instance_id == instance.id:
                 return True
         return False
+
+    def getImages(self):
+        """ getImages - Lists all images in TASHI_IMAGE_PATH that have the
+        .img extension
+        """
+        return [img for img in os.listdir(Config.TASHI_IMAGE_PATH) if img.endswith('.img')]