Add support for cloud data to xdmod-shredder and xdmod-ingestor

bin/xdmod-shredder

@@ -233,12 +233,17 @@ function main()
         $logger->notice("Job errors written to '$jobErrorLogFile'");
     }
 
-    if (!$dryRun && in_array($format, array('pbs', 'slurm', 'lsf', 'sge', 'uge'))) {
-        $logger->notice('Normalizing data');
+    if (!$dryRun) {
+        $logger->notice('Normalizing data!');
 
         try {
             $ingestor = $shredder->getJobIngestor();
-            $ingestor->ingest();
+            // The cloud shredders do not have jobs to ingest and return false when
+            // getJobInestor is called for them so we don't have to hard code skippping 
+            // those formats here.
+            if($ingestor !== false){
+                $ingestor->ingest();
+            }
         } catch (Exception $e) {
             $logger->crit(array(
                 'message'    => 'Ingestion failed: ' . $e->getMessage(),

classes/OpenXdmod/Shredder/Genericcloud.php

@@ -7,14 +7,14 @@
 
 namespace OpenXdmod\Shredder;
 
+use CCR\DB\iDatabase;
+
 class Genericcloud extends aCloud
 {
     /**
      * @inheritdoc
      */
-    public function shredDirectory($directory)
-    {
-        $this->setEtlPipeline(['jobs-cloud-ingest-eucalyptus']);
-        return parent::shredDirectory($directory);
+    public function __construct(iDatabase $db){
+        parent::__construct($db, ['jobs-cloud-ingest-eucalyptus']);
     }
 }

classes/OpenXdmod/Shredder/Openstack.php

@@ -7,14 +7,14 @@
 
 namespace OpenXdmod\Shredder;
 
+use CCR\DB\iDatabase;
+
 class OpenStack extends aCloud
 {
     /**
      * @inheritdoc
      */
-    public function shredDirectory($directory)
-    {
-        $this->setEtlPipeline(['jobs-cloud-ingest-openstack']);
-        return parent::shredDirectory($directory);
+    public function __construct(iDatabase $db){
+        parent::__construct($db, ['jobs-cloud-ingest-openstack']);
     }
 }

classes/OpenXdmod/Shredder/aCloud.php

@@ -19,9 +19,10 @@ abstract class aCloud extends Shredder
     /**
      * @inheritdoc
      */
-    public function __construct(iDatabase $db)
+    public function __construct(iDatabase $db, array $pipelines)
     {
         $this->logger = \Log::singleton('null');
+        $this->etlPipelines = $pipelines;
     }
 
     /**
@@ -60,7 +61,10 @@ public function shredDirectory($directory)
         );
     }
 
-    public function setEtlPipeline(array $pipelines){
-        $this->etlPipelines = $pipelines;
+    /**
+     * Returns false so the same function in the parent class is not called since the cloud formats do not have jobs to ingest
+     */
+    public function getJobIngestor(){
+        return false;
     }
 }