Merge pull request #732 from eiffel777/update-start-stop-events

Update events that are used to determine VM session starts and stops
ubccr · Dec 21, 2018 · cfed4d8 · cfed4d8
2 parents 8579517 + 9066759
commit cfed4d8
Show file tree

Hide file tree

Showing 248 changed files with 333 additions and 267 deletions.
diff --git a/classes/ETL/Ingestor/CloudStateReconstructorTransformIngestor.php b/classes/ETL/Ingestor/CloudStateReconstructorTransformIngestor.php
@@ -25,14 +25,22 @@
 class CloudStateReconstructorTransformIngestor extends pdoIngestor implements iAction
 {
     // Discrete Event Types
+    // Start events
     const START = 2;
-    const STOP = 4;
-    const TERMINATE = 6;
     const RESUME = 8;
     const STATE_REPORT = 16;
+    const UNSHELVE = 20;
+    const UNPAUSE = 57;
+    const UNSUSPEND = 61;
+    const POWER_ON = 59;
+
+    //End events
+    const STOP = 4;
+    const TERMINATE = 6;
     const SUSPEND = 17;
     const SHELVE = 19;
-    const UNSHELVE = 20;
+    const POWER_OFF = 45;
+    const PAUSE = 55;
 
     private $_stop_event_ids;
     private $_start_event_ids;
@@ -46,8 +54,8 @@ public function __construct(aOptions $options, EtlConfiguration $etlConfig, Log
     {
         parent::__construct($options, $etlConfig, $logger);
 
-        $this->_stop_event_ids = array(self::STOP, self::TERMINATE, self::SUSPEND, self::SHELVE);
-        $this->_start_event_ids = array(self::START, self::RESUME, self::STATE_REPORT, self::UNSHELVE);
+        $this->_stop_event_ids = array(self::STOP, self::TERMINATE, self::SUSPEND, self::SHELVE, self::POWER_OFF, self::PAUSE);
+        $this->_start_event_ids = array(self::START, self::RESUME, self::STATE_REPORT, self::UNSHELVE, self::UNPAUSE, self::UNSUSPEND, self::POWER_ON);
         $this->_all_event_ids = array_merge($this->_start_event_ids, $this->_stop_event_ids);
         $this->_end_time = $etlConfig->getVariableStore()->endDate ? date('Y-m-d H:i:s', strtotime($etlConfig->getVariableStore()->endDate)) : null;
 

diff --git a/classes/OpenXdmod/DataWarehouseInitializer.php b/classes/OpenXdmod/DataWarehouseInitializer.php
@@ -138,7 +138,7 @@ public function ingestAll($startDate = null, $endDate = null)
      */
     public function ingestAllShredded($startDate = null, $endDate = null)
     {
-        if( $this->realmEnabled('Jobs')){
+        if( $this->isRealmEnabled('Jobs')){
             $this->logger->debug('Ingesting shredded data to staging tables');
             $this->runEtlPipeline('staging-ingest-common');
             $this->runEtlPipeline('staging-ingest-jobs');
@@ -153,7 +153,7 @@ public function ingestAllShredded($startDate = null, $endDate = null)
      */
     public function ingestAllStaging($startDate = null, $endDate = null)
     {
-        if( $this->realmEnabled('Jobs')){
+        if( $this->isRealmEnabled('Jobs')){
             $this->logger->debug('Ingesting staging data to HPCDB');
             $this->runEtlPipeline('hpcdb-ingest-common');
             $this->runEtlPipeline('hpcdb-ingest-jobs');
@@ -168,7 +168,7 @@ public function ingestAllStaging($startDate = null, $endDate = null)
      */
     public function ingestAllHpcdb($startDate = null, $endDate = null)
     {
-        if( $this->realmEnabled('Jobs')){
+        if( $this->isRealmEnabled('Jobs')){
             $this->logger->debug('Ingesting HPCDB data to modw');
 
             if ($startDate !== null || $endDate !== null) {
@@ -206,7 +206,7 @@ public function ingestAllHpcdb($startDate = null, $endDate = null)
      */
     public function ingestCloudDataOpenStack()
     {
-        if( $this->realmEnabled('Cloud') ){
+        if( $this->isRealmEnabled('Cloud') ){
             try{
                 $this->logger->notice('Ingesting OpenStack event log data');
                 $this->runEtlPipeline('jobs-cloud-extract-openstack');
@@ -229,7 +229,7 @@ public function ingestCloudDataOpenStack()
      */
     public function ingestCloudDataGeneric()
     {
-        if( $this->realmEnabled('Cloud') ){
+        if( $this->isRealmEnabled('Cloud') ){
             try{
                 $this->logger->notice('Ingesting generic cloud log files');
                 $this->runEtlPipeline('jobs-cloud-extract-eucalyptus');
@@ -252,7 +252,7 @@ public function ingestCloudDataGeneric()
      */
     public function aggregateCloudData()
     {
-        if( $this->realmEnabled('Cloud') ){
+        if( $this->isRealmEnabled('Cloud') ){
             $this->logger->notice('Aggregating Cloud data');
             $this->runEtlPipeline('cloud-state-pipeline');
 
@@ -288,7 +288,7 @@ public function initializeAggregation($startDate = null, $endDate = null)
      */
     public function aggregateAllJobs($lastModifiedStartDate)
     {
-        if( $this->realmEnabled('Jobs') ){
+        if( $this->isRealmEnabled('Jobs') ){
             $this->runEtlPipeline(
                 'jobs-xdw-aggregate',
                 array('last-modified-start-date' => $lastModifiedStartDate)
@@ -353,7 +353,7 @@ public function aggregate(
      *
      * @param string $realm The realm you are checking to see if exists
      */
-    private function realmEnabled($realm)
+    public function isRealmEnabled($realm)
     {
         $realms = $this->warehouseDb->query("SELECT * FROM moddb.realms WHERE display = :realm", [':realm' => $realm]);
         return (count($realms) > 0);

diff --git a/classes/OpenXdmod/Migration/DatabasesMigration.php b/classes/OpenXdmod/Migration/DatabasesMigration.php
@@ -81,6 +81,7 @@ public function execute()
             'mod_hpcdb'       => 'hpcdb',
             'modw_etl'        => 'logger',
             'modw_supremm'    => 'datawarehouse',
+            'modw_cloud'      => 'datawarehouse'
         );
 
         $dir = BASE_DIR . '/db/migrations/'

diff --git a/classes/OpenXdmod/Migration/Version800To810/DatabasesMigration.php b/classes/OpenXdmod/Migration/Version800To810/DatabasesMigration.php
@@ -2,9 +2,34 @@
 
 namespace OpenXdmod\Migration\Version800To810;
 
+use CCR\DB;
+use OpenXdmod\DataWarehouseInitializer;
+use OpenXdmod\Setup\Console;
+
 /**
 * Migrate databases from version 8.0.0 to 8.1.0.
 */
 class DatabasesMigration extends \OpenXdmod\Migration\DatabasesMigration
 {
+    /**
+     * @see \OpenXdmod\Migration\Migration::execute
+     **/
+    public function execute()
+    {
+        parent::execute();
+
+        $hpcdbDb = DB::factory('hpcdb');
+        $dwDb = DB::factory('datawarehouse');
+        $dwi = new DataWarehouseInitializer($hpcdbDb, $dwDb);
+
+        if($dwi->isRealmEnabled('Cloud')){
+            $console = Console::factory();
+            $console->displayMessage(<<<"EOT"
+There have been updates to cloud aggregation statistics to make the data more accurate.
+If you have the Cloud realm enabled it is recommended that you re-ingest and aggregate
+your cloud data using the commands recommended in our documentation.
+EOT
+            );
+        }
+    }
 }
diff --git a/configuration/etl/etl_data.d/cloud_common/event_type.json b/configuration/etl/etl_data.d/cloud_common/event_type.json
@@ -54,5 +54,12 @@
     [51, "RESIZE_END",                        "Resize Instance End",           "End resizing of instance"],
     [52, "RESIZE_PREP_START",                 "Instance Resize Prep Start",    "Start prep for resizing instance"],
     [53, "RESIZE_PREP_END",                   "Instance Resize Prep End",	     "End prep for resizing instance"],
-    [54, "START_SHELVE",                      "Start Shelve",                  "Start shelving of instance"]
+    [54, "PAUSE_START",                       "Start Pause Instance",          "Start pausing an instance"],
+    [55, "PAUSE_END",                         "End Pause Instance",            "End pausing an instance"],
+    [56, "UNPAUSE_START",                     "Start Unpause Instance",        "Start Unpausing an instance"],
+    [57, "UNPAUSE_END",                       "End Unpause Instance",          "End Unpausing an instance"],
+    [58, "POWER_ON_START",                    "Power On Instance Start",       "Start powering on an instance"],
+    [59, "POWER_ON_END",                      "Power On Instance End",         "End powering on an instance"],
+    [60, "UNSUSPEND_START",                   "Unsuspend Instance Start",      "Start unsuspending an instance"],
+    [61, "UNSUSPEND_END",                     "Unsuspend Instance End",        "End unsuspending an instance"]
 ]
diff --git a/configuration/etl/etl_data.d/cloud_openstack/openstack_event_map.json b/configuration/etl/etl_data.d/cloud_openstack/openstack_event_map.json
@@ -36,7 +36,7 @@
     [34, 42,    "compute.instance.finish_resize.start"],
     [35, 43,    "compute.instance.finish_resize.end"],
     [36, 44,    "compute.instance.power_off.start"],
-    [37, 17,    "compute.instance.power_off.end"],
+    [37, 45,    "compute.instance.power_off.end"],
     [38, 46,    "compute.instance.rebuild.start"],
     [39, 47,    "compute.instance.rebuild.end"],
     [40, 48,    "compute.instance.resize.confirm.start"],
@@ -46,6 +46,13 @@
     [44, 52,    "compute.instance.resize.prep.start"],
     [45, 53,    "compute.instance.resize.prep.end"],
     [46, 54,    "compute.instance.shelve_offload.start"],
-    [47, 7,    "compute.instance.power_on.start"],
-    [48, 8,    "compute.instance.power_off.end"]
+    [47, 7,     "compute.instance.resume.start"],
+    [48, 8,     "compute.instance.resume.end"],
+    [49, 58,    "compute.instance.power_on.start"],
+    [50, 59,    "compute.instance.power_on.end"],
+    [51, 20,    "compute.instance.unshelve.end"],
+    [52, 57,    "compute.instance.unpause.end"],
+    [53, 61,    "compute.instance.unsuspend.end"],
+    [54, 17,    "compute.instance.suspend.end"],
+    [55, 55,    "compute.instance.pause.end"]
 ]
diff --git a/configuration/etl/etl_tables.d/cloud_common/cloud_transient.json b/configuration/etl/etl_tables.d/cloud_common/cloud_transient.json
@@ -105,15 +105,17 @@
                 "name": "PRIMARY",
                 "columns": [
                     "resource_id",
-                    "instance"
+                    "instance",
+                    "start_time"
                 ],
                 "is_unique": true
             },
             {
                 "name": "increment_key",
                 "columns": [
                     "instance_id",
-                    "resource_id"
+                    "resource_id",
+                    "start_time"
                 ],
                 "is_unique": true
             }

diff --git a/db/migrations/8.0.0-8.1.0/modw_cloud.sql b/db/migrations/8.0.0-8.1.0/modw_cloud.sql
@@ -0,0 +1,14 @@
+USE modw_cloud;
+
+-- When adding new OpenStack events to track we are also updating the mapping of
+-- somme OpenStack events. The events that already exist in the staging and event
+-- need to have their mappings updated. Some of the updates include mapping the
+-- compute.instance.power_on.start event to POWER_ON_START event and
+-- compute.instance.resume.start to REQUEST_RESUME and compute.instance.resume.end 
+-- to RESUME.
+
+UPDATE `openstack_staging_event` SET event_type_id = 58 WHERE event_type_id = 7;
+UPDATE `openstack_staging_event` SET event_type_id = 45 WHERE event_type_id IN (8,17);
+
+UPDATE `event` SET event_type_id = 58 WHERE event_type_id = 7;
+UPDATE `event` SET event_type_id = 45 WHERE event_type_id IN (8,17);
diff --git a/open_xdmod/modules/xdmod/integration_tests/lib/Controllers/UsageExplorerTest.php b/open_xdmod/modules/xdmod/integration_tests/lib/Controllers/UsageExplorerTest.php
@@ -620,7 +620,7 @@ public function provideFilterIdLookup()
                     )
                 ),
                 'expected' => array(
-                    'value' => array('zealous', '1755.9078'),
+                    'value' => array('zealous', '1754.1644'),
                     'xpath' => '//rows//row//cell/value'
                 ),
                 'additional_data' => array(
@@ -667,7 +667,7 @@ public function provideFilterIdLookup()
                     )
                 ),
                 'expected' => array(
-                    'value' => array('zealous', '1755.9078'),
+                    'value' => array('zealous', '1754.1644'),
                     'xpath' => '//rows//row//cell/value'
                 ),
                 'additional_data' => array(

diff --git a/open_xdmod/modules/xdmod/integration_tests/scripts/bootstrap.sh b/open_xdmod/modules/xdmod/integration_tests/scripts/bootstrap.sh
@@ -44,4 +44,6 @@ then
     yum -y install ~/rpmbuild/RPMS/*/*.rpm
     ~/bin/services start
     expect $BASEDIR/xdmod-upgrade.tcl | col -b
+    sudo -u xdmod xdmod-shredder -r openstack -d $REF_DIR/openstack -f openstack
+    sudo -u xdmod xdmod-ingestor
 fi
diff --git a/...ed/reference/Cloud/configuration/cloud_avg_wallduration_hours/aggregate-Day-reference.csv b/...ed/reference/Cloud/configuration/cloud_avg_wallduration_hours/aggregate-Day-reference.csv
@@ -8,7 +8,7 @@ start,end
 "Instance Type","Average Wall Hours per Session"
 c2.m4,52.40465278
 c4.m16,35.26490741
-c1.m4,24.49683934
-c1.m1,1.10615741
+c1.m4,23.19631766
+c1.m1,1.10337963
 c2.m8,0.23583333
 ---------
diff --git a/.../reference/Cloud/configuration/cloud_avg_wallduration_hours/aggregate-Month-reference.csv b/.../reference/Cloud/configuration/cloud_avg_wallduration_hours/aggregate-Month-reference.csv
@@ -8,7 +8,7 @@ start,end
 "Instance Type","Average Wall Hours per Session"
 c2.m4,52.40465278
 c4.m16,35.26490741
-c1.m4,24.49683934
-c1.m1,1.10615741
+c1.m4,23.19631766
+c1.m1,1.10337963
 c2.m8,0.23583333
 ---------
diff --git a/...eference/Cloud/configuration/cloud_avg_wallduration_hours/aggregate-Quarter-reference.csv b/...eference/Cloud/configuration/cloud_avg_wallduration_hours/aggregate-Quarter-reference.csv
@@ -8,7 +8,7 @@ start,end
 "Instance Type","Average Wall Hours per Session"
 c2.m4,52.40465278
 c4.m16,35.26490741
-c1.m4,24.49683934
-c1.m1,1.10615741
+c1.m4,23.19631766
+c1.m1,1.10337963
 c2.m8,0.23583333
 ---------
diff --git a/...d/reference/Cloud/configuration/cloud_avg_wallduration_hours/aggregate-Year-reference.csv b/...d/reference/Cloud/configuration/cloud_avg_wallduration_hours/aggregate-Year-reference.csv
@@ -8,7 +8,7 @@ start,end
 "Instance Type","Average Wall Hours per Session"
 c2.m4,52.40465278
 c4.m16,35.26490741
-c1.m4,24.49683934
-c1.m1,1.10615741
+c1.m4,23.19631766
+c1.m1,1.10337963
 c2.m8,0.23583333
 ---------
diff --git a/...d/reference/Cloud/configuration/cloud_avg_wallduration_hours/timeseries-Day-reference.csv b/...d/reference/Cloud/configuration/cloud_avg_wallduration_hours/timeseries-Day-reference.csv
@@ -7,7 +7,7 @@ start,end
 ---------
 Day,"[c2.m4] Average Wall Hours per Session","[c4.m16] Average Wall Hours per Session","[c1.m4] Average Wall Hours per Session","[c1.m1] Average Wall Hours per Session","[c2.m8] Average Wall Hours per Session"
 2018-04-18,0.70833333,2.21361111,1.91267974,0.01555556,0.23583333
-2018-04-19,7.89694444,0,6.31328431,0.23756944,0
+2018-04-19,7.89694444,0,5.55785088,0.23340278,0
 2018-04-20,24.00000000,0,4.90686111,0,0
 2018-04-21,24.00000000,0,24.00000000,0,0
 2018-04-22,24.00000000,0,24.00000000,0,0

diff --git a/...reference/Cloud/configuration/cloud_avg_wallduration_hours/timeseries-Month-reference.csv b/...reference/Cloud/configuration/cloud_avg_wallduration_hours/timeseries-Month-reference.csv
@@ -6,5 +6,5 @@ start,end
 2018-04-18,2018-04-30
 ---------
 Month,"[c2.m4] Average Wall Hours per Session","[c4.m16] Average Wall Hours per Session","[c1.m4] Average Wall Hours per Session","[c1.m1] Average Wall Hours per Session","[c2.m8] Average Wall Hours per Session"
-2018-04,52.40465278,35.26490741,24.49683934,1.10615741,0.23583333
+2018-04,52.40465278,35.26490741,23.19631766,1.10337963,0.23583333
 ---------
diff --git a/...ference/Cloud/configuration/cloud_avg_wallduration_hours/timeseries-Quarter-reference.csv b/...ference/Cloud/configuration/cloud_avg_wallduration_hours/timeseries-Quarter-reference.csv
@@ -6,5 +6,5 @@ start,end
 2018-04-18,2018-04-30
 ---------
 Quarter,"[c2.m4] Average Wall Hours per Session","[c4.m16] Average Wall Hours per Session","[c1.m4] Average Wall Hours per Session","[c1.m1] Average Wall Hours per Session","[c2.m8] Average Wall Hours per Session"
-"2018 Q2",52.40465278,35.26490741,24.49683934,1.10615741,0.23583333
+"2018 Q2",52.40465278,35.26490741,23.19631766,1.10337963,0.23583333
 ---------
diff --git a/.../reference/Cloud/configuration/cloud_avg_wallduration_hours/timeseries-Year-reference.csv b/.../reference/Cloud/configuration/cloud_avg_wallduration_hours/timeseries-Year-reference.csv
@@ -6,5 +6,5 @@ start,end
 2018-04-18,2018-04-30
 ---------
 Year,"[c2.m4] Average Wall Hours per Session","[c4.m16] Average Wall Hours per Session","[c1.m4] Average Wall Hours per Session","[c1.m1] Average Wall Hours per Session","[c2.m8] Average Wall Hours per Session"
-2018,52.40465278,35.26490741,24.49683934,1.10615741,0.23583333
+2018,52.40465278,35.26490741,23.19631766,1.10337963,0.23583333
 ---------
diff --git a/...rrent/expected/reference/Cloud/configuration/cloud_core_time/timeseries-Day-reference.csv b/...rrent/expected/reference/Cloud/configuration/cloud_core_time/timeseries-Day-reference.csv
@@ -7,7 +7,7 @@ start,end
 ---------
 Day,"[c1.m4] Core Hours: Total","[c4.m16] Core Hours: Total","[c2.m4] Core Hours: Total","[c1.m1] Core Hours: Total","[c2.m8] Core Hours: Total"
 2018-04-18,32.5156,17.7089,1.4167,0.0156,0.4717
-2018-04-19,107.3258,0,31.5878,0.9503,0
+2018-04-19,105.5992,0,31.5878,0.9336,0
 2018-04-20,49.0686,0,48.0000,0,0
 2018-04-21,72.0000,0,48.0000,0,0
 2018-04-22,72.0000,0,48.0000,0,0

diff --git a/...pected/reference/Cloud/configuration/cloud_num_sessions_ended/aggregate-Day-reference.csv b/...pected/reference/Cloud/configuration/cloud_num_sessions_ended/aggregate-Day-reference.csv
@@ -6,7 +6,7 @@ start,end
 2018-04-18,2018-04-30
 ---------
 "Instance Type","Number of Sessions Ended"
-c1.m4,37
+c1.m4,39
 c1.m1,6
 c2.m4,4
 c4.m16,3

diff --git a/...cted/reference/Cloud/configuration/cloud_num_sessions_ended/aggregate-Month-reference.csv b/...cted/reference/Cloud/configuration/cloud_num_sessions_ended/aggregate-Month-reference.csv
@@ -6,7 +6,7 @@ start,end
 2018-04-18,2018-04-30
 ---------
 "Instance Type","Number of Sessions Ended"
-c1.m4,37
+c1.m4,39
 c1.m1,6
 c2.m4,4
 c4.m16,3

diff --git a/...ed/reference/Cloud/configuration/cloud_num_sessions_ended/aggregate-Quarter-reference.csv b/...ed/reference/Cloud/configuration/cloud_num_sessions_ended/aggregate-Quarter-reference.csv
@@ -6,7 +6,7 @@ start,end
 2018-04-18,2018-04-30
 ---------
 "Instance Type","Number of Sessions Ended"
-c1.m4,37
+c1.m4,39
 c1.m1,6
 c2.m4,4
 c4.m16,3

diff --git a/...ected/reference/Cloud/configuration/cloud_num_sessions_ended/aggregate-Year-reference.csv b/...ected/reference/Cloud/configuration/cloud_num_sessions_ended/aggregate-Year-reference.csv
@@ -6,7 +6,7 @@ start,end
 2018-04-18,2018-04-30
 ---------
 "Instance Type","Number of Sessions Ended"
-c1.m4,37
+c1.m4,39
 c1.m1,6
 c2.m4,4
 c4.m16,3

diff --git a/...ected/reference/Cloud/configuration/cloud_num_sessions_ended/timeseries-Day-reference.csv b/...ected/reference/Cloud/configuration/cloud_num_sessions_ended/timeseries-Day-reference.csv
@@ -7,7 +7,7 @@ start,end
 ---------
 Day,"[c1.m4] Number of Sessions Ended","[c1.m1] Number of Sessions Ended","[c2.m4] Number of Sessions Ended","[c4.m16] Number of Sessions Ended","[c2.m8] Number of Sessions Ended"
 2018-04-18,10,1,1,2,1
-2018-04-19,16,4,1,0,0
+2018-04-19,18,4,1,0,0
 2018-04-20,7,0,0,0,0
 2018-04-21,0,0,0,0,0
 2018-04-22,0,0,0,0,0

diff --git a/...ted/reference/Cloud/configuration/cloud_num_sessions_ended/timeseries-Month-reference.csv b/...ted/reference/Cloud/configuration/cloud_num_sessions_ended/timeseries-Month-reference.csv
@@ -6,5 +6,5 @@ start,end
 2018-04-18,2018-04-30
 ---------
 Month,"[c1.m4] Number of Sessions Ended","[c1.m1] Number of Sessions Ended","[c2.m4] Number of Sessions Ended","[c4.m16] Number of Sessions Ended","[c2.m8] Number of Sessions Ended"
-2018-04,37,6,4,3,1
+2018-04,39,6,4,3,1
 ---------
diff --git a/...d/reference/Cloud/configuration/cloud_num_sessions_ended/timeseries-Quarter-reference.csv b/...d/reference/Cloud/configuration/cloud_num_sessions_ended/timeseries-Quarter-reference.csv
@@ -6,5 +6,5 @@ start,end
 2018-04-18,2018-04-30
 ---------
 Quarter,"[c1.m4] Number of Sessions Ended","[c1.m1] Number of Sessions Ended","[c2.m4] Number of Sessions Ended","[c4.m16] Number of Sessions Ended","[c2.m8] Number of Sessions Ended"
-"2018 Q2",37,6,4,3,1
+"2018 Q2",39,6,4,3,1
 ---------
diff --git a/...cted/reference/Cloud/configuration/cloud_num_sessions_ended/timeseries-Year-reference.csv b/...cted/reference/Cloud/configuration/cloud_num_sessions_ended/timeseries-Year-reference.csv
@@ -6,5 +6,5 @@ start,end
 2018-04-18,2018-04-30
 ---------
 Year,"[c1.m4] Number of Sessions Ended","[c1.m1] Number of Sessions Ended","[c2.m4] Number of Sessions Ended","[c4.m16] Number of Sessions Ended","[c2.m8] Number of Sessions Ended"
-2018,37,6,4,3,1
+2018,39,6,4,3,1
 ---------