Update cloud realm to not throw away event precision (ubccr#811)

Update cloud realm to take the full date from openstack. This allows events to be imported properly and in the correct order so that the statistics are accurate.
jtpalmer · Feb 25, 2019 · ee9c5f2 · ee9c5f2
1 parent eec4f8c
commit ee9c5f2
Show file tree

Hide file tree

Showing 21 changed files with 171 additions and 134 deletions.
diff --git a/classes/OpenXdmod/Migration/Version800To810/DatabasesMigration.php b/classes/OpenXdmod/Migration/Version800To810/DatabasesMigration.php
@@ -52,19 +52,69 @@ public function execute()
         $hpcdbDb = DB::factory('hpcdb');
         $dwDb = DB::factory('datawarehouse');
         $dwi = new DataWarehouseInitializer($hpcdbDb, $dwDb);
-
+        $db = DB::factory('database');
         if($dwi->isRealmEnabled('Cloud')){
+
             $console = Console::factory();
             $console->displayMessage(<<<"EOT"
-There have been updates to cloud aggregation statistics to make the data more accurate.
-If you have the Cloud realm enabled it is recommended that you re-ingest and aggregate
-your cloud data using the commands recommended in our documentation.
+
+!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+WARNING WARNING WARNING WARNING WARNING WARNING WARNING WARNING WARNING
+!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+
+There have been updates to cloud data the current data is INVALID.
+If you need the current data back it up NOW.
+After you press enter it will be removed.
+Canceling the upgrade process now will break XDMoD.
+
+After the upgrade is complete re-ingest and aggregate your cloud data using the
+commands recommended in our documentation.
+
+!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+WARNING WARNING WARNING WARNING WARNING WARNING WARNING WARNING WARNING
+!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+
 EOT
             );
+            $console->prompt("Press ENTER to continue.");
+
+            $db->execute('
+                DROP TABLE IF EXISTS `modw_cloud`.`account`;
+                DROP TABLE IF EXISTS `modw_cloud`.`asset`;
+                DROP TABLE IF EXISTS `modw_cloud`.`asset_type`;
+                DROP TABLE IF EXISTS `modw_cloud`.`avail_zone`;
+                DROP TABLE IF EXISTS `modw_cloud`.`cloud_events_transient`;
+                DROP TABLE IF EXISTS `modw_cloud`.`cloud_resource_metadata`;
+                DROP TABLE IF EXISTS `modw_cloud`.`cloudfact_by_day`;
+                DROP TABLE IF EXISTS `modw_cloud`.`cloudfact_by_month`;
+                DROP TABLE IF EXISTS `modw_cloud`.`cloudfact_by_quarter`;
+                DROP TABLE IF EXISTS `modw_cloud`.`cloudfact_by_year`;
+                DROP TABLE IF EXISTS `modw_cloud`.`event`;
+                DROP TABLE IF EXISTS `modw_cloud`.`event_asset`;
+                DROP TABLE IF EXISTS `modw_cloud`.`event_reconstructed`;
+                DROP TABLE IF EXISTS `modw_cloud`.`event_type`;
+                DROP TABLE IF EXISTS `modw_cloud`.`generic_cloud_raw_event`;
+                DROP TABLE IF EXISTS `modw_cloud`.`generic_cloud_raw_instance_type`;
+                DROP TABLE IF EXISTS `modw_cloud`.`generic_cloud_raw_volume`;
+                DROP TABLE IF EXISTS `modw_cloud`.`generic_cloud_staging_event`;
+                DROP TABLE IF EXISTS `modw_cloud`.`host`;
+                DROP TABLE IF EXISTS `modw_cloud`.`image`;
+                DROP TABLE IF EXISTS `modw_cloud`.`instance`;
+                DROP TABLE IF EXISTS `modw_cloud`.`instance_data`;
+                DROP TABLE IF EXISTS `modw_cloud`.`instance_type`;
+                DROP TABLE IF EXISTS `modw_cloud`.`job_record_event`;
+                DROP TABLE IF EXISTS `modw_cloud`.`memory_buckets`;
+                DROP TABLE IF EXISTS `modw_cloud`.`openstack_event_map`;
+                DROP TABLE IF EXISTS `modw_cloud`.`openstack_raw_event`;
+                DROP TABLE IF EXISTS `modw_cloud`.`openstack_raw_instance_type`;
+                DROP TABLE IF EXISTS `modw_cloud`.`openstack_staging_event`;
+                DROP TABLE IF EXISTS `modw_cloud`.`processor_buckets`;
+                DROP TABLE IF EXISTS `modw_cloud`.`record_type`;
+                DROP TABLE IF EXISTS `modw_cloud`.`region`;
+                DROP TABLE IF EXISTS `modw_cloud`.`user`;
+            ');
         }
 
-        $db = DB::factory('database');
-
         $result = $db->query('SELECT id FROM Users');
         foreach ($result as $row)
         {

diff --git a/configuration/etl/etl_action_defs.d/cloud_common/cloud_transient.json b/configuration/etl/etl_action_defs.d/cloud_common/cloud_transient.json
@@ -5,9 +5,9 @@
     "source_query": {
         "records": {
             "instance_id": "e.instance_id",
-            "start_time": "e.start_time",
+            "start_time": "FROM_UNIXTIME(e.start_time)",
             "start_event_type_id": "e.start_event_id",
-            "end_time": "e.end_time",
+            "end_time": "FROM_UNIXTIME(e.end_time)",
             "end_event_type_id": "e.end_event_id",
             "resource_id": "it.resource_id",
             "instance": "it.provider_identifier",
@@ -16,11 +16,11 @@
             "num_cores": "itt.num_cores",
             "memory_mb": "itt.memory_mb",
             "disk_gb": "itt.disk_gb",
-            "start_time_ts": "UNIX_TIMESTAMP(e.start_time)",
-            "end_time_ts": "UNIX_TIMESTAMP(e.end_time)",
-            "start_day_id": "YEAR(e.start_time) * 100000 + DAYOFYEAR(e.start_time)",
-            "end_day_id": "YEAR(e.end_time) * 100000 + DAYOFYEAR(e.end_time)",
-            "wallduration": "UNIX_TIMESTAMP(e.end_time) - UNIX_TIMESTAMP(e.start_time)",
+            "start_time_ts": "FLOOR(e.start_time)",
+            "end_time_ts": "FLOOR(e.end_time)",
+            "start_day_id": "YEAR(FROM_UNIXTIME(e.start_time)) * 100000 + DAYOFYEAR(FROM_UNIXTIME(e.start_time))",
+            "end_day_id": "YEAR(FROM_UNIXTIME(e.end_time)) * 100000 + DAYOFYEAR(FROM_UNIXTIME(e.end_time))",
+            "wallduration": "FLOOR(e.end_time) - FLOOR(e.start_time)",
             "submission_venue_id": "ev.submission_venue_id"
         },
         "joins": [

diff --git a/configuration/etl/etl_action_defs.d/cloud_generic/staging_event.json b/configuration/etl/etl_action_defs.d/cloud_generic/staging_event.json
@@ -18,7 +18,7 @@
     "source_query": {
         "records": {
             "resource_id": "raw.resource_id",
-            "event_time_utc": "raw.event_time_utc",
+            "event_time_utc": "UNIX_TIMESTAMP(CONVERT_TZ(raw.event_time_utc,'+00:00', @@session.time_zone))",
             "event_type_id": "COALESCE(etype.event_type_id, -1)",
             "record_type_id": "COALESCE(rtype.record_type_id, -1)",
             "account_id": "COALESCE(acct.account_id, -1)",

diff --git a/configuration/etl/etl_action_defs.d/cloud_generic/volume.json b/configuration/etl/etl_action_defs.d/cloud_generic/volume.json
@@ -11,7 +11,7 @@
             "asset_type_id": "atype.asset_type_id",
             "account_id": "act.account_id",
             "provider_identifier": "raw.provider_volume_identifier",
-            "create_time_utc": "raw.create_time_utc",
+            "create_time_utc": "UNIX_TIMESTAMP(CONVERT_TZ(raw.create_time_utc,'+00:00', @@session.time_zone))",
             "size": "raw.disk_gb"
         },
 

diff --git a/configuration/etl/etl_action_defs.d/cloud_openstack/raw_cloud_job_logs.json b/configuration/etl/etl_action_defs.d/cloud_openstack/raw_cloud_job_logs.json
@@ -1,6 +1,6 @@
 {
-    "#": "Load raw event data from OpenStack log files. The resource id will be specified on",
-    "#": "the ETL command line at run time.",
+    "#": "Load raw event data from OpenStack log files. The resource id will be",
+    "#": "specified on the ETL command line at run time.",
 
     "table_definition": [
         {
@@ -18,19 +18,19 @@
             "event_type": "event_type",
             "record_type": "record_type",
             "event_time_utc": "generated",
-    	      "create_time_utc": "created_at",
+            "create_time_utc": "created_at",
             "provider_instance_identifier": "instance_id",
             "hostname": "host",
             "user_name": "user_name",
             "user_id": "user_id",
             "resource_id": "${RESOURCE_ID}",
             "instance_type": "instance_type",
             "provider_account": "user_name",
-    	      "project_name": "project_name",
-    	      "request_id": "request_id",
-    	      "project_id": "project_id",
-    	      "openstack_resource_id": "resource_id",
-    	      "disk_gb": "disk_gb",
+            "project_name": "project_name",
+            "request_id": "request_id",
+            "project_id": "project_id",
+            "openstack_resource_id": "resource_id",
+            "disk_gb": "disk_gb",
             "size": "size",
             "volume_id": "volume_id",
             "state": "state"

diff --git a/configuration/etl/etl_action_defs.d/cloud_openstack/staging_event.json b/configuration/etl/etl_action_defs.d/cloud_openstack/staging_event.json
@@ -13,7 +13,7 @@
     "source_query": {
         "records": {
             "resource_id": "raw.resource_id",
-            "event_time_utc": "raw.event_time_utc",
+            "event_time_utc": "UNIX_TIMESTAMP(CONVERT_TZ(raw.event_time_utc,'+00:00', @@session.time_zone))",
             "event_type_id": "COALESCE(etype.event_type_id, -1)",
             "record_type_id": "COALESCE(rtype.record_type_id, -1)",
             "account_id": "COALESCE(acct.account_id, -1)",

diff --git a/configuration/etl/etl_action_defs.d/cloud_openstack/volume.json b/configuration/etl/etl_action_defs.d/cloud_openstack/volume.json
@@ -12,7 +12,7 @@
             "resource_id": "raw.resource_id",
             "account_id": "act.account_id",
             "provider_identifier": "raw.openstack_resource_id",
-            "create_time_utc": "raw.event_time_utc",
+            "create_time_utc": "UNIX_TIMESTAMP(CONVERT_TZ(raw.event_time_utc,'+00:00', @@session.time_zone))",
             "size": "raw.size"
         },
 

diff --git a/configuration/etl/etl_tables.d/cloud_common/asset.json b/configuration/etl/etl_tables.d/cloud_common/asset.json
@@ -41,17 +41,17 @@
             },
             {
                 "name": "create_time_utc",
-                "type": "datetime",
+                "type": "decimal(16, 6)",
                 "nullable": false,
-                "default": "0000-00-00 00:00:00",
-                "comment": "The time that the asset was created in UTC."
+                "default": 0,
+                "comment": "The time that the asset was created as a unix timestamp to the microsecond."
             },
             {
                 "name": "destroy_time_utc",
-                "type": "datetime",
+                "type": "decimal(16, 6)",
                 "nullable": true,
                 "default": null,
-                "comment": "The time that the asset was destroyed in UTC."
+                "comment": "The time that the asset was destroyed as a unix timestamp to the microsecond."
             },
             {
                 "name": "size",

diff --git a/configuration/etl/etl_tables.d/cloud_common/event.json b/configuration/etl/etl_tables.d/cloud_common/event.json
@@ -27,9 +27,9 @@
             },
             {
                 "name": "event_time_utc",
-                "type": "datetime",
+                "type": "decimal(16, 6)",
                 "nullable": false,
-                "default": "0000-00-00 00:00:00"
+                "default": 0
             },
             {
                 "name": "event_type_id",

diff --git a/configuration/etl/etl_tables.d/cloud_common/event_reconstructed.json b/configuration/etl/etl_tables.d/cloud_common/event_reconstructed.json
@@ -15,7 +15,7 @@
             },
             {
                 "name": "start_time",
-                "type": "datetime",
+                "type": "decimal(16, 6)",
                 "nullable": false
             },
             {
@@ -25,7 +25,7 @@
             },
             {
                 "name": "end_time",
-                "type": "datetime",
+                "type": "decimal(16, 6)",
                 "nullable": false
             },
             {

diff --git a/configuration/etl/etl_tables.d/cloud_common/instance_type.json b/configuration/etl/etl_tables.d/cloud_common/instance_type.json
@@ -58,17 +58,17 @@
             },
             {
                 "name": "start_time",
-                "type": "datetime",
+                "type": "decimal(16, 6)",
                 "nullable": false,
-                "default": "0000-00-00 00:00:00",
-                "comment": "First time that this configuration was seen, defaults to unknown."
+                "default": 0,
+                "comment": "First time that this configuration was seen as a unix timestamp to the microsecond. defaults to unknown."
             },
             {
                 "name": "end_time",
-                "type": "datetime",
+                "type": "decimal(16, 6)",
                 "nullable": true,
                 "default": null,
-                "comment": "End time for this configuration, NULL if it is still in effect."
+                "comment": "End time for this configuration as a unix timestamp to the microsecond., NULL if it is still in effect."
             }
         ],
 

diff --git a/configuration/etl/etl_tables.d/cloud_generic/raw_event.json b/configuration/etl/etl_tables.d/cloud_generic/raw_event.json
@@ -24,10 +24,10 @@
             },
             {
                 "name": "event_time_utc",
-                "type": "datetime",
+                "type": "decimal(16, 6)",
                 "nullable": false,
-                "default": "0000-00-00 00:00:00",
-                "comment": "The time of the event in UTC."
+                "default": 0,
+                "comment": "The time of the event as a unix timestamp to the microsecond.."
             },
             {
                 "name": "event_type",

diff --git a/configuration/etl/etl_tables.d/cloud_generic/raw_instance_type.json b/configuration/etl/etl_tables.d/cloud_generic/raw_instance_type.json
@@ -54,10 +54,10 @@
             },
             {
                 "name": "start_time",
-                "type": "datetime",
+                "type": "decimal(16, 6)",
                 "nullable": true,
-                "default": "0000-00-00 00:00:00",
-                "comment": "First time this configuration was encountered, defaults to unknown."
+                "default": 0,
+                "comment": "First time this configuration was encountered as a unix timestamp to the microsecond., defaults to unknown."
             }
         ],
         "indexes": [

diff --git a/configuration/etl/etl_tables.d/cloud_generic/raw_volume.json b/configuration/etl/etl_tables.d/cloud_generic/raw_volume.json
@@ -34,24 +34,24 @@
             },
             {
                 "name": "event_time_utc",
-                "type": "datetime",
+                "type": "char(26)",
                 "nullable": false,
-                "default": "0000-00-00 00:00:00",
+                "default": "0000-00-00T00:00:00.000000",
                 "comment": "The time of the event in UTC."
             },
             {
                 "name": "attach_time_utc",
-                "type": "datetime",
+                "type": "char(26)",
                 "nullable": false,
-                "default": "0000-00-00 00:00:00",
-                "comment": "The time that the volume was attached to an instance in UTC."
+                "default": "0000-00-00T00:00:00.000000",
+                "comment": "The time of the event in UTC."
             },
             {
                 "name": "create_time_utc",
-                "type": "datetime",
+                "type": "char(26)",
                 "nullable": false,
-                "default": "0000-00-00 00:00:00",
-                "comment": "The time that the volume was created in UTC."
+                "default": "0000-00-00T00:00:00.000000",
+                "comment": "The time of the event in UTC."
             },
             {
                 "name": "provider_volume_identifier",

diff --git a/configuration/etl/etl_tables.d/cloud_generic/staging_event.json b/configuration/etl/etl_tables.d/cloud_generic/staging_event.json
@@ -18,10 +18,10 @@
             },
             {
                 "name": "event_time_utc",
-                "type": "datetime",
+                "type": "decimal(16, 6)",
                 "nullable": false,
-                "default": "0000-00-00 00:00:00",
-                "comment": "The time of the event in UTC."
+                "default": 0,
+                "comment": "The time of the event as a unix timestamp to the microsecond.."
             },
             {
                 "name": "event_type_id",