No more truncation of aggregate tables

classes/ETL/Ingestor/CloudStateReconstructorTransformIngestor.php

@@ -132,8 +132,9 @@ protected function getSourceQueryString()
         // is lost. To work around this we add a dummy row filled with zeroes.
         $colCount = count($this->etlSourceQuery->records);
         $unionValues = array_fill(0, $colCount, 0);
+        $subSelect = "(SELECT DISTINCT instance_id from modw_cloud.event WHERE last_modified > \"" . $this->_last_modified . "\")";
 
-        $sql = "$sql WHERE event_type_id IN (" . implode(',', $this->_all_event_ids) . ")\nUNION ALL\nSELECT " . implode(',', $unionValues) . "\nORDER BY 1 DESC, 2 DESC, 3 ASC, 4 DESC";
+        $sql = "$sql WHERE instance_id IN " . $subSelect . " AND event_type_id IN (" . implode(',', $this->_all_event_ids) . ")\nUNION ALL\nSELECT " . implode(',', $unionValues) . "\nORDER BY 1 DESC, 2 DESC, 3 ASC, 4 DESC";
 
         return $sql;
     }

configuration/etl/etl.d/cloud_state_machine.json

@@ -28,14 +28,14 @@
             "name": "cloud-state-action",
             "class": "CloudStateReconstructorTransformIngestor",
             "definition_file": "cloud_common/cloud_state.json",
-            "truncate_destination": true,
+            "truncate_destination": false,
             "description": "Reconstructs discrete runs from cloud events"
         },
         {
             "name": "cloud-transient",
             "class": "DatabaseIngestor",
             "definition_file": "cloud_common/cloud_transient.json",
-            "truncate_destination": true,
+            "truncate_destination": false,
             "description": "Builds intermediate cloud event table"
         },
         {
@@ -46,7 +46,7 @@
             "class": "SimpleAggregator",
             "description": "Aggregate cloud records.",
             "definition_file": "cloud_common/cloud_metrics_aggregation.json",
-            "truncate_destination": true,
+            "truncate_destination": false,
             "table_prefix": "cloudfact_by_",
             "aggregation_units": [
                 "day", "month", "quarter", "year"

configuration/etl/etl_action_defs.d/cloud_common/cloud_metrics_aggregation.json

@@ -5,8 +5,8 @@
         },
         "aggregation_period_query": {
             "overseer_restrictions": {
-                "#last_modified_start_date": "last_modified >= ${VALUE}",
-                "#last_modified_end_date": "last_modified <= ${VALUE}",
+                "last_modified_start_date": "last_modified >= ${VALUE}",
+                "last_modified_end_date": "last_modified <= ${VALUE}",
                 "include_only_resource_codes": "resource_id IN ${VALUE}",
                 "exclude_resource_codes": "resource_id NOT IN ${VALUE}"
             }

configuration/etl/etl_action_defs.d/cloud_common/cloud_transient.json

@@ -23,7 +23,8 @@
             "wallduration": "FLOOR(e.end_time_ts) - FLOOR(e.start_time_ts)",
             "person_id": "ev.person_id",
             "systemaccount_id": "ev.systemaccount_id",
-            "submission_venue_id": "ev.submission_venue_id"
+            "submission_venue_id": "ev.submission_venue_id",
+            "last_modified": "e.last_modified"
         },
         "joins": [
             {

configuration/etl/etl_tables.d/cloud_common/cloud_transient.json

@@ -103,6 +103,13 @@
                 "name": "submission_venue_id",
                 "type": "int(5)",
                 "nullable": true
+            },
+            {
+                "name": "last_modified",
+                "type": "timestamp",
+                "nullable": false,
+                "default": "CURRENT_TIMESTAMP",
+                "extra": "on update CURRENT_TIMESTAMP"
             }
         ],
         "indexes": [

configuration/etl/etl_tables.d/cloud_common/event.json

@@ -68,6 +68,13 @@
                 "type": "int(5)",
                 "nullable": false,
                 "default": -1
+            },
+            {
+                "name": "last_modified",
+                "type": "timestamp",
+                "nullable": false,
+                "default": "CURRENT_TIMESTAMP",
+                "extra": "on update CURRENT_TIMESTAMP"
             }
         ],
         "indexes": [

configuration/etl/etl_tables.d/cloud_common/event_reconstructed.json

@@ -32,6 +32,13 @@
                 "name": "end_event_id",
                 "type": "int(11)",
                 "nullable": false
+            },
+            {
+                "name": "last_modified",
+                "type": "timestamp",
+                "nullable": false,
+                "default": "CURRENT_TIMESTAMP",
+                "extra": "on update CURRENT_TIMESTAMP"
             }
         ]
     }

open_xdmod/modules/xdmod/integration_tests/scripts/post_upgrade_ingest.sh

@@ -0,0 +1,37 @@
+#!/bin/bash
+# Tests that cloud ETL works after upgrade + reingestion
+
+BASEDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+REF_SOURCE=`realpath $BASEDIR/../../tests/artifacts/xdmod-test-artifacts/xdmod/referencedata`
+REF_DIR=/var/tmp/referencedata
+
+sudo -u xdmod xdmod-shredder -r openstack -d $REF_DIR/openstack_upgrade -f openstack
+sudo -u xdmod xdmod-ingestor
+
+phpunit="$(readlink -f ../../../../vendor/bin/phpunit)"
+
+if [ "$1" = "--junit-output-dir" ];
+then
+    REGUSER="--log-junit $2/xdmod-regression-user.xml"
+    PI="--log-junit $2/xdmod-regression-principalinvestigator.xml"
+    CD="--log-junit $2/xdmod-regression-centerdirector.xml"
+    CS="--log-junit $2/xdmod-regression-centerstaff.xml"
+    PUB="--log-junit $2/xdmod-regression-public.xml"
+fi
+
+REG_TEST_USER_ROLE=usr $phpunit $REGUSER lib/Controllers/UsageExplorerCloudTest.php & usrpid=$!
+REG_TEST_USER_ROLE=pi $phpunit $PI lib/Controllers/UsageExplorerCloudTest.php & pipid=$!
+REG_TEST_USER_ROLE=cd $phpunit $CD lib/Controllers/UsageExplorerCloudTest.php & cdpid=$!
+REG_TEST_USER_ROLE=cs $phpunit $CS lib/Controllers/UsageExplorerCloudTest.php & cspid=$!
+$phpunit $PUB lib/Controllers/UsageExplorerCloudTest.php & pubpid=$!
+
+EXIT_STATUS=0
+for pid in $usrpid $pipid $cdpid $cspid $pubpid;
+do
+    wait "$pid"
+    if [ "$?" -ne "0" ];
+    then
+        EXIT_STATUS=1
+    fi
+done
+exit $EXIT_STATUS

open_xdmod/modules/xdmod/regression_tests/lib/Controllers/UsageExplorerCloudPostIngestTest.php

@@ -0,0 +1,32 @@
+<?php
+
+namespace Controllers;
+
+class UsageExplorerCloudPostIngestTest extends UsageExplorerTest
+{
+
+    public function csvExportProvider(){
+        parent::$baseDir = __DIR__ . '/../../../tests/artifacts/xdmod-test-artifacts/xdmod/post_ingest/';
+        parent::defaultSetup();
+
+        $statistics = array(
+            'cloud_num_sessions_ended',
+            'cloud_num_sessions_running',
+            'cloud_num_sessions_started',
+        );
+
+        $group_bys = array(
+            'none',
+        );
+
+        $varSettings = array(
+            'realm' => array('Cloud'),
+            'dataset_type' => array('aggregate', 'timeseries'),
+            'statistic' => $statistics,
+            'group_by' => $group_bys,
+            'aggregation_unit' => array_keys($this->aggregationUnits)
+        );
+
+        return parent::generateTests($varSettings, '2018-06-01', '2018-06-07');
+    }
+}

shippable.yml

@@ -31,6 +31,7 @@ build:
         - ./open_xdmod/modules/xdmod/automated_tests/runtests.sh --headless --log-junit `pwd`/shippable/testresults --sso
         - ./vendor/phpunit/phpunit/phpunit -c ./open_xdmod/modules/xdmod/integration_tests/phpunit.xml.dist --testsuite sso --log-junit `pwd`/shippable/testresults/xdmod-sso-integration.xml
         - mv ./configuration/portal_settings.ini.old ./configuration/portal_settings.ini
+        - ./open_xdmod/modules/xdmod/integration_tests/scripts/post_upgrade_ingest.sh
         - if [ -s /var/log/xdmod/apache-error.log ]; then cat /var/log/xdmod/apache-error.log; false; fi
     on_failure:
         - cat /var/log/xdmod/*

tests/artifacts/xdmod-test-artifacts/xdmod/post_ingest/none/cloud_num_sessions_ended/aggregate-Day-reference.csv

@@ -0,0 +1,10 @@
+title
+"Number of Sessions Ended"
+parameters
+
+start,end
+2018-06-01,2018-06-07
+---------
+Summary,"Number of Sessions Ended"
+Screwdriver,23
+---------

tests/artifacts/xdmod-test-artifacts/xdmod/post_ingest/none/cloud_num_sessions_ended/aggregate-Month-reference.csv

@@ -0,0 +1,10 @@
+title
+"Number of Sessions Ended"
+parameters
+
+start,end
+2018-06-01,2018-06-07
+---------
+Summary,"Number of Sessions Ended"
+Screwdriver,23
+---------

tests/artifacts/xdmod-test-artifacts/xdmod/post_ingest/none/cloud_num_sessions_ended/aggregate-Quarter-reference.csv

@@ -0,0 +1,10 @@
+title
+"Number of Sessions Ended"
+parameters
+
+start,end
+2018-06-01,2018-06-07
+---------
+Summary,"Number of Sessions Ended"
+Screwdriver,23
+---------

tests/artifacts/xdmod-test-artifacts/xdmod/post_ingest/none/cloud_num_sessions_ended/aggregate-Year-reference.csv

@@ -0,0 +1,10 @@
+title
+"Number of Sessions Ended"
+parameters
+
+start,end
+2018-06-01,2018-06-07
+---------
+Summary,"Number of Sessions Ended"
+Screwdriver,23
+---------

tests/artifacts/xdmod-test-artifacts/xdmod/post_ingest/none/cloud_num_sessions_ended/timeseries-Day-reference.csv

@@ -0,0 +1,16 @@
+title
+"Number of Sessions Ended"
+parameters
+
+start,end
+2018-06-01,2018-06-07
+---------
+Day,"[Screwdriver] Number of Sessions Ended"
+2018-06-01,0
+2018-06-02,0
+2018-06-03,0
+2018-06-04,0
+2018-06-05,3
+2018-06-06,0
+2018-06-07,20
+---------

tests/artifacts/xdmod-test-artifacts/xdmod/post_ingest/none/cloud_num_sessions_ended/timeseries-Month-reference.csv

@@ -0,0 +1,10 @@
+title
+"Number of Sessions Ended"
+parameters
+
+start,end
+2018-06-01,2018-06-07
+---------
+Month,"[Screwdriver] Number of Sessions Ended"
+2018-04,23
+---------

tests/artifacts/xdmod-test-artifacts/xdmod/post_ingest/none/cloud_num_sessions_ended/timeseries-Quarter-reference.csv

@@ -0,0 +1,10 @@
+title
+"Number of Sessions Ended"
+parameters
+
+start,end
+2018-06-01,2018-06-07
+---------
+Quarter,"[Screwdriver] Number of Sessions Ended"
+"2018 Q2",23
+---------

tests/artifacts/xdmod-test-artifacts/xdmod/post_ingest/none/cloud_num_sessions_ended/timeseries-Year-reference.csv

@@ -0,0 +1,10 @@
+title
+"Number of Sessions Ended"
+parameters
+
+start,end
+2018-06-01,2018-06-07
+---------
+Year,"[Screwdriver] Number of Sessions Ended"
+2018,23
+---------

tests/artifacts/xdmod-test-artifacts/xdmod/post_ingest/none/cloud_num_sessions_running/aggregate-Day-reference.csv

@@ -0,0 +1,10 @@
+title
+"Number of Active Sessions"
+parameters
+
+start,end
+2018-06-01,2018-06-07
+---------
+Summary,"Number of Active Sessions (Number of Sessions)"
+Screwdriver,23
+---------

tests/artifacts/xdmod-test-artifacts/xdmod/post_ingest/none/cloud_num_sessions_running/aggregate-Month-reference.csv

@@ -0,0 +1,10 @@
+title
+"Number of Active Sessions"
+parameters
+
+start,end
+2018-06-01,2018-06-07
+---------
+Summary,"Number of Active Sessions (Number of Sessions)"
+Screwdriver,23
+---------

tests/artifacts/xdmod-test-artifacts/xdmod/post_ingest/none/cloud_num_sessions_running/aggregate-Quarter-reference.csv

@@ -0,0 +1,10 @@
+title
+"Number of Active Sessions"
+parameters
+
+start,end
+2018-06-01,2018-06-07
+---------
+Summary,"Number of Active Sessions (Number of Sessions)"
+Screwdriver,23
+---------

tests/artifacts/xdmod-test-artifacts/xdmod/post_ingest/none/cloud_num_sessions_running/aggregate-Year-reference.csv

@@ -0,0 +1,10 @@
+title
+"Number of Active Sessions"
+parameters
+
+start,end
+2018-06-01,2018-06-07
+---------
+Summary,"Number of Active Sessions (Number of Sessions)"
+Screwdriver,23
+---------

tests/artifacts/xdmod-test-artifacts/xdmod/post_ingest/none/cloud_num_sessions_running/timeseries-Day-reference.csv

@@ -0,0 +1,16 @@
+title
+"Number of Active Sessions"
+parameters
+
+start,end
+2018-06-01,2018-06-07
+---------
+Day,"[Screwdriver] Number of Active Sessions (Number of Sessions)"
+2018-06-01,17
+2018-06-02,17
+2018-06-03,17
+2018-06-04,17
+2018-06-05,23
+2018-06-06,20
+2018-06-07,20
+---------

tests/artifacts/xdmod-test-artifacts/xdmod/post_ingest/none/cloud_num_sessions_running/timeseries-Month-reference.csv

@@ -0,0 +1,10 @@
+title
+"Number of Active Sessions"
+parameters
+
+start,end
+2018-06-01,2018-06-07
+---------
+Month,"[Screwdriver] Number of Active Sessions (Number of Sessions)"
+2018-04,23
+---------

tests/artifacts/xdmod-test-artifacts/xdmod/post_ingest/none/cloud_num_sessions_running/timeseries-Quarter-reference.csv

@@ -0,0 +1,10 @@
+title
+"Number of Active Sessions"
+parameters
+
+start,end
+2018-06-01,2018-06-07
+---------
+Quarter,"[Screwdriver] Number of Active Sessions (Number of Sessions)"
+"2018 Q2",23
+---------

tests/artifacts/xdmod-test-artifacts/xdmod/post_ingest/none/cloud_num_sessions_running/timeseries-Year-reference.csv

@@ -0,0 +1,10 @@
+title
+"Number of Active Sessions"
+parameters
+
+start,end
+2018-06-01,2018-06-07
+---------
+Year,"[Screwdriver] Number of Active Sessions (Number of Sessions)"
+2018,23
+---------

tests/artifacts/xdmod-test-artifacts/xdmod/post_ingest/none/cloud_num_sessions_started/aggregate-Day-reference.csv

@@ -0,0 +1,10 @@
+title
+"Number of Sessions Started"
+parameters
+
+start,end
+2018-06-01,2018-06-07
+---------
+Summary,"Number of Sessions Started"
+Screwdriver,23
+---------

tests/artifacts/xdmod-test-artifacts/xdmod/post_ingest/none/cloud_num_sessions_started/aggregate-Month-reference.csv

@@ -0,0 +1,10 @@
+title
+"Number of Sessions Started"
+parameters
+
+start,end
+2018-06-01,2018-06-07
+---------
+Summary,"Number of Sessions Started"
+Screwdriver,23
+---------

tests/artifacts/xdmod-test-artifacts/xdmod/post_ingest/none/cloud_num_sessions_started/aggregate-Quarter-reference.csv

@@ -0,0 +1,10 @@
+title
+"Number of Sessions Started"
+parameters
+
+start,end
+2018-06-01,2018-06-07
+---------
+Summary,"Number of Sessions Started"
+Screwdriver,23
+---------