Merge pull request #1956 from eiffel777/add-pi-resource-organization-id

Adding resource_organization_id, person_organization_id, piperson_organization_id to cloudfact tables
ubccr · Dec 10, 2024 · 4c30a7a · 4c30a7a
2 parents 0471b2c + 659a6fc
commit 4c30a7a
Show file tree

Hide file tree

Showing 6 changed files with 60 additions and 0 deletions.
diff --git a/configuration/etl/etl_action_defs.d/cloud_common/cloud_metrics_aggregation.json b/configuration/etl/etl_action_defs.d/cloud_common/cloud_metrics_aggregation.json
@@ -29,6 +29,7 @@
                 "host_resource_id": "sr.resource_id",
                 "account_id": "sr.account_id",
                 "person_id": "sr.person_id",
+                "person_organization_id": "sr.person_organization_id",
                 "systemaccount_id": "sr.systemaccount_id",
                 "processorbucket_id": "sr.processorbucket_id",
                 "memorybucket_id": "sr.memorybucket_id",
@@ -47,6 +48,7 @@
                 "domain_id": "sr.domain_id",
                 "service_provider": "sr.service_provider",
                 "principalinvestigator_person_id": "sr.principalinvestigator_person_id",
+                "piperson_organization_id": "sr.piperson_organization_id",
                 "fos_id": "sr.fos_id",
                 "instance_state_id": "sr.instance_state_id"
             },
@@ -65,8 +67,10 @@
               "domain_id",
               "service_provider",
               "principalinvestigator_person_id",
+              "piperson_organization_id",
               "fos_id",
               "person_id",
+              "person_organization_id",
               "instance_state_id"
             ],
             "joins":[

diff --git a/configuration/etl/etl_action_defs.d/cloud_common/cloud_metrics_aggregation_by_day.json b/configuration/etl/etl_action_defs.d/cloud_common/cloud_metrics_aggregation_by_day.json
@@ -29,6 +29,7 @@
                 "host_resource_id": "sr.resource_id",
                 "account_id": "sr.account_id",
                 "person_id": "sr.person_id",
+                "person_organization_id": "sr.person_organization_id",
                 "systemaccount_id": "sr.systemaccount_id",
                 "processorbucket_id": "sr.processorbucket_id",
                 "memorybucket_id": "sr.memorybucket_id",
@@ -47,6 +48,7 @@
                 "domain_id": "sr.domain_id",
                 "service_provider": "sr.service_provider",
                 "principalinvestigator_person_id": "sr.principalinvestigator_person_id",
+                "piperson_organization_id": "sr.piperson_organization_id",
                 "fos_id": "sr.fos_id",
                 "instance_state_id": "sr.instance_state_id",
                 "session_id_list": "GROUP_CONCAT(sr.session_id)"
@@ -66,8 +68,10 @@
               "domain_id",
               "service_provider",
               "principalinvestigator_person_id",
+              "piperson_organization_id",
               "fos_id",
               "person_id",
+              "person_organization_id",
               "instance_state_id"
             ],
             "joins":[

diff --git a/configuration/etl/etl_action_defs.d/cloud_common/session_records.json b/configuration/etl/etl_action_defs.d/cloud_common/session_records.json
@@ -23,12 +23,14 @@
             "end_day_id": "YEAR(FROM_UNIXTIME(e.end_time_ts)) * 100000 + DAYOFYEAR(FROM_UNIXTIME(e.end_time_ts))",
             "wallduration": "FLOOR(e.end_time_ts) - FLOOR(e.start_time_ts)",
             "person_id": "ev.person_id",
+            "person_organization_id": "COALESCE(p.organization_id, -1)",
             "systemaccount_id": "ev.systemaccount_id",
             "submission_venue_id": "ev.submission_venue_id",
             "domain_id": "ev.domain_id",
             "service_provider": "ev.service_provider",
             "account_id": "a.account_id",
             "principalinvestigator_person_id": "a.principalinvestigator_person_id",
+            "piperson_organization_id": "COALESCE(pi.organization_id, -1)",
             "fos_id": "a.fos_id",
             "host_id": "ev.host_id",
             "instance_state_id": "etype.instance_state_id"
@@ -75,6 +77,20 @@
                 "schema": "${SOURCE_SCHEMA}",
                 "alias": "etype",
                 "on": "e.start_event_id = etype.event_type_id"
+            },
+            {
+                "name": "person",
+                "schema": "${UTILITY_SCHEMA}",
+                "alias": "p",
+                "type": "LEFT",
+                "on": "ev.person_id = p.id"
+            },
+            {
+                "name": "person",
+                "schema": "${UTILITY_SCHEMA}",
+                "alias": "pi",
+                "type": "LEFT",
+                "on": "a.principalinvestigator_person_id = pi.id"
             }
         ],
         "orderby": [

diff --git a/configuration/etl/etl_tables.d/cloud_common/cloudfact_by_.json b/configuration/etl/etl_tables.d/cloud_common/cloudfact_by_.json
@@ -35,6 +35,12 @@
                 "type": "int(11)",
                 "nullable": false,
                 "comment": "DIMENSION: The person id associated with a VM instance."
+            },{
+                "name": "person_organization_id",
+                "type": "int(11)",
+                "nullable": false,
+                "default": -1,
+                "comment": "DIMENSION: The organization of the person to whom the VM is assigned."
             },{
                 "name": "systemaccount_id",
                 "type": "int(11)",
@@ -127,6 +133,12 @@
                 "nullable": false,
                 "default": -1,
                 "comment": "DIMENSION: The PI that owns the allocations that these VM's ran under. References principalinvestigator.person_id"
+            },{
+                "name": "piperson_organization_id",
+                "type": "int(11)",
+                "nullable": false,
+                "default": -1,
+                "comment": "DIMENSION: The organization of the PI that owns the project that funds these VM's. References piperson.organization_id"
             },{
                 "name": "fos_id",
                 "type": "int(11)",

diff --git a/configuration/etl/etl_tables.d/cloud_common/cloudfact_by_day.json b/configuration/etl/etl_tables.d/cloud_common/cloudfact_by_day.json
@@ -40,6 +40,12 @@
                 "type": "int(11)",
                 "nullable": false,
                 "comment": "DIMENSION: The person id associated with a VM instance."
+            },{
+                "name": "person_organization_id",
+                "type": "int(11)",
+                "nullable": false,
+                "default": -1,
+                "comment": "DIMENSION: The organization of the person that ran the jobs."
             },{
                 "name": "systemaccount_id",
                 "type": "int(11)",
@@ -132,6 +138,12 @@
                 "nullable": false,
                 "default": -1,
                 "comment": "DIMENSION: The PI that owns the allocations that these VM's ran under. References principalinvestigator.person_id"
+            },{
+                "name": "piperson_organization_id",
+                "type": "int(11)",
+                "nullable": false,
+                "default": -1,
+                "comment": "DIMENSION: The organization of the PI that owns the project that funds these VMs. References piperson.organization_id"
             },{
                 "name": "fos_id",
                 "type": "int(11)",

diff --git a/configuration/etl/etl_tables.d/cloud_common/session_records.json b/configuration/etl/etl_tables.d/cloud_common/session_records.json
@@ -107,6 +107,12 @@
                 "type": "int(11)",
                 "nullable": true
             },
+            {
+                "name": "person_organization_id",
+                "type": "int(11)",
+                "nullable": false,
+                "default": -1
+            },
             {
                 "name": "systemaccount_id",
                 "type": "int(11)",
@@ -146,6 +152,12 @@
                 "default": -1,
                 "comment": "DIMENSION: The PI that owns the allocations that these VM's ran under. References principalinvestigator.person_id"
             },
+            {
+                "name": "piperson_organization_id",
+                "type": "int(11)",
+                "nullable": false,
+                "default": -1
+            },
             {
                 "name": "fos_id",
                 "type": "int(11)",