Add quality of service group by with Slurm support

classes/OpenXdmod/Shredder/Slurm.php

@@ -37,6 +37,7 @@ class Slurm extends Shredder
         'jobidraw',
         'cluster',
         'partition',
+        'qos',
         'account',
         'group',
         'gid',
@@ -70,6 +71,7 @@ class Slurm extends Shredder
         'job_id_raw',
         'cluster_name',
         'partition_name',
+        'qos_name',
         'account_name',
         'group_name',
         'gid_number',
@@ -111,6 +113,7 @@ class Slurm extends Shredder
         'job_name'        => 'job_name',
         'resource_name'   => 'cluster_name',
         'queue_name'      => 'partition_name',
+        'qos_name'        => 'qos_name',
         'user_name'       => 'user_name',
         'uid_number'      => 'uid_number',
         'group_name'      => 'group_name',

configuration/datawarehouse.d/ref/Jobs-group-bys.json

@@ -72,6 +72,9 @@
     "provider": {
         "$ref": "datawarehouse.d/ref/group-by-common.json#/provider"
     },
+    "qos": {
+        "$ref": "datawarehouse.d/ref/group-by-common.json#/qos"
+    },
     "quarter": {
         "$ref": "datawarehouse.d/ref/group-by-time-period.json#/quarter"
     },

configuration/datawarehouse.d/ref/group-by-common.json

@@ -210,6 +210,33 @@
         "description_html": "A service provider is an institution that hosts resources.",
         "name": "Service Provider"
     },
+    "qos": {
+        "attribute_table_schema": "modw",
+        "attribute_to_aggregate_table_key_map": [
+            {
+                "id": "qos_id"
+            }
+        ],
+        "attribute_values_query": {
+            "joins": [
+                {
+                    "name": "qos"
+                }
+            ],
+            "orderby": [
+                "name"
+            ],
+            "records": {
+                "id": "id",
+                "name": "name",
+                "order_id": "name",
+                "short_name": "name"
+            }
+        },
+        "category": "Administrative",
+        "description_html": "The quality of service of the job",
+        "name": "QOS"
+    },
     "queue": {
         "additional_join_constraints": [
             {

configuration/etl/etl.d/hpcdb-xdw.json

@@ -81,6 +81,10 @@
         "name": "principal-investigator",
         "definition_file": "jobs/xdw/principal-investigator.json",
         "description": "principal investigator records"
+    }, {
+        "name": "qos",
+        "definition_file": "jobs/xdw/qos.json",
+        "description": "qos records"
     }, {
         "name": "request",
         "definition_file": "jobs/xdw/request.json",

configuration/etl/etl.d/hpcdb.json

@@ -44,6 +44,7 @@
                 "common/hpcdb/people-on-accounts-history.json",
                 "common/hpcdb/requests.json",
                 "common/hpcdb/principal-investigators.json",
+                "common/hpcdb/qos.json",
                 "common/hpcdb/resource-allocated.json",
                 "common/hpcdb/resource-specs.json",
                 "common/hpcdb/system-accounts.json",
@@ -120,6 +121,11 @@
             "description": "Ingest resource allocated data",
             "definition_file": "common/hpcdb/resource-allocated.json"
         },
+        {
+            "name": "qos",
+            "description": "Ingest QOS",
+            "definition_file": "common/hpcdb/qos.json"
+        },
         {
             "name": "accounts",
             "description": "Ingest accounts",

configuration/etl/etl.d/staging.json

@@ -37,6 +37,7 @@
                 "common/staging/organization.json",
                 "common/staging/pi-resource.json",
                 "common/staging/pi.json",
+                "common/staging/qos.json",
                 "common/staging/resource-config.json",
                 "common/staging/resource-spec.json",
                 "common/staging/resource-type.json",
@@ -200,6 +201,11 @@
             "description": "Ingest job User/PI/Resource permutations",
             "definition_file": "jobs/staging/user-pi-resource.json"
         },
+        {
+            "name": "qos",
+            "description": "Ingest job QOS",
+            "definition_file": "jobs/staging/qos.json"
+        },
         {
             "name": "job",
             "description": "Ingest job data",

configuration/etl/etl.d/xdmod-migration-9_5_0-10_0_0.json

@@ -213,6 +213,98 @@
                     "schema": "mod_logger"
                 }
             }
+        },
+        {
+            "name": "alter-shredded_job_slurm",
+            "description": "Alter mod_shredder.shredded_job_slurm to add qos_name column",
+            "class": "ExecuteSql",
+            "sql_file_list": [
+                {
+                    "sql_file": "migrations/9.5.0-10.0.0/mod_shredder/alter-shredded_job_slurm.sql",
+                    "delimiter": ";"
+                }
+            ],
+            "endpoints": {
+                "destination": {
+                    "type": "mysql",
+                    "name": "Shredder/Staging Database",
+                    "config": "shredder",
+                    "schema": "mod_shredder"
+                }
+            }
+        },
+        {
+            "name": "update-mod_shredder-qos",
+            "description": "Add QOS table and columns to mod_shredder",
+            "class": "ManageTables",
+            "definition_file_list": [
+                "jobs/shredder/job.json",
+                "common/staging/qos.json",
+                "jobs/staging/job.json"
+            ],
+            "endpoints": {
+                "destination": {
+                    "type": "mysql",
+                    "name": "Shredder/Staging Database",
+                    "config": "shredder",
+                    "schema": "mod_shredder"
+                }
+            }
+        },
+        {
+            "name": "update-mod_hpcdb-qos",
+            "description": "Add QOS table and columns to mod_hpcdb",
+            "class": "ManageTables",
+            "definition_file_list": [
+                "common/hpcdb/qos.json",
+                "jobs/hpcdb/jobs.json"
+            ],
+            "endpoints": {
+                "destination": {
+                    "type": "mysql",
+                    "name": "HPCDB Database",
+                    "config": "hpcdb",
+                    "schema": "mod_hpcdb"
+                }
+            }
+        },
+        {
+            "name": "update-modw-qos",
+            "description": "Add QOS table and columns to modw",
+            "class": "ManageTables",
+            "definition_file_list": [
+                "jobs/xdw/job-records.json",
+                "jobs/xdw/qos.json"
+            ],
+            "endpoints": {
+                "destination": {
+                    "type": "mysql",
+                    "name": "Data Warehouse",
+                    "config": "datawarehouse",
+                    "schema": "modw"
+                }
+            }
+        },
+        {
+            "name": "insert-modw-qos",
+            "description": "Insert QOS modw data",
+            "class": "StructuredFileIngestor",
+            "namespace": "ETL\\Ingestor",
+            "options_class": "IngestorOptions",
+            "definition_file": "jobs/xdw/qos.json",
+            "endpoints": {
+                "source": {
+                    "type": "jsonfile",
+                    "name": "unknown qos data",
+                    "path": "jobs/xdw/qos.json"
+                },
+                "destination": {
+                    "type": "mysql",
+                    "name": "Data Warehouse",
+                    "config": "datawarehouse",
+                    "schema": "modw"
+                }
+            }
         }
     ],
     "cloud-migration-9-5-0_10-0-0": [

configuration/etl/etl_action_defs.d/common/hpcdb/qos.json

@@ -0,0 +1,18 @@
+{
+    "table_definition": {
+        "$ref": "${table_definition_dir}/common/hpcdb/qos.json#/table_definition"
+    },
+    "source_query": {
+        "records": {
+            "qos_id": "qos.qos_id",
+            "qos_name": "qos.qos_name"
+        },
+        "joins": [
+            {
+                "schema": "${SOURCE_SCHEMA}",
+                "name": "staging_qos",
+                "alias": "qos"
+            }
+        ]
+    }
+}

configuration/etl/etl_action_defs.d/jobs/hpc-aggregation-day.json

@@ -64,6 +64,7 @@
             "processorbucket_id": "(SELECT id FROM ${UTILITY_SCHEMA}.processor_buckets pb WHERE task.processor_count BETWEEN pb.min_processors AND pb.max_processors)",
             "gpu_count": "task.gpu_count",
             "gpubucket_id": "(SELECT id FROM ${UTILITY_SCHEMA}.gpu_buckets gb WHERE task.gpu_count BETWEEN gb.min_gpus AND gb.max_gpus)",
+            "qos_id": "record.qos_id",
             "submitted_job_count": "SUM( IF(task.submit_time_ts BETWEEN ${:PERIOD_START_TS} AND ${:PERIOD_END_TS}, 1, 0) )",
             "ended_job_count": "SUM( IF(task.end_day_id BETWEEN ${:PERIOD_START_DAY_ID} AND ${:PERIOD_END_DAY_ID}, 1, 0) )",
             "started_job_count": "SUM( IF(task.start_day_id BETWEEN ${:PERIOD_START_DAY_ID} AND ${:PERIOD_END_DAY_ID}, 1, 0) )",
@@ -110,7 +111,8 @@
             "account_id",
             "systemaccount_id",
             "allocation_id",
-            "principalinvestigator_person_id"
+            "principalinvestigator_person_id",
+            "qos_id"
         ],
         "joins": [
             {

configuration/etl/etl_action_defs.d/jobs/hpc-aggregation.json

@@ -64,6 +64,7 @@
             "processorbucket_id": "(SELECT id FROM ${UTILITY_SCHEMA}.processor_buckets pb WHERE task.processor_count BETWEEN pb.min_processors AND pb.max_processors)",
             "gpu_count": "task.gpu_count",
             "gpubucket_id": "(SELECT id FROM ${UTILITY_SCHEMA}.gpu_buckets gb WHERE task.gpu_count BETWEEN gb.min_gpus AND gb.max_gpus)",
+            "qos_id": "record.qos_id",
             "submitted_job_count": "SUM( IF(task.submit_time_ts BETWEEN ${:PERIOD_START_TS} AND ${:PERIOD_END_TS}, 1, 0) )",
             "ended_job_count": "SUM( IF(task.end_day_id BETWEEN ${:PERIOD_START_DAY_ID} AND ${:PERIOD_END_DAY_ID}, 1, 0) )",
             "started_job_count": "SUM( IF(task.start_day_id BETWEEN ${:PERIOD_START_DAY_ID} AND ${:PERIOD_END_DAY_ID}, 1, 0) )",
@@ -109,7 +110,8 @@
             "account_id",
             "systemaccount_id",
             "allocation_id",
-            "principalinvestigator_person_id"
+            "principalinvestigator_person_id",
+            "qos_id"
         ],
         "joins": [
             {

configuration/etl/etl_action_defs.d/jobs/hpcdb/jobs.json

@@ -9,6 +9,7 @@
             "resource_id": "r.resource_id",
             "allocation_breakdown_id": "upr.user_pi_resource_id",
             "allocation_id": "pr.pi_resource_id",
+            "qos_id": "IFNULL(q.qos_id, -1)",
             "account_id": "p.pi_id",
             "local_jobid": "j.job_id",
             "local_job_array_index": "IFNULL(j.job_array_index, -1)",
@@ -75,6 +76,13 @@
                 "alias": "upr",
                 "type": "LEFT",
                 "on": "j.user_name = upr.user_name AND j.pi_name = upr.pi_name AND j.resource_name = upr.resource_name"
+            },
+            {
+                "name": "staging_qos",
+                "schema": "${SOURCE_SCHEMA}",
+                "alias": "q",
+                "type": "LEFT",
+                "on": "j.qos_name = q.qos_name"
             }
         ],
         "where": [

configuration/etl/etl_action_defs.d/jobs/staging/job.json

@@ -16,6 +16,7 @@
             "group_name": "group_name",
             "gid_number": "gid_number",
             "pi_name": "pi_name",
+            "qos_name": "qos_name",
             "account_name": "account_name",
             "project_name": "project_name",
             "start_time": "start_time",

configuration/etl/etl_action_defs.d/jobs/staging/qos.json

@@ -0,0 +1,20 @@
+{
+    "table_definition": {
+        "$ref": "${table_definition_dir}/common/staging/qos.json#/table_definition"
+    },
+    "source_query": {
+        "records": {
+            "qos_name": "DISTINCT j.qos_name"
+        },
+        "joins": [
+            {
+                "name": "shredded_job",
+                "schema": "${SOURCE_SCHEMA}",
+                "alias": "j"
+            }
+        ],
+        "where": [
+            "j.qos_name IS NOT NULL"
+        ]
+    }
+}

configuration/etl/etl_action_defs.d/jobs/xdw/job-records-job-tasks.json

@@ -31,6 +31,7 @@
             "local_job_array_index": "j.local_job_array_index",
             "local_job_id_raw": "j.local_job_id_raw",
             "job_name": "j.jobname",
+            "qos_id": "j.qos_id",
             "queue": "COALESCE(j.queue, 'NA')",
             "node_count": "j.nodecount",
             "systemaccount_id": "sa.system_account_id",
@@ -179,6 +180,7 @@
             "allocation_id": "allocation_id",
             "allocation_resource_id": "allocation_resource_id",
             "fos_id": "fos_id",
+            "qos_id": "qos_id",
             "queue": "queue",
             "person_id": "person_id",
             "person_organization_id": "person_organization_id",

configuration/etl/etl_action_defs.d/jobs/xdw/qos.json

@@ -0,0 +1,18 @@
+{
+    "table_definition": {
+        "$ref": "${table_definition_dir}/jobs/xdw/qos.json#/table_definition"
+    },
+    "source_query": {
+        "records": {
+            "id": "j.qos_id",
+            "name": "j.qos_name"
+        },
+        "joins": [
+            {
+                "name": "hpcdb_qos",
+                "schema": "${SOURCE_SCHEMA}",
+                "alias": "j"
+            }
+        ]
+    }
+}

configuration/etl/etl_data.d/jobs/xdw/qos.json

@@ -0,0 +1,4 @@
+[
+    ["id", "name"],
+    [-1, "Unknown"]
+]

configuration/etl/etl_pipelines.d/jobs-xdw.json

@@ -35,6 +35,7 @@
                 "jobs/xdw/processor-buckets.json",
                 "jobs/xdw/quarters.json",
                 "jobs/xdw/queue.json",
+                "jobs/xdw/qos.json",
                 "jobs/xdw/request.json",
                 "jobs/xdw/resource-allocated.json",
                 "jobs/xdw/resource-fact.json",
@@ -143,6 +144,18 @@
                 }
             }
         },
+        {
+            "name": "qos",
+            "definition_file": "jobs/xdw/qos.json",
+            "description": "QOS data",
+            "endpoints": {
+                "source": {
+                    "type": "jsonfile",
+                    "name": "unknown qos data",
+                    "path": "jobs/xdw/qos.json"
+                }
+            }
+        },
         {
             "name": "schema-version",
             "definition_file": "jobs/xdw/schema-version-history.json",

configuration/etl/etl_sql.d/jobs/shredder/job-slurm.sql

@@ -8,6 +8,7 @@ CREATE TABLE ${DESTINATION_SCHEMA}.`shredded_job_slurm` (
   `job_name` tinytext NOT NULL,
   `cluster_name` tinytext NOT NULL,
   `partition_name` tinytext NOT NULL,
+  `qos_name` tinytext DEFAULT NULL,
   `user_name` tinytext NOT NULL,
   `uid_number` int(10) unsigned DEFAULT NULL,
   `group_name` tinytext NOT NULL,