Ensure resourcetypes table is correctly populated.

classes/OpenXdmod/DataWarehouseInitializer.php

@@ -148,7 +148,7 @@ public function ingestAllShredded($startDate = null, $endDate = null)
         }
 
         $this->logger->debug('Ingesting shredded data to staging tables');
-        Utilities::runEtlPipeline(array('staging-ingest-common', 'staging-ingest-jobs', 'ingest-resource-types'), $this->logger);
+        Utilities::runEtlPipeline(array('staging-ingest-common', 'staging-ingest-jobs'), $this->logger);
     }
 
     /**

configuration/etl/etl.d/staging.json

@@ -95,6 +95,16 @@
                 }
             }
         },
+        {
+            "name": "ResourceTypesStagingUnknown",
+            "description": "Populates the resource types table w/ the Unknown resource",
+            "namespace": "ETL\\Maintenance",
+            "class": "ExecuteSql",
+            "options_class": "MaintenanceOptions",
+            "sql_file_list": [
+                "cloud_openstack/unknown_resource_type.sql"
+            ]
+        },
         {
             "name": "resource-config",
             "description": "Ingest resource configuration file",

configuration/etl/etl.d/xdmod-migration-8_1_2-8_5_0.json

@@ -209,6 +209,69 @@
                 }
             }
         },
+        {
+            "name": "ResourceTypesStagingUnknown",
+            "description": "Populates the resource types table w/ the Unknown resource",
+            "namespace": "ETL\\Maintenance",
+            "class": "ExecuteSql",
+            "options_class": "MaintenanceOptions",
+            "sql_file_list": [
+                "cloud_openstack/unknown_resource_type.sql"
+            ],
+            "endpoints": {
+                "destination": {
+                    "type": "mysql",
+                    "name": "Shredder/Staging Database",
+                    "config": "database",
+                    "schema": "mod_shredder",
+                    "definition_file": "common/staging/resource-type-realms.json"
+                }
+            }
+        },
+        {
+            "name": "ResourceTypesHpcdb",
+            "description": "Ingest resource types",
+            "definition_file": "common/hpcdb/resource-types.json",
+            "namespace": "ETL\\Ingestor",
+            "class": "DatabaseIngestor",
+            "options_class": "IngestorOptions",
+            "endpoints": {
+                "source": {
+                    "type": "mysql",
+                    "name": "Staging tables",
+                    "config": "database",
+                    "schema": "mod_shredder"
+                },
+                "destination": {
+                    "type": "mysql",
+                    "name": "HPCDB Database",
+                    "config": "database",
+                    "schema": "mod_hpcdb"
+                }
+            }
+        },
+        {
+            "name": "ResourceTypesDatawarehouse",
+            "definition_file": "jobs/xdw/resource-type.json",
+            "namespace": "ETL\\Ingestor",
+            "description": "resource type records",
+            "class": "DatabaseIngestor",
+            "options_class": "IngestorOptions",
+            "endpoints": {
+                "source": {
+                    "type": "mysql",
+                    "name": "HPCDB",
+                    "config": "datawarehouse",
+                    "schema": "mod_hpcdb"
+                },
+                "destination": {
+                    "type": "mysql",
+                    "name": "Datawarehouse",
+                    "config": "datawarehouse",
+                    "schema": "modw"
+                }
+            }
+        },
         {
             "class": "DatabaseIngestor",
             "name": "IngestInitialRealms",

configuration/etl/etl_sql.d/cloud_openstack/unknown_resource_type.sql

@@ -1,2 +1,3 @@
-INSERT INTO ${DESTINATION_SCHEMA}.`staging_resource_type` (resource_type_id, resource_type_description, resource_type_abbrev)
-VALUES (-1, 'Unknown Resource Type', 'UNK');
+SET SESSION sql_mode='NO_AUTO_VALUE_ON_ZERO';
+INSERT IGNORE INTO ${DESTINATION_SCHEMA}.`staging_resource_type` (resource_type_id, resource_type_description, resource_type_abbrev)
+VALUES ('0', 'Unknown Resource Type', 'UNK');

tests/ci/validate.sh

@@ -20,4 +20,12 @@ do
     fi
 done
 
+# Check that the unknown resource is in the database with key 0
+unkrescount=$(echo 'SELECT COUNT(*) from resourcetype WHERE id = 0 and abbrev = '"'"'UNK'"'" | mysql -N modw)
+if [ $unkrescount -ne 1 ];
+then
+    echo "Missing / inconsistent 'UNK' row in modw.resourcetype"
+    exitcode=1
+fi
+
 exit $exitcode