Merge pull request #1272 from eiffel777/cloud-set-unknown-dimension-t…

…o-negative-1

changing account, instance and instance_type tables to have global un…

configuration/etl/etl.d/jobs_cloud_common.json

@@ -40,6 +40,7 @@
             "namespace": "ETL\\Maintenance",
             "options_class": "MaintenanceOptions",
             "definition_file_list": [
+                "cloud_common/account.json",
                 "cloud_common/event.json",
                 "cloud_common/asset.json",
                 "cloud_common/instance_data.json",
@@ -97,6 +98,49 @@
         {
             "#": "Note that any actions run after this cannot truncate the tables set here",
 
+            "name": "CloudAccountUnknownInitializer",
+            "description": "Initialize values for unknown hosts, images, etc. that are specific to a resource",
+            "class": "StructuredFileIngestor",
+            "definition_file": "cloud_common/account.json",
+            "endpoints": {
+                "source": {
+                    "type": "jsonfile",
+                    "name": "Cloud record types",
+                    "path": "cloud_common/account.json"
+                }
+            }
+        },
+        {
+            "#": "Note that any actions run after this cannot truncate the tables set here",
+
+            "name": "CloudInstanceUnknownInitializer",
+            "description": "Initialize values for unknown hosts, images, etc. that are specific to a resource",
+            "class": "StructuredFileIngestor",
+            "definition_file": "cloud_common/instance.json",
+            "endpoints": {
+                "source": {
+                    "type": "jsonfile",
+                    "name": "Cloud record types",
+                    "path": "cloud_common/instance.json"
+                }
+            }
+        },
+        {
+            "name": "CloudInstanceTypeUnknownInitializer",
+            "description": "Initialize values for unknown hosts, images, etc. that are specific to a resource",
+            "class": "StructuredFileIngestor",
+            "definition_file": "cloud_common/instance_type.json",
+            "endpoints": {
+                "source": {
+                    "type": "jsonfile",
+                    "name": "Cloud record types",
+                    "path": "cloud_common/instance_type.json"
+                }
+            }
+        },
+        {
+            "#": "Note that any actions run after this cannot truncate the tables set here",
+
             "name": "CloudUnknownDimensionInitializer",
             "description": "Initialize values for unknown hosts, images, etc. that are specific to a resource",
             "class": "DatabaseIngestor",

configuration/etl/etl_action_defs.d/cloud_common/account.json

@@ -0,0 +1,8 @@
+{
+    "#": "This action loads data from a structured file and only needs the table definition",
+    "table_definition": [
+        {
+            "$ref": "${table_definition_dir}/cloud_common/account.json#/table_definition"
+        }
+    ]
+}

configuration/etl/etl_action_defs.d/cloud_common/initialize_unknown_dimension_values.json

@@ -14,20 +14,11 @@
     "#": "the first part of the key and not -1. We will continue to use -1 for global unknowns.",
 
     "table_definition": [
-        {
-            "$ref": "${table_definition_dir}/cloud_common/account.json#/table_definition"
-        },
         {
             "$ref": "${table_definition_dir}/cloud_common/host.json#/table_definition"
         },
         {
             "$ref": "${table_definition_dir}/cloud_common/image.json#/table_definition"
-        },
-        {
-            "$ref": "${table_definition_dir}/cloud_common/instance.json#/table_definition"
-        },
-        {
-            "$ref": "${table_definition_dir}/cloud_common/instance_type.json#/table_definition"
         }
     ],
 
@@ -61,12 +52,6 @@
     },
 
     "destination_record_map": {
-        "account": {
-            "resource_id": "resource_id",
-            "account_id": "unknown_id",
-            "provider_account": "unknown_key",
-            "display": "unknown_display"
-        },
         "host": {
             "resource_id": "resource_id",
             "host_id": "unknown_id",
@@ -76,18 +61,6 @@
             "resource_id": "resource_id",
             "image_id": "unknown_id",
             "image": "unknown_key"
-        },
-        "instance": {
-            "resource_id": "resource_id",
-            "instance_id": "unknown_id",
-            "account_id": "unknown_id",
-            "provider_identifier": "unknown_key"
-        },
-        "instance_type": {
-            "resource_id": "resource_id",
-            "instance_type_id": "unknown_id",
-            "instance_type": "unknown_key",
-            "display": "unknown_display"
         }
     }
 }

configuration/etl/etl_action_defs.d/cloud_common/instance.json

@@ -0,0 +1,8 @@
+{
+    "#": "This action loads data from a structured file and only needs the table definition",
+    "table_definition": [
+        {
+            "$ref": "${table_definition_dir}/cloud_common/instance.json#/table_definition"
+        }
+    ]
+}

configuration/etl/etl_action_defs.d/cloud_common/instance_type.json

@@ -0,0 +1,8 @@
+{
+    "#": "This action loads data from a structured file and only needs the table definition",
+    "table_definition": [
+        {
+            "$ref": "${table_definition_dir}/cloud_common/instance_type.json#/table_definition"
+        }
+    ]
+}

configuration/etl/etl_data.d/cloud_common/account.json

@@ -0,0 +1,4 @@
+[
+    ["resource_id", "account_id", "provider_account", "display"],
+    [-1,            1,            "Unknown",          "Unknown"]
+]

configuration/etl/etl_data.d/cloud_common/instance.json

@@ -0,0 +1,4 @@
+[
+    ["resource_id", "instance_id", "account_id", "provider_identifier", "person_id"],
+    [-1,            1,              1,           "unknown",             -1]
+]

configuration/etl/etl_data.d/cloud_common/instance_type.json

@@ -0,0 +1,4 @@
+[
+    ["resource_id", "instance_type_id", "instance_type", "display", "description", "num_cores", "memory_mb", "disk_gb", "start_time", "end_time"],
+    [-1,            1,                  "Unknown",       "Unknown", null,            0,           0,           0,         0,            null]
+]