Merge pull request #191 from Sanketika-Obsrv/#234api-integration

Sanketika-Obsrv/issue-tracker#OBS-143: fix: set denorm fields to empty on reupload schema
Sanketika-Obsrv · Jun 19, 2024 · a9fc4af · a9fc4af
2 parents 6866545 + e9faf55
commit a9fc4af
Show file tree

Hide file tree

Showing 4 changed files with 16 additions and 11 deletions.
diff --git a/api-service/src/v2/configs/Config.ts b/api-service/src/v2/configs/Config.ts
@@ -49,8 +49,10 @@ export const config = {
     masterDataset: "master-dataset"
   },
   "redis_config": {
-    "redis_host": process.env.redis_host || "localhost",
-    "redis_port": process.env.redis_port || 6379
+    "denorm_redis_host": process.env.denorm_redis_host,
+    "denorm_redis_port": parseInt(process.env.denorm_redis_port as string),
+    "dedup_redis_host": process.env.dedup_redis_host,
+    "dedup_redis_port": parseInt(process.env.dedup_redis_port as string)
   },
   "exclude_datasource_validation": process.env.exclude_datasource_validation ? process.env.exclude_datasource_validation.split(",") : ["system-stats", "failed-events-summary", "masterdata-system-stats", "system-events"], // list of datasource names to skip validation while calling query API
   "telemetry_dataset": process.env.telemetry_dataset || `${env}.system.telemetry.events`,

diff --git a/api-service/src/v2/configs/DatasetConfigDefault.ts b/api-service/src/v2/configs/DatasetConfigDefault.ts
@@ -22,8 +22,8 @@ export const defaultMasterConfig = {
         "dedup_period": 604800, // 7 days
     },
     "denorm_config": {
-        "redis_db_host": config.redis_config.redis_host,
-        "redis_db_port": config.redis_config.redis_port,
+        "redis_db_host": config.redis_config.denorm_redis_host,
+        "redis_db_port": config.redis_config.denorm_redis_port,
         "denorm_fields": []
     },
     "router_config": {
@@ -34,8 +34,8 @@ export const defaultMasterConfig = {
         "data_key": "",
         "timestamp_key": ingestionConfig.indexCol["Event Arrival Time"],
         "entry_topic": config.telemetry_service_config.kafka.topics.createMasterDataset,
-        "redis_db_host": config.redis_config.redis_host,
-        "redis_db_port": config.redis_config.redis_port,
+        "redis_db_host": config.redis_config.denorm_redis_host,
+        "redis_db_port": config.redis_config.denorm_redis_port,
         "index_data": true,
         "redis_db": 3,
         "file_upload_path": []
@@ -66,8 +66,8 @@ export const defaultDatasetConfig = {
         "dedup_period": 604800, // 7 days
     },
     "denorm_config": {
-        "redis_db_host": config.redis_config.redis_host,
-        "redis_db_port": config.redis_config.redis_port,
+        "redis_db_host": config.redis_config.denorm_redis_host,
+        "redis_db_port": config.redis_config.denorm_redis_port,
         "denorm_fields": []
     },
     "router_config": {
@@ -78,8 +78,8 @@ export const defaultDatasetConfig = {
         "data_key": "",
         "timestamp_key": ingestionConfig.indexCol["Event Arrival Time"],
         "entry_topic": config.telemetry_service_config.kafka.topics.createDataset,
-        "redis_db_host": config.redis_config.redis_host,
-        "redis_db_port": config.redis_config.redis_port,
+        "redis_db_host": config.redis_config.dedup_redis_host,
+        "redis_db_port": config.redis_config.dedup_redis_port,
         "index_data": true,
         "redis_db": 0,
         "file_upload_path": []

diff --git a/api-service/src/v2/controllers/DatasetStatusTransition/ReadyToPublishSchema.json b/api-service/src/v2/controllers/DatasetStatusTransition/ReadyToPublishSchema.json
@@ -201,7 +201,6 @@
             "required": [
               "denorm_key",
               "denorm_out_field",
-              "dataset_name",
               "dataset_id"
             ],
             "additionalProperties": false

diff --git a/api-service/src/v2/controllers/DatasetUpdate/DatasetUpdate.ts b/api-service/src/v2/controllers/DatasetUpdate/DatasetUpdate.ts
@@ -345,6 +345,10 @@ const setDenormConfigs = (newDenormPayload: Record<string, any>, datasetDenormCo
     const { denorm_fields } = newDenormPayload;
     const existingDenormFields = _.get(datasetDenormConfigs, "denorm_fields") || []
 
+    if (_.isEmpty(denorm_fields)) {
+        return { denorm_fields }
+    }
+
     const getDenormPayload = (action: string) => {
         return _.compact(_.flatten(_.map(denorm_fields, payload => {
             if (payload.action == action) return payload.values